@jmoyers/harness 0.1.11 → 0.1.20

package/src/store/control-plane-store.ts

@@ -3,7 +3,6 @@ import { mkdirSync } from 'node:fs';
 import { dirname } from 'node:path';
 import { randomUUID } from 'node:crypto';
 import {
-  applyTaskLinearInput,
   normalizeGitHubPrJobRow,
   normalizeGitHubPullRequestRow,
   normalizeGitHubSyncStateRow,
@@ -13,7 +12,6 @@ import {
   normalizeProjectSettingsRow,
   asString,
   asStringOrNull,
-  defaultTaskLinearRecord,
   normalizeNonEmptyLabel,
   normalizeRepositoryRow,
   normalizeStoredConversationRow,
@@ -23,7 +21,6 @@ import {
   normalizeTaskRow,
   normalizeTelemetryRow,
   normalizeTelemetrySource,
-  serializeTaskLinear,
   sqliteStatementChanges,
   uniqueValues,
 } from './control-plane-store-normalize.ts';
@@ -40,13 +37,11 @@ import type {
   ControlPlaneProjectTaskFocusMode,
   ControlPlaneProjectThreadSpawnMode,
   ControlPlaneRepositoryRecord,
-  ControlPlaneTaskLinearRecord,
   ControlPlaneTaskRecord,
   ControlPlaneTaskScopeKind,
   ControlPlaneTaskStatus,
   ControlPlaneTelemetryRecord,
   ControlPlaneTelemetrySummary,
-  TaskLinearInput,
 } from './control-plane-store-types.ts';
 import type { PtyExit } from '../pty/pty_host.ts';
 import type { CodexTelemetrySource } from '../control-plane/codex-telemetry.ts';
@@ -64,7 +59,7 @@ const DEFAULT_RUNTIME_STATUS_MODEL_JSON = JSON.stringify({
   attentionReason: null,
   lastKnownWork: null,
   lastKnownWorkAt: null,
-  phaseHint: null,
+  activityHint: null,
   observedAt: new Date(0).toISOString(),
 } satisfies StreamSessionStatusModel | null);
 
@@ -89,11 +84,28 @@ function initialRuntimeStatusModel(
     attentionReason: null,
     lastKnownWork: null,
     lastKnownWorkAt: null,
-    phaseHint: null,
+    activityHint: null,
     observedAt,
   };
 }
 
+function normalizeTaskTitle(value: string | null | undefined): string {
+  if (value === undefined || value === null) {
+    return '';
+  }
+  return value.trim();
+}
+
+function normalizeTaskBody(value: string, field: string): string {
+  if (typeof value !== 'string') {
+    throw new Error(`expected string for ${field}`);
+  }
+  if (value.trim().length === 0) {
+    throw new Error(`${field} must be non-empty`);
+  }
+  return value;
+}
+
 export type {
   ControlPlaneAutomationPolicyRecord,
   ControlPlaneAutomationPolicyScope,
@@ -107,7 +119,6 @@ export type {
   ControlPlaneProjectTaskFocusMode,
   ControlPlaneProjectThreadSpawnMode,
   ControlPlaneRepositoryRecord,
-  ControlPlaneTaskLinearRecord,
   ControlPlaneTaskRecord,
   ControlPlaneTaskScopeKind,
   ControlPlaneTelemetryRecord,
@@ -204,17 +215,15 @@ interface CreateTaskInput {
   workspaceId: string;
   repositoryId?: string;
   projectId?: string;
-  title: string;
-  description?: string;
-  linear?: TaskLinearInput;
+  title?: string | null;
+  body?: string;
 }
 
 interface UpdateTaskInput {
-  title?: string;
-  description?: string;
+  title?: string | null;
+  body?: string;
   repositoryId?: string | null;
   projectId?: string | null;
-  linear?: TaskLinearInput | null;
 }
 
 interface ListTaskQuery {
@@ -356,15 +365,35 @@ interface ListGitHubSyncStateQuery {
 }
 
 const CONTROL_PLANE_SCHEMA_VERSION = 1;
+const TELEMETRY_COMPACTION_SHADOW_TABLE = 'session_telemetry_compaction_shadow';
+const TELEMETRY_COMPACTION_OLD_TABLE = 'session_telemetry_compaction_old';
+
+interface OnlineCopyForwardCompactionStepResult {
+  readonly state: 'idle' | 'copying' | 'finalized';
+  readonly copiedRows: number;
+}
 
 export class SqliteControlPlaneStore {
   private readonly db: DatabaseSync;
+  private readonly inMemory: boolean;
+  private telemetryCopyForwardRequested = false;
+  private telemetryCopyForwardActive = false;
+  private telemetryCopyForwardCursorRowId = 0;
+  private readonly busyTimeoutMs: number;
 
-  constructor(filePath = ':memory:') {
+  constructor(filePath = ':memory:', options?: { busyTimeoutMs?: number }) {
     const resolvedPath = this.preparePath(filePath);
+    this.inMemory = resolvedPath === ':memory:';
+    this.busyTimeoutMs =
+      typeof options?.busyTimeoutMs === 'number' &&
+      Number.isFinite(options.busyTimeoutMs) &&
+      options.busyTimeoutMs > 0
+        ? Math.floor(options.busyTimeoutMs)
+        : 5000;
     this.db = new DatabaseSync(resolvedPath);
     this.configureConnection();
     this.initializeSchema();
+    this.ensureIncrementalAutoVacuumMode();
   }
 
   close(): void {
@@ -884,6 +913,144 @@ export class SqliteControlPlaneStore {
     return sqliteStatementChanges(result) > 0;
   }
 
+  pruneTelemetryOlderThan(cutoffIngestedAt: string, limit = 1000): number {
+    const safeLimit = Number.isFinite(limit) ? Math.max(1, Math.floor(limit)) : 1000;
+    const result = this.db
+      .prepare(
+        `
+        DELETE FROM session_telemetry
+        WHERE telemetry_id IN (
+          SELECT telemetry_id
+          FROM session_telemetry
+          WHERE ingested_at < ?
+          ORDER BY telemetry_id ASC
+          LIMIT ?
+        )
+      `,
+      )
+      .run(cutoffIngestedAt, safeLimit);
+    const changes = sqliteStatementChanges(result);
+    if (changes > 0) {
+      this.telemetryCopyForwardRequested = true;
+    }
+    return changes;
+  }
+
+  countTelemetryOlderThan(cutoffIngestedAt: string): number {
+    const row = this.db
+      .prepare(
+        `
+        SELECT COUNT(*) AS count
+        FROM session_telemetry
+        WHERE ingested_at < ?
+      `,
+      )
+      .get(cutoffIngestedAt);
+    if (row === undefined) {
+      return 0;
+    }
+    const asRow = asRecord(row);
+    const count = asNumberOrNull(asRow.count, 'count');
+    return count ?? 0;
+  }
+
+  checkpointWal(mode: 'PASSIVE' | 'TRUNCATE' = 'PASSIVE'): void {
+    this.db.exec(`PRAGMA wal_checkpoint(${mode});`);
+  }
+
+  compactFreelistPages(maxPages: number): void {
+    const safeMaxPages = Number.isFinite(maxPages) ? Math.max(1, Math.floor(maxPages)) : 1;
+    this.db.exec(`PRAGMA incremental_vacuum(${String(safeMaxPages)});`);
+  }
+
+  runOnlineCopyForwardCompactionStep(
+    batchSize = 5000,
+    finalizeTailRows = 1200,
+  ): OnlineCopyForwardCompactionStepResult {
+    if (this.inMemory) {
+      return {
+        state: 'idle',
+        copiedRows: 0,
+      };
+    }
+
+    const safeBatchSize = Number.isFinite(batchSize) ? Math.max(1, Math.floor(batchSize)) : 5000;
+    const safeFinalizeTailRows = Number.isFinite(finalizeTailRows)
+      ? Math.max(1, Math.floor(finalizeTailRows))
+      : 1200;
+
+    if (!this.telemetryCopyForwardActive) {
+      if (!this.telemetryCopyForwardRequested) {
+        return { state: 'idle', copiedRows: 0 };
+      }
+      if (this.countTotalTelemetryRows() === 0) {
+        this.telemetryCopyForwardRequested = false;
+        return { state: 'idle', copiedRows: 0 };
+      }
+      this.db.exec('BEGIN IMMEDIATE TRANSACTION');
+      try {
+        this.resetTelemetryCompactionShadowTable();
+        this.db.exec('COMMIT');
+      } catch (error) {
+        this.db.exec('ROLLBACK');
+        throw error;
+      }
+      this.telemetryCopyForwardActive = true;
+      this.telemetryCopyForwardCursorRowId = 0;
+    }
+
+    this.db.exec('BEGIN IMMEDIATE TRANSACTION');
+    let copiedRows: number;
+    let remainingRows: number;
+    try {
+      copiedRows = this.copyTelemetryCompactionBatch(
+        this.telemetryCopyForwardCursorRowId,
+        safeBatchSize,
+      );
+      if (copiedRows > 0) {
+        this.telemetryCopyForwardCursorRowId = this.readTelemetryCompactionShadowCursorRowId();
+      }
+      remainingRows = this.countTelemetryRowsAfterId(this.telemetryCopyForwardCursorRowId);
+      this.db.exec('COMMIT');
+    } catch (error) {
+      this.db.exec('ROLLBACK');
+      this.resetTelemetryCompactionStateAfterFailure();
+      throw error;
+    }
+
+    if (remainingRows > safeFinalizeTailRows) {
+      return { state: 'copying', copiedRows };
+    }
+
+    this.db.exec('BEGIN IMMEDIATE TRANSACTION');
+    try {
+      const tailCopied = this.copyTelemetryCompactionBatch(
+        this.telemetryCopyForwardCursorRowId,
+        safeFinalizeTailRows,
+      );
+      if (tailCopied > 0) {
+        this.telemetryCopyForwardCursorRowId = this.readTelemetryCompactionShadowCursorRowId();
+      }
+      const postTailRemaining = this.countTelemetryRowsAfterId(
+        this.telemetryCopyForwardCursorRowId,
+      );
+      if (postTailRemaining > 0) {
+        this.db.exec('COMMIT');
+        return { state: 'copying', copiedRows: copiedRows + tailCopied };
+      }
+      this.swapInTelemetryCompactionShadowTable();
+      this.telemetryCopyForwardRequested = false;
+      this.telemetryCopyForwardActive = false;
+      this.telemetryCopyForwardCursorRowId = 0;
+      this.db.exec('COMMIT');
+      return { state: 'finalized', copiedRows: copiedRows + tailCopied };
+    } catch (error) {
+      this.db.exec('ROLLBACK');
+      this.resetTelemetryCompactionStateAfterFailure();
+      throw error;
+    }
+  }
+
   latestTelemetrySummary(sessionId: string): ControlPlaneTelemetrySummary | null {
     const row = this.db
       .prepare(
@@ -1266,9 +1433,8 @@ export class SqliteControlPlaneStore {
   }
 
   createTask(input: CreateTaskInput): ControlPlaneTaskRecord {
-    const title = normalizeNonEmptyLabel(input.title, 'title');
-    const description = input.description ?? '';
-    const linear = applyTaskLinearInput(defaultTaskLinearRecord(), input.linear ?? {});
+    const title = normalizeTaskTitle(input.title);
+    const body = normalizeTaskBody(input.body ?? title, 'body');
     this.db.exec('BEGIN IMMEDIATE TRANSACTION');
     try {
       const existing = this.getTask(input.taskId);
@@ -1277,6 +1443,9 @@ export class SqliteControlPlaneStore {
       }
       const repositoryId = input.repositoryId ?? null;
       const projectId = input.projectId ?? null;
+      if (repositoryId === null && projectId === null) {
+        throw new Error('task scope required: repositoryId or projectId');
+      }
       if (repositoryId !== null) {
         const repository = this.getActiveRepository(repositoryId);
         this.assertScopeMatch(input, repository, 'task');
@@ -1300,8 +1469,7 @@ export class SqliteControlPlaneStore {
             scope_kind,
             project_id,
             title,
-            description,
-            linear_json,
+            body,
             status,
             order_index,
             claimed_by_controller_id,
@@ -1312,7 +1480,7 @@ export class SqliteControlPlaneStore {
             completed_at,
             created_at,
             updated_at
-          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'draft', ?, NULL, NULL, NULL, NULL, NULL, NULL, ?, ?)
+          ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'draft', ?, NULL, NULL, NULL, NULL, NULL, NULL, ?, ?)
         `,
         )
         .run(
@@ -1324,8 +1492,7 @@ export class SqliteControlPlaneStore {
           scopeKind,
           projectId,
           title,
-          description,
-          serializeTaskLinear(linear),
+          body,
           orderIndex,
           createdAt,
           createdAt,
@@ -1355,8 +1522,7 @@ export class SqliteControlPlaneStore {
           scope_kind,
           project_id,
           title,
-          description,
-          linear_json,
+          body,
           status,
           order_index,
           claimed_by_controller_id,
@@ -1423,8 +1589,7 @@ export class SqliteControlPlaneStore {
           scope_kind,
           project_id,
           title,
-          description,
-          linear_json,
+          body,
           status,
           order_index,
           claimed_by_controller_id,
@@ -1450,19 +1615,14 @@ export class SqliteControlPlaneStore {
     if (existing === null) {
       return null;
     }
-    const title =
-      update.title === undefined ? existing.title : normalizeNonEmptyLabel(update.title, 'title');
-    const description =
-      update.description === undefined ? existing.description : update.description;
+    const title = update.title === undefined ? existing.title : normalizeTaskTitle(update.title);
+    const body = update.body === undefined ? existing.body : normalizeTaskBody(update.body, 'body');
     const repositoryId =
       update.repositoryId === undefined ? existing.repositoryId : update.repositoryId;
     const projectId = update.projectId === undefined ? existing.projectId : update.projectId;
-    const linear =
-      update.linear === undefined
-        ? existing.linear
-        : update.linear === null
-          ? defaultTaskLinearRecord()
-          : applyTaskLinearInput(existing.linear, update.linear);
+    if (repositoryId === null && projectId === null) {
+      throw new Error('task scope required: repositoryId or projectId');
+    }
     if (repositoryId !== null) {
       const repository = this.getActiveRepository(repositoryId);
       this.assertScopeMatch(existing, repository, 'task');
@@ -1482,22 +1642,12 @@ export class SqliteControlPlaneStore {
           scope_kind = ?,
           project_id = ?,
           title = ?,
-          description = ?,
-          linear_json = ?,
+          body = ?,
           updated_at = ?
         WHERE task_id = ?
       `,
       )
-      .run(
-        repositoryId,
-        scopeKind,
-        projectId,
-        title,
-        description,
-        serializeTaskLinear(linear),
-        updatedAt,
-        taskId,
-      );
+      .run(repositoryId, scopeKind, projectId, title, body, updatedAt, taskId);
     return this.getTask(taskId);
   }
 
@@ -2756,8 +2906,7 @@ export class SqliteControlPlaneStore {
         scope_kind TEXT NOT NULL DEFAULT 'global',
         project_id TEXT REFERENCES directories(directory_id),
         title TEXT NOT NULL,
-        description TEXT NOT NULL DEFAULT '',
-        linear_json TEXT NOT NULL DEFAULT '{}',
+        body TEXT NOT NULL DEFAULT '',
         status TEXT NOT NULL,
         order_index INTEGER NOT NULL,
         claimed_by_controller_id TEXT,
@@ -2774,13 +2923,20 @@ export class SqliteControlPlaneStore {
       CREATE INDEX IF NOT EXISTS idx_tasks_scope
       ON tasks (tenant_id, user_id, workspace_id, order_index, created_at, task_id);
     `);
-    this.ensureColumnExists('tasks', 'linear_json', `linear_json TEXT NOT NULL DEFAULT '{}'`);
     this.ensureColumnExists('tasks', 'scope_kind', `scope_kind TEXT NOT NULL DEFAULT 'global'`);
     this.ensureColumnExists(
       'tasks',
       'project_id',
       `project_id TEXT REFERENCES directories(directory_id)`,
     );
+    this.ensureColumnExists('tasks', 'body', `body TEXT NOT NULL DEFAULT ''`);
+    if (this.columnExists('tasks', 'description')) {
+      this.db.exec(`
+        UPDATE tasks
+        SET body = description
+        WHERE (body IS NULL OR TRIM(body) = '') AND description IS NOT NULL
+      `);
+    }
     this.db.exec(`
       CREATE INDEX IF NOT EXISTS idx_tasks_scope_kind
       ON tasks (tenant_id, user_id, workspace_id, scope_kind, repository_id, project_id, order_index);
@@ -2803,6 +2959,10 @@ export class SqliteControlPlaneStore {
       SET scope_kind = 'repository'
       WHERE scope_kind = 'global' AND repository_id IS NOT NULL AND project_id IS NULL;
     `);
+    this.db.exec(`
+      DELETE FROM tasks
+      WHERE repository_id IS NULL AND project_id IS NULL;
+    `);
 
     this.db.exec(`
       CREATE TABLE IF NOT EXISTS project_settings (
@@ -2960,18 +3120,162 @@ export class SqliteControlPlaneStore {
   }
 
   private configureConnection(): void {
+    this.db.exec('PRAGMA auto_vacuum = INCREMENTAL;');
     this.db.exec('PRAGMA journal_mode = WAL;');
     this.db.exec('PRAGMA synchronous = NORMAL;');
-    this.db.exec('PRAGMA busy_timeout = 2000;');
+    this.db.exec(`PRAGMA busy_timeout = ${String(this.busyTimeoutMs)};`);
   }
 
-  private ensureColumnExists(table: string, column: string, definition: string): void {
+  private ensureIncrementalAutoVacuumMode(): void {
+    if (this.inMemory) {
+      return;
+    }
+    const modeRow = this.db.prepare('PRAGMA auto_vacuum;').get();
+    if (modeRow === undefined) {
+      return;
+    }
+    const mode = asNumberOrNull(asRecord(modeRow).auto_vacuum, 'auto_vacuum');
+    if (mode === 2) {
+      return;
+    }
+    try {
+      this.db.exec('PRAGMA auto_vacuum = INCREMENTAL;');
+      this.db.exec('VACUUM;');
+    } catch {
+      // Best-effort migration only; maintenance can still run without mode flip.
+    }
+  }
+
+  private countTotalTelemetryRows(): number {
+    const row = this.db.prepare('SELECT COUNT(*) AS count FROM session_telemetry;').get();
+    if (row === undefined) {
+      return 0;
+    }
+    return asNumberOrNull(asRecord(row).count, 'count') ?? 0;
+  }
+
+  private countTelemetryRowsAfterId(telemetryId: number): number {
+    const row = this.db
+      .prepare('SELECT COUNT(*) AS count FROM session_telemetry WHERE telemetry_id > ?;')
+      .get(telemetryId);
+    if (row === undefined) {
+      return 0;
+    }
+    return asNumberOrNull(asRecord(row).count, 'count') ?? 0;
+  }
+
+  private copyTelemetryCompactionBatch(afterTelemetryId: number, limit: number): number {
+    const safeLimit = Number.isFinite(limit) ? Math.max(1, Math.floor(limit)) : 1;
+    const result = this.db
+      .prepare(
+        `
+        INSERT INTO ${TELEMETRY_COMPACTION_SHADOW_TABLE} (
+          telemetry_id,
+          source,
+          session_id,
+          provider_thread_id,
+          event_name,
+          severity,
+          summary,
+          observed_at,
+          ingested_at,
+          payload_json,
+          fingerprint
+        )
+        SELECT
+          telemetry_id,
+          source,
+          session_id,
+          provider_thread_id,
+          event_name,
+          severity,
+          summary,
+          observed_at,
+          ingested_at,
+          payload_json,
+          fingerprint
+        FROM session_telemetry
+        WHERE telemetry_id > ?
+        ORDER BY telemetry_id ASC
+        LIMIT ?
+      `,
+      )
+      .run(afterTelemetryId, safeLimit);
+    return sqliteStatementChanges(result);
+  }
+
+  private readTelemetryCompactionShadowCursorRowId(): number {
+    const row = this.db
+      .prepare(
+        `
+        SELECT telemetry_id
+        FROM ${TELEMETRY_COMPACTION_SHADOW_TABLE}
+        ORDER BY telemetry_id DESC
+        LIMIT 1
+      `,
+      )
+      .get();
+    if (row === undefined) {
+      return 0;
+    }
+    return asNumberOrNull(asRecord(row).telemetry_id, 'telemetry_id') ?? 0;
+  }
+
+  private resetTelemetryCompactionShadowTable(): void {
+    this.db.exec(`DROP TABLE IF EXISTS ${TELEMETRY_COMPACTION_SHADOW_TABLE};`);
+    this.db.exec(`
+      CREATE TABLE ${TELEMETRY_COMPACTION_SHADOW_TABLE} (
+        telemetry_id INTEGER PRIMARY KEY AUTOINCREMENT,
+        source TEXT NOT NULL,
+        session_id TEXT,
+        provider_thread_id TEXT,
+        event_name TEXT,
+        severity TEXT,
+        summary TEXT,
+        observed_at TEXT NOT NULL,
+        ingested_at TEXT NOT NULL,
+        payload_json TEXT NOT NULL,
+        fingerprint TEXT NOT NULL UNIQUE
+      );
+    `);
+  }
+
+  private swapInTelemetryCompactionShadowTable(): void {
+    this.db.exec('DROP INDEX IF EXISTS idx_session_telemetry_session;');
+    this.db.exec('DROP INDEX IF EXISTS idx_session_telemetry_thread;');
+    this.db.exec(`ALTER TABLE session_telemetry RENAME TO ${TELEMETRY_COMPACTION_OLD_TABLE};`);
+    this.db.exec(`ALTER TABLE ${TELEMETRY_COMPACTION_SHADOW_TABLE} RENAME TO session_telemetry;`);
+    this.db.exec(`
+      CREATE INDEX IF NOT EXISTS idx_session_telemetry_session
+      ON session_telemetry (session_id, observed_at DESC, telemetry_id DESC);
+    `);
+    this.db.exec(`
+      CREATE INDEX IF NOT EXISTS idx_session_telemetry_thread
+      ON session_telemetry (provider_thread_id, observed_at DESC, telemetry_id DESC);
+    `);
+    this.db.exec(`DROP TABLE ${TELEMETRY_COMPACTION_OLD_TABLE};`);
+  }
+
+  private resetTelemetryCompactionStateAfterFailure(): void {
+    this.telemetryCopyForwardActive = false;
+    this.telemetryCopyForwardCursorRowId = 0;
+    try {
+      this.db.exec(`DROP TABLE IF EXISTS ${TELEMETRY_COMPACTION_SHADOW_TABLE};`);
+    } catch {
+      // Best-effort cleanup only.
+    }
+  }
+
+  private columnExists(table: string, column: string): boolean {
     const rows = this.db.prepare(`PRAGMA table_info(${table})`).all();
-    const exists = rows.some((row) => {
+    return rows.some((row) => {
       const asRow = row as Record<string, unknown>;
       return asRow['name'] === column;
     });
-    if (exists) {
+  }
+
+  private ensureColumnExists(table: string, column: string, definition: string): void {
+    if (this.columnExists(table, column)) {
       return;
     }
     this.db.exec(`ALTER TABLE ${table} ADD COLUMN ${definition};`);