@winspan/claude-forge 8.41.0 → 8.51.0

package/src/core/event-fields.ts

@@ -0,0 +1,32 @@
+/**
+ * Typed getters for ForgeEvent.tool_input fields.
+ *
+ * Centralises the defensive-access pattern that previously lived as
+ * `(e.tool_input as any)?.command ?? ''` across 30+ call sites. Each
+ * getter returns `string | undefined`; callers decide their own fallback.
+ *
+ * Adding a new tool field? Extend ToolInputFields in core/types.ts first,
+ * then add a getter here if it needs a normalised access entry.
+ */
+import type { ForgeEvent } from './types.js';
+
+export function getCommand(e: ForgeEvent): string | undefined {
+  const c = e.tool_input?.command;
+  return typeof c === 'string' ? c : undefined;
+}
+
+export function getFilePath(e: ForgeEvent): string | undefined {
+  const fp = e.tool_input?.file_path ?? e.tool_input?.notebook_path;
+  return typeof fp === 'string' && fp.length > 0 ? fp : undefined;
+}
+
+export function getUserPrompt(e: ForgeEvent): string | undefined {
+  if (typeof e.user_prompt === 'string') return e.user_prompt;
+  const fallback = e.tool_input?.user_prompt;
+  return typeof fallback === 'string' ? fallback : undefined;
+}
+
+export function getSubagentType(e: ForgeEvent): string | undefined {
+  const s = e.tool_input?.subagent_type;
+  return typeof s === 'string' ? s : undefined;
+}

package/src/core/queue/index.ts

@@ -0,0 +1,192 @@
+/**
+ * Hook failure buffer queue — at-least-once event delivery
+ *
+ * When the daemon is not reachable (socket missing / nc fail), hook scripts
+ * write the raw event JSON to a per-file queue on disk. On the next daemon
+ * startup, `replayQueue` scans the directory and replays every file into
+ * SQLiteStorage.
+ *
+ * Design decisions:
+ * - One file per event: avoids concurrent-write collisions across 5 hook processes
+ * - File name: <iso-timestamp>-<event_id>.json  (sortable + unique)
+ * - Dedup: events table PRIMARY KEY; UNIQUE constraint error → silently skip
+ * - Capacity: MAX_FILES=500, TTL=7d; overflow → drop oldest; expired → dead-letter
+ */
+
+import { mkdirSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { randomUUID } from 'node:crypto';
+import type { SQLiteStorage } from '../storage/sqlite.js';
+import type { ForgeEvent } from '../types.js';
+import { logger } from '../utils/logger.js';
+import { FORGE_PATHS } from '../constants.js';
+
+// ── Constants ──────────────────────────────────────────────────────────────
+export const QUEUE_DIR = FORGE_PATHS.queue();
+export const DEAD_DIR = FORGE_PATHS.queueDead();
+export const MAX_FILES = 500;
+export const TTL_DAYS = 7;
+
+const TTL_MS = TTL_DAYS * 24 * 60 * 60 * 1000;
+
+// ── Helpers ────────────────────────────────────────────────────────────────
+
+function ensureDirs(): void {
+  mkdirSync(QUEUE_DIR, { recursive: true });
+  mkdirSync(DEAD_DIR, { recursive: true });
+}
+
+/** List all *.json queue files, sorted oldest-first (by filename). */
+function listQueueFiles(): string[] {
+  try {
+    return readdirSync(QUEUE_DIR)
+      .filter((f) => f.endsWith('.json'))
+      .sort(); // ISO timestamp prefix → lexicographic = chronological
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * pruneQueue — enforce MAX_FILES cap.
+ * Deletes the oldest files (smallest filename) until count < MAX_FILES.
+ * Called by enqueueEvent before writing a new file.
+ */
+export function pruneQueue(): void {
+  const files = listQueueFiles();
+  if (files.length < MAX_FILES) return;
+
+  const toDelete = files.slice(0, files.length - MAX_FILES + 1);
+  for (const f of toDelete) {
+    try {
+      rmSync(join(QUEUE_DIR, f));
+    } catch (err) {
+      logger.warn(`[Queue] Failed to prune ${f}: ${err}`);
+    }
+  }
+}
+
+// ── Public API ─────────────────────────────────────────────────────────────
+
+/**
+ * enqueueEvent — write a single event to the queue directory.
+ *
+ * Designed to be called synchronously from hook scripts via a bash child
+ * process. This TypeScript version is used by tests; the bash implementation
+ * mirrors the same file-naming convention.
+ */
+export function enqueueEvent(event: ForgeEvent): void {
+  ensureDirs();
+  pruneQueue();
+
+  const eventId = event.event_id ?? randomUUID();
+  const ts = new Date().toISOString().replace(/[:.]/g, '-').replace('Z', 'Z');
+  const filename = `${ts}-${eventId}.json`;
+  const filePath = join(QUEUE_DIR, filename);
+
+  writeFileSync(filePath, JSON.stringify({ ...event, event_id: eventId }), 'utf8');
+}
+
+export interface ReplayResult {
+  replayed: number;
+  skipped: number;
+  dead: number;
+}
+
+/**
+ * replayQueue — called by daemon after socket starts listening.
+ *
+ * For each *.json in QUEUE_DIR:
+ *   - If mtime > TTL → move to dead-letter
+ *   - Parse JSON → storage.writeEvent()
+ *     - Success → rm file, replayed++
+ *     - UNIQUE constraint → rm file, skipped++ (already in DB)
+ *     - Other error → move to dead-letter, dead++
+ *   - Bad JSON → move to dead-letter, dead++
+ */
+export async function replayQueue(storage: SQLiteStorage): Promise<ReplayResult> {
+  ensureDirs();
+
+  const files = listQueueFiles();
+  if (files.length === 0) {
+    return { replayed: 0, skipped: 0, dead: 0 };
+  }
+
+  logger.info(`[Queue] Replaying ${files.length} queued event(s)…`);
+
+  let replayed = 0;
+  let skipped = 0;
+  let dead = 0;
+  const now = Date.now();
+
+  for (const filename of files) {
+    const filePath = join(QUEUE_DIR, filename);
+
+    // ── TTL check ──────────────────────────────────────────────────────────
+    try {
+      const stat = statSync(filePath);
+      if (now - stat.mtimeMs > TTL_MS) {
+        moveToDeadLetter(filePath, filename);
+        dead++;
+        logger.warn(`[Queue] TTL expired, dead-lettered: ${filename}`);
+        continue;
+      }
+    } catch (err) {
+      logger.warn(`[Queue] Cannot stat ${filename}, skipping: ${err}`);
+      continue;
+    }
+
+    // ── Read + parse ───────────────────────────────────────────────────────
+    let event: ForgeEvent;
+    try {
+      const raw = await readFile(filePath, 'utf8');
+      event = JSON.parse(raw) as ForgeEvent;
+    } catch {
+      moveToDeadLetter(filePath, filename);
+      dead++;
+      logger.warn(`[Queue] Bad JSON, dead-lettered: ${filename}`);
+      continue;
+    }
+
+    // ── Write to storage ───────────────────────────────────────────────────
+    try {
+      storage.writeEvent(event);
+      rmSync(filePath);
+      replayed++;
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      if (isUniqueConstraintError(msg)) {
+        // Already in DB — remove the queue file silently
+        try { rmSync(filePath); } catch { /* ignore */ }
+        skipped++;
+      } else {
+        moveToDeadLetter(filePath, filename);
+        dead++;
+        logger.warn(`[Queue] writeEvent failed, dead-lettered ${filename}: ${msg}`);
+      }
+    }
+  }
+
+  return { replayed, skipped, dead };
+}
+
+// ── Internal helpers ───────────────────────────────────────────────────────
+
+function moveToDeadLetter(filePath: string, filename: string): void {
+  try {
+    mkdirSync(DEAD_DIR, { recursive: true });
+    renameSync(filePath, join(DEAD_DIR, filename));
+  } catch (err) {
+    logger.warn(`[Queue] Failed to move ${filename} to dead-letter: ${err}`);
+    try { rmSync(filePath); } catch { /* ignore */ }
+  }
+}
+
+/**
+ * Detect SQLite UNIQUE constraint violations.
+ * better-sqlite3 throws: "UNIQUE constraint failed: events.event_id"
+ */
+function isUniqueConstraintError(message: string): boolean {
+  return message.includes('UNIQUE constraint failed');
+}

package/src/core/storage/base.ts

@@ -0,0 +1,302 @@
+/**
+ * SQLiteBase — 数据库初始化、schema、迁移、辅助
+ *
+ * 拆分自 sqlite.ts。仅负责：
+ * - 打开数据库连接 + 配置 PRAGMA
+ * - 加载 schema.sql（缺失则抛错）
+ * - 增量迁移（hasColumn / addColumnIfMissing）
+ * - 一次性 sessions 回填
+ *
+ * 不涉及业务读写。所有业务读写由 *Operations 类组合 db 实例实现。
+ */
+
+import Database from 'better-sqlite3';
+import { readFileSync, existsSync, mkdirSync, statSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { DATABASE } from '../constants.js';
+import { logger } from '../utils/logger.js';
+
+export class SQLiteBase {
+  protected db: Database.Database;
+  protected dbPath: string;
+
+  constructor(dbPath: string) {
+    this.dbPath = dbPath;
+
+    const dir = dirname(dbPath);
+    if (!existsSync(dir)) {
+      mkdirSync(dir, { recursive: true });
+    }
+
+    this.db = new Database(dbPath);
+
+    // WAL mode optimization for better concurrency
+    this.db.pragma('journal_mode = WAL');
+    this.db.pragma('synchronous = NORMAL');
+    this.db.pragma('wal_autocheckpoint = 1000'); // Increased from 100 for better performance
+    this.db.pragma(`cache_size = ${DATABASE.CACHE_SIZE}`);
+    this.db.pragma(`busy_timeout = ${DATABASE.BUSY_TIMEOUT}`);
+
+    // Additional performance optimizations
+    this.db.pragma('temp_store = MEMORY'); // Store temp tables in memory
+    this.db.pragma('mmap_size = 30000000000'); // 30GB memory-mapped I/O
+    this.db.pragma('page_size = 4096'); // Optimal page size for most systems
+
+    // Order matters:
+    //   1) runColumnMigrations() ALTERs missing columns on PRE-EXISTING tables.
+    //      Must run BEFORE initSchema() because schema.sql contains indexes that
+    //      reference new columns (e.g. idx_skill_invocations_workflow). On a
+    //      legacy DB those CREATE INDEX statements would otherwise fail because
+    //      CREATE TABLE IF NOT EXISTS skips the old table and never adds the column.
+    //   2) initSchema() applies schema.sql (idempotent CREATE TABLE/INDEX).
+    //   3) runIndexMigrations() patches indexes that may still be missing on
+    //      legacy DBs (defensive; schema.sql already creates them on new DBs).
+    this.runColumnMigrations();
+    this.initSchema();
+    this.runIndexMigrations();
+    this.runPostMigrations();
+  }
+
+  private initSchema(): void {
+    const thisDir = dirname(fileURLToPath(import.meta.url));
+    const schemaPath = join(thisDir, 'schema.sql');
+
+    if (!existsSync(schemaPath)) {
+      throw new Error(
+        `[SQLiteStorage] schema.sql not found at ${schemaPath}. ` +
+        `This indicates a broken build/install. Run \`npm run build\` or reinstall the package.`,
+      );
+    }
+    const schema = readFileSync(schemaPath, 'utf-8');
+    this.db.exec(schema);
+  }
+
+  protected hasTable(name: string): boolean {
+    try {
+      const row = this.db
+        .prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name = ?`)
+        .get(name);
+      return !!row;
+    } catch {
+      return false;
+    }
+  }
+
+  protected hasColumn(table: string, column: string): boolean {
+    try {
+      const rows = this.db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>;
+      return rows.some(r => r.name === column);
+    } catch {
+      return false;
+    }
+  }
+
+  protected addColumnIfMissing(table: string, column: string, definition: string): void {
+    // Skip silently if the table doesn't exist yet — new DBs hit this path on
+    // first boot (initSchema runs after column migrations); the column will be
+    // created by schema.sql's CREATE TABLE.
+    if (!this.hasTable(table)) return;
+    if (!this.hasColumn(table, column)) {
+      try {
+        this.db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
+        logger.info(`[SQLiteStorage] migration: added ${table}.${column}`);
+      } catch (err) {
+        logger.warn(`[SQLiteStorage] migration: ALTER ${table} ADD ${column} failed: ${err}`);
+      }
+    }
+  }
+
+  protected hasIndex(name: string): boolean {
+    try {
+      const row = this.db
+        .prepare(`SELECT name FROM sqlite_master WHERE type='index' AND name = ?`)
+        .get(name);
+      return !!row;
+    } catch {
+      return false;
+    }
+  }
+
+  /**
+   * Create an index only if it's missing. The CREATE INDEX IF NOT EXISTS already
+   * makes this safe; the hasIndex guard makes it explicit which indexes are still
+   * being patched by migrations (vs. provided by schema.sql).
+   */
+  private createIndexIfMissing(name: string, ddl: string): void {
+    if (this.hasIndex(name)) return;
+    try {
+      this.db.exec(ddl);
+      logger.debug(`[SQLiteStorage] migration created ${name}`);
+    } catch (err) {
+      logger.warn(`[SQLiteStorage] migration: create ${name} failed: ${err}`);
+    }
+  }
+
+  /**
+   * Column migrations run BEFORE initSchema() so legacy tables get missing
+   * columns (e.g. skill_invocations.workflow) before schema.sql's CREATE INDEX
+   * statements try to reference them.
+   */
+  private runColumnMigrations(): void {
+    this.addColumnIfMissing('sessions', 'first_prompt', 'TEXT');
+
+    this.addColumnIfMissing('routing_events', 'skill_confidence', 'REAL');
+    this.addColumnIfMissing('routing_events', 'skill_source', 'TEXT');
+
+    this.addColumnIfMissing('skill_invocations', 'workflow', 'TEXT');
+    this.addColumnIfMissing('skill_invocations', 'phase', 'TEXT');
+    this.addColumnIfMissing('skill_invocations', 'feature_slug', 'TEXT');
+    this.addColumnIfMissing('skill_invocations', 'artifact_path', 'TEXT');
+  }
+
+  /**
+   * Index migrations run AFTER initSchema(). Each call is guarded by hasIndex
+   * so:
+   *   - new DBs: schema.sql already created the index → skipped
+   *   - legacy DBs: schema.sql skipped (existing index) for indexes it tries to
+   *     create or covered above; remaining migration-only / pre-H1 indexes are
+   *     patched here.
+   * CREATE INDEX IF NOT EXISTS is preserved as a second line of defense.
+   */
+  private runIndexMigrations(): void {
+    this.createIndexIfMissing(
+      'idx_skill_invocations_workflow',
+      `CREATE INDEX IF NOT EXISTS idx_skill_invocations_workflow ON skill_invocations(workflow, phase)`,
+    );
+    this.createIndexIfMissing(
+      'idx_skill_invocations_feature',
+      `CREATE INDEX IF NOT EXISTS idx_skill_invocations_feature ON skill_invocations(feature_slug)`,
+    );
+
+    this.createIndexIfMissing(
+      'idx_sessions_start_time',
+      `CREATE INDEX IF NOT EXISTS idx_sessions_start_time ON sessions(start_time DESC)`,
+    );
+
+    // Composite indexes for high-frequency session-scoped queries
+    this.createIndexIfMissing(
+      'idx_events_session_ts',
+      `CREATE INDEX IF NOT EXISTS idx_events_session_ts ON events(session_id, timestamp DESC)`,
+    );
+    this.createIndexIfMissing(
+      'idx_routing_events_session_ts',
+      `CREATE INDEX IF NOT EXISTS idx_routing_events_session_ts ON routing_events(session_id, ts DESC)`,
+    );
+    this.createIndexIfMissing(
+      'idx_skill_invocations_session_ts',
+      `CREATE INDEX IF NOT EXISTS idx_skill_invocations_session_ts ON skill_invocations(session_id, timestamp DESC)`,
+    );
+
+    // Phase 1 Refactor: Additional performance indexes
+    this.createIndexIfMissing(
+      'idx_routing_events_obeyed_ts',
+      `CREATE INDEX IF NOT EXISTS idx_routing_events_obeyed_ts ON routing_events(obeyed, ts DESC)`,
+    );
+    this.createIndexIfMissing(
+      'idx_events_session_hook',
+      `CREATE INDEX IF NOT EXISTS idx_events_session_hook ON events(session_id, hook_type, timestamp DESC)`,
+    );
+    this.createIndexIfMissing(
+      'idx_injections_session_handler',
+      `CREATE INDEX IF NOT EXISTS idx_injections_session_handler ON injections(session_id, source_handler)`,
+    );
+    this.createIndexIfMissing(
+      'idx_routing_events_type_ts',
+      `CREATE INDEX IF NOT EXISTS idx_routing_events_type_ts ON routing_events(routed_to_type, ts DESC)`,
+    );
+  }
+
+  /**
+   * One-shot data migrations + deprecated table cleanup.
+   */
+  private runPostMigrations(): void {
+    this.backfillSessionsIfNeeded();
+
+    const deprecatedTables = [
+      'quality_issues', 'distill_results', 'v2_decisions',
+      'v2_tool_events', 'experiment_assignments', 'routing_rule_states',
+    ];
+    for (const table of deprecatedTables) {
+      try { this.db.exec(`DROP TABLE IF EXISTS ${table}`); } catch { /* ignore */ }
+    }
+  }
+
+  /**
+   * 将 events 表按 session_id 聚合回填到 sessions 表，仅在 sessions 尚未对齐
+   * （没有任何一行带 first_prompt）时触发。
+   */
+  private backfillSessionsIfNeeded(): void {
+    const eventsExists = this.db
+      .prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='events'`)
+      .get();
+    if (!eventsExists) return;
+
+    const aggRow = this.db
+      .prepare(`SELECT COUNT(*) AS total, COUNT(first_prompt) AS with_prompt FROM sessions`)
+      .get() as { total: number; with_prompt: number };
+
+    if (aggRow.with_prompt > 0) return;
+
+    logger.info('[SQLiteStorage] backfilling sessions aggregate from events...');
+    try {
+      const backfill = this.db.transaction(() => {
+        this.db.exec(`
+          INSERT INTO sessions (
+            session_id, project_path, status, first_prompt,
+            start_time, end_time, last_event_time, event_count
+          )
+          SELECT
+            e.session_id,
+            e.project_path,
+            'active' AS status,
+            (SELECT substr(COALESCE(e2.user_prompt, json_extract(e2.tool_input, '$.user_prompt')), 1, 200)
+             FROM events e2
+             WHERE e2.session_id = e.session_id
+               AND e2.hook_type = 'UserPromptSubmit'
+               AND (e2.user_prompt IS NOT NULL OR json_extract(e2.tool_input, '$.user_prompt') IS NOT NULL)
+             ORDER BY e2.timestamp ASC LIMIT 1) AS first_prompt,
+            MIN(e.timestamp) AS start_time,
+            MAX(e.timestamp) AS end_time,
+            MAX(e.timestamp) AS last_event_time,
+            COUNT(*) AS event_count
+          FROM events e
+          GROUP BY e.session_id, e.project_path
+          ON CONFLICT(session_id) DO UPDATE SET
+            first_prompt = COALESCE(sessions.first_prompt, excluded.first_prompt),
+            end_time = MAX(COALESCE(sessions.end_time, ''), excluded.end_time),
+            last_event_time = MAX(COALESCE(sessions.last_event_time, ''), excluded.last_event_time),
+            event_count = excluded.event_count,
+            updated_at = datetime('now')
+        `);
+      });
+      backfill();
+      const migrated = this.db.prepare('SELECT COUNT(*) AS c FROM sessions').get() as { c: number };
+      logger.info(`[SQLiteStorage] backfilled ${migrated.c} sessions from events`);
+    } catch (err) {
+      logger.warn(`[SQLiteStorage] backfillSessionsIfNeeded failed: ${err}`);
+    }
+  }
+
+  /**
+   * @internal 仅供 *Operations 子类、migration、初始化使用。
+   *
+   * Routes / Handlers / Analytics 禁止直接调用。需要查询请走 SQLiteStorage facade
+   * 上的具名方法（query* / get* / list* / count* / aggregate*）；若 facade 缺方法，
+   * 应在对应 Operations 类下沉 SQL 并补 facade 转发，而不是 db.prepare() 直写。
+   *
+   * 历史：H2 (2026-05) 已收敛 27 处越权 SQL，新增越权请走同样的下沉模式。
+   */
+  getDatabase(): Database.Database {
+    return this.db;
+  }
+
+  getDbPath(): string {
+    return this.dbPath;
+  }
+
+  protected getDbSizeMb(): number {
+    if (!existsSync(this.dbPath)) return 0;
+    return statSync(this.dbPath).size / (1024 * 1024);
+  }
+}