@jack200714/mafw 4.5.1

package/gateway/dist/memory/gateway-db.js

@@ -0,0 +1,465 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GatewayDatabase = void 0;
+// Gateway unified database (SQLite). Owns the T1 observation store (single
+// writer = this gateway; the opencode-side plugin pushes observations via
+// POST /api/obs/capture) plus a generic kv_store for small critical state
+// (manager sessions, reflect cursor, registry snapshot) that must move with
+// the data directory and never be lost to project-directory operations.
+//
+// Turn IDs are database-driven, so plugin/serve restarts can never renumber
+// turns or duplicate writes (database-level UNIQUE dedup on observations).
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
+class GatewayDatabase {
+    db;
+    constructor(dbPath) {
+        const dir = path.dirname(dbPath);
+        if (!fs.existsSync(dir))
+            fs.mkdirSync(dir, { recursive: true });
+        this.db = new better_sqlite3_1.default(dbPath);
+        this.db.pragma('journal_mode = WAL');
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS t1_observations (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        session_id TEXT NOT NULL,
+        turn_id INTEGER NOT NULL,
+        source TEXT NOT NULL,
+        content TEXT NOT NULL,
+        failure INTEGER DEFAULT 0,
+        created_at INTEGER DEFAULT (unixepoch()),
+        UNIQUE(session_id, turn_id, source, content)
+      );
+      CREATE INDEX IF NOT EXISTS idx_t1_session ON t1_observations(session_id, turn_id);
+
+      CREATE TABLE IF NOT EXISTS trajectory_events (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        project_id TEXT NOT NULL,
+        session_id TEXT NOT NULL,
+        turn_id INTEGER NOT NULL,
+        seq INTEGER NOT NULL,
+        event_type TEXT NOT NULL,
+        tool_name TEXT,
+        call_id TEXT,
+        tool_state TEXT,
+        agent TEXT,
+        model TEXT,
+        input_summary TEXT,
+        output_summary TEXT,
+        error TEXT,
+        tokens TEXT,
+        cost REAL,
+        finish TEXT,
+        time_ms REAL NOT NULL,
+        duration_ms REAL,
+        created_at INTEGER DEFAULT (unixepoch())
+      );
+      CREATE INDEX IF NOT EXISTS idx_traj_evt_session ON trajectory_events(session_id, turn_id, seq);
+      CREATE INDEX IF NOT EXISTS idx_traj_evt_ttl ON trajectory_events(created_at);
+
+      CREATE TABLE IF NOT EXISTS trajectory_turns (
+        project_id TEXT NOT NULL,
+        session_id TEXT NOT NULL,
+        turn_id INTEGER NOT NULL,
+        turn_start_ms REAL NOT NULL,
+        turn_end_ms REAL,
+        duration_ms REAL,
+        tool_count INTEGER DEFAULT 0,
+        tool_error_count INTEGER DEFAULT 0,
+        reasoning_count INTEGER DEFAULT 0,
+        agent_switch_count INTEGER DEFAULT 0,
+        tokens TEXT,
+        cost REAL DEFAULT 0,
+        finish TEXT,
+        model TEXT,
+        provider TEXT,
+        agent TEXT,
+        user_text TEXT,
+        created_at INTEGER DEFAULT (unixepoch()),
+        PRIMARY KEY (session_id, turn_id)
+      );
+      CREATE INDEX IF NOT EXISTS idx_traj_turn_ttl ON trajectory_turns(created_at);
+
+      CREATE TABLE IF NOT EXISTS anchor_units (
+        anchor TEXT NOT NULL,
+        unit_id TEXT NOT NULL,
+        weight REAL DEFAULT 1,
+        created_at INTEGER DEFAULT (unixepoch()),
+        PRIMARY KEY (anchor, unit_id)
+      );
+      CREATE INDEX IF NOT EXISTS idx_anchor_units_anchor ON anchor_units(anchor);
+      CREATE INDEX IF NOT EXISTS idx_anchor_units_unit ON anchor_units(unit_id);
+
+      CREATE TABLE IF NOT EXISTS anchor_edges (
+        unit_a TEXT NOT NULL,
+        unit_b TEXT NOT NULL,
+        shared_anchors INTEGER NOT NULL,
+        weight REAL NOT NULL,
+        updated_at INTEGER DEFAULT (unixepoch()),
+        PRIMARY KEY (unit_a, unit_b)
+      );
+      CREATE INDEX IF NOT EXISTS idx_anchor_edges_a ON anchor_edges(unit_a);
+      CREATE INDEX IF NOT EXISTS idx_anchor_edges_b ON anchor_edges(unit_b);
+
+      CREATE TABLE IF NOT EXISTS kv_store (
+        scope TEXT NOT NULL,
+        key TEXT NOT NULL,
+        value TEXT NOT NULL,
+        updated_at INTEGER DEFAULT (unixepoch()),
+        PRIMARY KEY (scope, key)
+      );
+
+      CREATE TABLE IF NOT EXISTS t1_archive (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        session_id TEXT NOT NULL,
+        turn_id INTEGER NOT NULL,
+        source TEXT NOT NULL,
+        content TEXT NOT NULL,
+        failure INTEGER DEFAULT 0,
+        created_at INTEGER NOT NULL,
+        archived_at INTEGER DEFAULT (unixepoch())
+      );
+      CREATE INDEX IF NOT EXISTS idx_t1_archive_session ON t1_archive(session_id, turn_id);
+
+      CREATE TABLE IF NOT EXISTS t1_noop_log (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        session_id TEXT NOT NULL,
+        turn_id INTEGER NOT NULL,
+        reason TEXT NOT NULL,
+        created_at INTEGER DEFAULT (unixepoch())
+      );
+      CREATE INDEX IF NOT EXISTS idx_noop_session ON t1_noop_log(session_id);
+      CREATE INDEX IF NOT EXISTS idx_noop_created ON t1_noop_log(created_at);
+
+      CREATE TABLE IF NOT EXISTS goal_outcomes (
+        goal_id TEXT PRIMARY KEY,
+        project_id TEXT,
+        verdict TEXT NOT NULL,
+        rounds INTEGER DEFAULT 0,
+        duration_ms INTEGER,
+        tokens_input INTEGER,
+        tokens_output INTEGER,
+        total_cost REAL,
+        tool_error_count INTEGER,
+        thumbs_up INTEGER DEFAULT 0,
+        thumbs_down INTEGER DEFAULT 0,
+        policy_version TEXT NOT NULL DEFAULT 'builtin-v1',
+        evolution_proposal_id TEXT,
+        failure_kind TEXT,
+        failure_signature TEXT,
+        created_at TEXT,
+        archived_at TEXT NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS idx_goal_outcomes_policy ON goal_outcomes(policy_version, verdict);
+      CREATE INDEX IF NOT EXISTS idx_goal_outcomes_project ON goal_outcomes(project_id, policy_version);
+
+      CREATE TABLE IF NOT EXISTS goal_sessions (
+        goal_id TEXT NOT NULL,
+        session_id TEXT NOT NULL,
+        phase TEXT NOT NULL,
+        loop INTEGER DEFAULT 1,
+        created_at TEXT NOT NULL,
+        PRIMARY KEY (goal_id, session_id)
+      );
+      CREATE INDEX IF NOT EXISTS idx_goal_sessions_session ON goal_sessions(session_id);
+
+      CREATE TABLE IF NOT EXISTS evolution_proposals (
+        id TEXT PRIMARY KEY,
+        requires TEXT NOT NULL DEFAULT 'policy',
+        component_diffs TEXT NOT NULL,
+        declared_prediction TEXT,
+        rationale TEXT,
+        status TEXT NOT NULL DEFAULT 'proposed',
+        validation_result TEXT,
+        created_at TEXT NOT NULL,
+        updated_at TEXT NOT NULL
+      );
+    `);
+        // Migration: add assistant_text column to trajectory_turns if not exists
+        try {
+            this.db.exec(`ALTER TABLE trajectory_turns ADD COLUMN assistant_text TEXT`);
+        }
+        catch {
+            // Column already exists, ignore
+        }
+        // Migration: add provider column to trajectory_turns if not exists
+        try {
+            this.db.exec(`ALTER TABLE trajectory_turns ADD COLUMN provider TEXT`);
+        }
+        catch {
+            // Column already exists, ignore
+        }
+        // Migration: add worker_role column to trajectory_turns if not exists
+        try {
+            this.db.exec(`ALTER TABLE trajectory_turns ADD COLUMN worker_role TEXT`);
+        }
+        catch {
+            // Column already exists, ignore
+        }
+    }
+    // ── T1 observations ─────────────────────────────────────────────────────
+    /**
+     * Next turn ID for a new user message: max existing turn for the session + 1.
+     * No rows yet → 1.
+     */
+    nextTurnId(session_id) {
+        const row = this.db
+            .prepare('SELECT COALESCE(MAX(turn_id), 0) AS m FROM t1_observations WHERE session_id = ?')
+            .get(session_id);
+        return row.m + 1;
+    }
+    /** Current turn ID for non-user observations: max existing turn, or 0. */
+    currentTurnId(session_id) {
+        const row = this.db
+            .prepare('SELECT COALESCE(MAX(turn_id), 0) AS m FROM t1_observations WHERE session_id = ?')
+            .get(session_id);
+        return row.m;
+    }
+    /**
+     * Insert one observation. Database-level dedup via the UNIQUE constraint:
+     * a duplicate (same session/turn/source/content) is silently ignored.
+     * Returns the row id, or null when deduplicated.
+     */
+    append(obs) {
+        const result = this.db
+            .prepare(`INSERT OR IGNORE INTO t1_observations (session_id, turn_id, source, content, failure)
+         VALUES (?, ?, ?, ?, ?)`)
+            .run(obs.session_id, obs.turn_id, obs.source, obs.content, obs.failure);
+        return result.changes > 0 ? Number(result.lastInsertRowid) : null;
+    }
+    /** Per-session turn summaries (grouped), optionally filtered to a session. */
+    listTurns(session_id) {
+        const where = session_id ? 'WHERE session_id = ?' : '';
+        const rows = this.db
+            .prepare(`SELECT session_id, turn_id,
+                COUNT(*) AS count,
+                MAX(CASE WHEN source = 'user_input' THEN 1 ELSE 0 END) AS has_user_input,
+                SUM(CASE WHEN source IN ('assistant_reply','tool_result','reasoning') AND length(trim(content)) > 0 THEN 1 ELSE 0 END) AS response_count,
+                MAX(created_at) AS last_ts
+         FROM t1_observations
+         ${where}
+         GROUP BY session_id, turn_id
+         ORDER BY session_id, turn_id`)
+            .all(...(session_id ? [session_id] : []));
+        return rows;
+    }
+    readTurn(session_id, turn_id) {
+        return this.db
+            .prepare('SELECT * FROM t1_observations WHERE session_id = ? AND turn_id = ? ORDER BY id')
+            .all(session_id, turn_id);
+    }
+    /** Remove all observations for a session up to (and including) a turn. */
+    deleteTurn(session_id, turn_id) {
+        const result = this.db
+            .prepare('DELETE FROM t1_observations WHERE session_id = ? AND turn_id = ?')
+            .run(session_id, turn_id);
+        return result.changes;
+    }
+    /**
+     * Archive a processed turn: move from t1_observations to t1_archive, then
+     * delete from t1_observations. Preserves the raw observation data for future
+     * re-extraction with upgraded pipelines. Returns the number of rows archived.
+     */
+    archiveTurn(session_id, turn_id) {
+        const txn = this.db.transaction(() => {
+            this.db
+                .prepare(`INSERT INTO t1_archive (session_id, turn_id, source, content, failure, created_at)
+           SELECT session_id, turn_id, source, content, failure, created_at
+           FROM t1_observations WHERE session_id = ? AND turn_id = ?`)
+                .run(session_id, turn_id);
+            const del = this.db
+                .prepare('DELETE FROM t1_observations WHERE session_id = ? AND turn_id = ?')
+                .run(session_id, turn_id);
+            return del.changes;
+        });
+        return txn();
+    }
+    /** Read archived observations for a specific turn. */
+    readArchiveTurn(session_id, turn_id) {
+        return this.db
+            .prepare('SELECT * FROM t1_archive WHERE session_id = ? AND turn_id = ? ORDER BY id')
+            .all(session_id, turn_id);
+    }
+    /** List archived turns (grouped), optionally filtered to a session. */
+    listArchiveTurns(session_id) {
+        const where = session_id ? 'WHERE session_id = ?' : '';
+        return this.db
+            .prepare(`SELECT session_id, turn_id,
+                COUNT(*) AS count,
+                MAX(CASE WHEN source = 'user_input' THEN 1 ELSE 0 END) AS has_user_input,
+                SUM(CASE WHEN source IN ('assistant_reply','tool_result','reasoning') AND length(trim(content)) > 0 THEN 1 ELSE 0 END) AS response_count,
+                MAX(created_at) AS last_ts
+         FROM t1_archive
+         ${where}
+         GROUP BY session_id, turn_id
+         ORDER BY session_id, turn_id`)
+            .all(...(session_id ? [session_id] : []));
+    }
+    archiveCount() {
+        return this.db.prepare('SELECT COUNT(*) AS c FROM t1_archive').get().c;
+    }
+    // ── Noop log (extraction coverage monitoring) ──────────────────────────
+    /**
+     * Log a turn where the extraction worker decided nothing was worth saving.
+     * Enables weekly review of what the worker skips — critical for diagnosing
+     * whether preference/temporal misses happen at extraction or retrieval.
+     */
+    logNoop(session_id, turn_id, reason) {
+        this.db
+            .prepare('INSERT INTO t1_noop_log (session_id, turn_id, reason) VALUES (?, ?, ?)')
+            .run(session_id, turn_id, reason);
+    }
+    /** List noop log entries, optionally filtered by session and/or since timestamp. */
+    listNoops(opts) {
+        const conditions = [];
+        const params = [];
+        if (opts?.session_id) {
+            conditions.push('session_id = ?');
+            params.push(opts.session_id);
+        }
+        if (opts?.since) {
+            conditions.push('created_at >= ?');
+            params.push(opts.since);
+        }
+        const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
+        const limit = opts?.limit ?? 100;
+        return this.db
+            .prepare(`SELECT * FROM t1_noop_log ${where} ORDER BY created_at DESC LIMIT ?`)
+            .all(...params, limit);
+    }
+    deleteSession(session_id) {
+        return this.db.prepare('DELETE FROM t1_observations WHERE session_id = ?').run(session_id).changes;
+    }
+    count() {
+        return this.db.prepare('SELECT COUNT(*) AS c FROM t1_observations').get().c;
+    }
+    // ── KV store (small critical state) ─────────────────────────────────────
+    kvGet(scope, key) {
+        const row = this.db.prepare('SELECT value FROM kv_store WHERE scope = ? AND key = ?').get(scope, key);
+        if (!row)
+            return null;
+        try {
+            return JSON.parse(row.value);
+        }
+        catch {
+            return row.value;
+        }
+    }
+    kvSet(scope, key, value) {
+        this.db
+            .prepare(`INSERT INTO kv_store (scope, key, value) VALUES (?, ?, ?)
+         ON CONFLICT(scope, key) DO UPDATE SET value = excluded.value, updated_at = unixepoch()`)
+            .run(scope, key, JSON.stringify(value));
+    }
+    kvDelete(scope, key) {
+        this.db.prepare('DELETE FROM kv_store WHERE scope = ? AND key = ?').run(scope, key);
+    }
+    kvAll(scope) {
+        const rows = this.db.prepare('SELECT key, value FROM kv_store WHERE scope = ? ORDER BY key').all(scope);
+        return rows.map((r) => {
+            try {
+                return { key: r.key, value: JSON.parse(r.value) };
+            }
+            catch {
+                return { key: r.key, value: r.value };
+            }
+        });
+    }
+    // ── Goal outcomes (RSI Phase 1) ─────────────────────────────────────────
+    upsertGoalOutcome(o) {
+        this.db.prepare(`
+      INSERT INTO goal_outcomes (goal_id, project_id, verdict, rounds, duration_ms,
+        tokens_input, tokens_output, total_cost, tool_error_count, thumbs_up, thumbs_down,
+        policy_version, evolution_proposal_id, failure_kind, failure_signature, created_at, archived_at)
+      VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
+      ON CONFLICT(goal_id) DO UPDATE SET
+        verdict=excluded.verdict, rounds=excluded.rounds, duration_ms=excluded.duration_ms,
+        tokens_input=excluded.tokens_input, tokens_output=excluded.tokens_output, total_cost=excluded.total_cost,
+        tool_error_count=excluded.tool_error_count, thumbs_up=excluded.thumbs_up, thumbs_down=excluded.thumbs_down,
+        policy_version=excluded.policy_version, evolution_proposal_id=excluded.evolution_proposal_id,
+        failure_kind=excluded.failure_kind, failure_signature=excluded.failure_signature,
+        created_at=excluded.created_at, archived_at=excluded.archived_at
+    `).run(o.goal_id, o.project_id, o.verdict, o.rounds, o.duration_ms, o.tokens_input, o.tokens_output, o.total_cost, o.tool_error_count, o.thumbs_up, o.thumbs_down, o.policy_version, o.evolution_proposal_id, o.failure_kind, o.failure_signature, o.created_at, o.archived_at);
+    }
+    listGoalOutcomes(filter) {
+        const conditions = [];
+        const params = [];
+        if (filter?.policy) {
+            conditions.push('policy_version = ?');
+            params.push(filter.policy);
+        }
+        if (filter?.verdict) {
+            conditions.push('verdict = ?');
+            params.push(filter.verdict);
+        }
+        if (filter?.project) {
+            conditions.push('project_id = ?');
+            params.push(filter.project);
+        }
+        const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
+        const limit = filter?.limit ?? 100;
+        return this.db.prepare(`SELECT * FROM goal_outcomes ${where} ORDER BY archived_at DESC LIMIT ?`).all(...params, limit);
+    }
+    addGoalSession(s) {
+        this.db.prepare(`
+      INSERT OR IGNORE INTO goal_sessions (goal_id, session_id, phase, loop, created_at)
+      VALUES (?,?,?,?,datetime('now'))
+    `).run(s.goal_id, s.session_id, s.phase, s.loop);
+    }
+    listGoalSessions(goalId) {
+        return this.db.prepare('SELECT session_id, phase, loop FROM goal_sessions WHERE goal_id = ?').all(goalId);
+    }
+    /** Most recent archived outcome among the goals a session participated in
+     *  (goal_sessions → goal_outcomes join). Powers the turn-compress curator's
+     *  outcome-feedback signal (env-probing curation, grade gi). */
+    getOutcomeForSession(sessionId) {
+        const row = this.db.prepare(`
+      SELECT o.* FROM goal_outcomes o
+      JOIN goal_sessions s ON s.goal_id = o.goal_id
+      WHERE s.session_id = ?
+      ORDER BY o.archived_at DESC LIMIT 1
+    `).get(sessionId);
+        return row ?? null;
+    }
+    close() {
+        this.db.close();
+    }
+}
+exports.GatewayDatabase = GatewayDatabase;