nemoris 0.1.0

package/src/runtime/context-ledger.js

@@ -0,0 +1,360 @@
+/**
+ * Context Ledger — event-sourced session store (Phase 1).
+ *
+ * Append-only event log with sha256 dedup, snapshots, and outbox
+ * for exactly-once side effects. SQLite + WAL mode.
+ */
+
+import { createHash, randomUUID } from "node:crypto";
+import { DatabaseSync } from "node:sqlite";
+import { canonicalJson } from "../utils/canonical-json.js";
+
+const VALID_KINDS = new Set([
+  "message_in", "message_out", "tool_call", "tool_result",
+  "memory_write", "state_patch", "compaction",
+]);
+
+export class ContextLedger {
+  #snapshotEveryNEvents;
+  #lastSnapshotEventId;
+  #sessionStates;
+
+  constructor({ dbPath, snapshotEveryNEvents = 10 }) {
+    this.dbPath = dbPath;
+    this.db = null;
+    this.#snapshotEveryNEvents = snapshotEveryNEvents;
+    this.#lastSnapshotEventId = new Map();
+    this.#sessionStates = new Map();
+  }
+
+  open() {
+    if (this.db) return;
+    this.db = new DatabaseSync(this.dbPath, { timeout: 5000 });
+    this.db.exec("pragma journal_mode = WAL;");
+    this.db.exec("pragma foreign_keys = ON;");
+    this.db.exec("pragma synchronous = NORMAL;");
+    this.db.exec("pragma busy_timeout = 5000;");
+    this._createTables();
+  }
+
+  _createTables() {
+    this.db.exec(`
+      create table if not exists context_events (
+        id text primary key,
+        session_id text not null,
+        ts integer not null,
+        kind text not null check (kind in (
+          'message_in','message_out','tool_call','tool_result','memory_write','state_patch','compaction'
+        )),
+        payload_json text not null,
+        token_cost_in integer,
+        token_cost_out integer,
+        hash text not null,
+        parent_event_id text,
+        foreign key(parent_event_id) references context_events(id)
+      );
+
+      create index if not exists idx_context_events_session_ts
+        on context_events(session_id, ts);
+
+      create index if not exists idx_context_events_session_id
+        on context_events(session_id, id);
+
+      create unique index if not exists uq_context_events_session_hash
+        on context_events(session_id, hash);
+    `);
+
+    this.db.exec(`
+      create table if not exists context_snapshots (
+        session_id text primary key,
+        at_event_id text not null,
+        ts integer not null,
+        state_json text not null,
+        summary_compact text,
+        working_set_ids_json text not null,
+        foreign key(at_event_id) references context_events(id)
+      );
+
+      create index if not exists idx_context_snapshots_ts
+        on context_snapshots(ts);
+    `);
+
+    this.db.exec(`
+      create table if not exists context_outbox (
+        id text primary key,
+        session_id text not null,
+        ts integer not null,
+        kind text not null,
+        payload_json text not null,
+        dedupe_key text not null,
+        status text not null check (status in ('pending','processing','done','failed')),
+        locked_at integer,
+        lock_owner text,
+        attempt_count integer not null default 0,
+        last_error text,
+        done_at integer
+      );
+
+      create unique index if not exists uq_context_outbox_dedupe
+        on context_outbox(session_id, dedupe_key);
+
+      create index if not exists idx_context_outbox_status_ts
+        on context_outbox(status, ts);
+    `);
+
+    this.db.exec(`
+      create table if not exists prompt_builds (
+        id text primary key,
+        session_id text not null,
+        ts integer not null,
+        token_budget integer not null,
+        chosen_item_ids_json text not null,
+        total_tokens_est integer not null,
+        prefix_hash text not null,
+        notes text
+      );
+
+      create index if not exists idx_prompt_builds_session_ts
+        on prompt_builds(session_id, ts);
+    `);
+
+    this.db.exec(`
+      create table if not exists context_summaries (
+        id integer primary key autoincrement,
+        session_id text not null,
+        depth integer not null default 0,
+        source_event_ids text not null default '[]',
+        summary_text text not null,
+        token_count integer not null default 0,
+        created_at text not null default (datetime('now'))
+      );
+
+      create index if not exists idx_context_summaries_session_depth
+        on context_summaries(session_id, depth);
+    `);
+  }
+
+  // --- Reducer ---
+
+  #reduceEvent(payload, currentState) {
+    const p = typeof payload === "string" ? JSON.parse(payload) : payload;
+    if (p && typeof p === "object" && !Array.isArray(p)) {
+      return { ...currentState, ...p };
+    }
+    return currentState;
+  }
+
+  // --- Event log ---
+
+  _computeHash(kind, payloadJson) {
+    const payload = kind + "\n" + canonicalJson(payloadJson);
+    return createHash("sha256").update(payload).digest("hex");
+  }
+
+  appendEvent({ session_id, kind, payload_json, ts, token_cost_in, token_cost_out, parent_event_id }) {
+    if (!VALID_KINDS.has(kind)) {
+      throw new Error(`Invalid event kind: ${kind}`);
+    }
+
+    const id = randomUUID();
+    const eventTs = ts ?? Date.now();
+    const hash = this._computeHash(kind, payload_json);
+    const payloadStr = typeof payload_json === "string" ? payload_json : JSON.stringify(payload_json);
+
+    // Dedup: if same session+hash exists, return existing event
+    const existing = this.db.prepare(
+      "select id, session_id, ts, kind, payload_json, token_cost_in, token_cost_out, hash, parent_event_id from context_events where session_id = ? and hash = ?"
+    ).get(session_id, hash);
+
+    if (existing) {
+      return existing;
+    }
+
+    const { lastInsertRowid } = this.db.prepare(`
+      insert into context_events (id, session_id, ts, kind, payload_json, token_cost_in, token_cost_out, hash, parent_event_id)
+      values (?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(id, session_id, eventTs, kind, payloadStr, token_cost_in ?? null, token_cost_out ?? null, hash, parent_event_id ?? null);
+
+    // Bug B: maintain reducer-derived working state
+    if (!this.#sessionStates.has(session_id)) {
+      const snap = this.db.prepare(
+        "select state_json from context_snapshots where session_id = ?"
+      ).get(session_id);
+      this.#sessionStates.set(session_id, snap ? JSON.parse(snap.state_json) : {});
+    }
+    const newState = this.#reduceEvent(payload_json, this.#sessionStates.get(session_id));
+    this.#sessionStates.set(session_id, newState);
+
+    // Bug C: lazy-init snapshot cadence from DB on first access
+    if (!this.#lastSnapshotEventId.has(session_id)) {
+      const snapRow = this.db.prepare(
+        "select at_event_id from context_snapshots where session_id = ? order by ts desc limit 1"
+      ).get(session_id);
+      if (snapRow?.at_event_id) {
+        const eventRow = this.db.prepare(
+          "select rowid from context_events where id = ?"
+        ).get(snapRow.at_event_id);
+        this.#lastSnapshotEventId.set(session_id, eventRow?.rowid ?? null);
+      } else {
+        this.#lastSnapshotEventId.set(session_id, null);
+      }
+    }
+
+    const lastRowid = this.#lastSnapshotEventId.get(session_id) ?? 0;
+    const { cnt } = this.db.prepare(
+      "select count(*) as cnt from context_events where session_id = ? and rowid > ?"
+    ).get(session_id, lastRowid);
+    if (cnt >= this.#snapshotEveryNEvents) {
+      // Bug B: snapshot stores reducer-derived state, not raw payload
+      this.writeSnapshot({ session_id, at_event_id: id, state_json: JSON.stringify(newState), summary_compact: null, working_set_ids_json: "[]" });
+      this.#lastSnapshotEventId.set(session_id, lastInsertRowid);
+    }
+
+    return { id, session_id, ts: eventTs, kind, payload_json: payloadStr, token_cost_in: token_cost_in ?? null, token_cost_out: token_cost_out ?? null, hash, parent_event_id: parent_event_id ?? null };
+  }
+
+  getEvents(sessionId, opts = {}) {
+    const { afterEventId } = opts;
+
+    if (afterEventId) {
+      // Bug A: use rowid as tiebreaker so same-ms events are never skipped
+      const ref = this.db.prepare("select ts, rowid from context_events where id = ?").get(afterEventId);
+      if (!ref) return [];
+      return this.db.prepare(
+        "select id, session_id, ts, kind, payload_json, token_cost_in, token_cost_out, hash, parent_event_id from context_events where session_id = ? and (ts > ? or (ts = ? and rowid > ?)) order by ts asc, rowid asc"
+      ).all(sessionId, ref.ts, ref.ts, ref.rowid);
+    }
+
+    return this.db.prepare(
+      "select id, session_id, ts, kind, payload_json, token_cost_in, token_cost_out, hash, parent_event_id from context_events where session_id = ? order by ts asc, rowid asc"
+    ).all(sessionId);
+  }
+
+  // --- Summaries ---
+
+  saveContextSummary({ session_id, depth, source_event_ids, summary_text, token_count }) {
+    const idsStr = Array.isArray(source_event_ids) ? JSON.stringify(source_event_ids) : source_event_ids;
+    return this.db.prepare(`
+      insert into context_summaries (session_id, depth, source_event_ids, summary_text, token_count)
+      values (?, ?, ?, ?, ?)
+    `).run(session_id, depth, idsStr, summary_text, token_count);
+  }
+
+  getContextSummaries({ session_id, depth }) {
+    if (depth !== undefined) {
+      return this.db.prepare(
+        "select id, session_id, depth, source_event_ids, summary_text, token_count, created_at from context_summaries where session_id = ? and depth = ? order by created_at desc"
+      ).all(session_id, depth);
+    }
+    return this.db.prepare(
+      "select id, session_id, depth, source_event_ids, summary_text, token_count, created_at from context_summaries where session_id = ? order by created_at desc"
+    ).all(session_id);
+  }
+
+  deleteContextSummaries({ session_id, depth }) {
+    this.db.prepare(
+      "delete from context_summaries where session_id = ? and depth = ?"
+    ).run(session_id, depth);
+  }
+
+  // --- Snapshots ---
+
+  writeSnapshot({ session_id, at_event_id, state_json, summary_compact, working_set_ids_json, ts }) {
+    const snapTs = ts ?? Date.now();
+    const stateStr = typeof state_json === "string" ? state_json : JSON.stringify(state_json);
+
+    this.db.prepare(`
+      insert into context_snapshots (session_id, at_event_id, ts, state_json, summary_compact, working_set_ids_json)
+      values (?, ?, ?, ?, ?, ?)
+      on conflict(session_id) do update set
+        at_event_id = excluded.at_event_id,
+        ts = excluded.ts,
+        state_json = excluded.state_json,
+        summary_compact = excluded.summary_compact,
+        working_set_ids_json = excluded.working_set_ids_json
+    `).run(session_id, at_event_id, snapTs, stateStr, summary_compact ?? null, working_set_ids_json);
+  }
+
+  getLatestSnapshot(sessionId) {
+    return this.db.prepare(
+      "select session_id, at_event_id, ts, state_json, summary_compact, working_set_ids_json from context_snapshots where session_id = ?"
+    ).get(sessionId) || null;
+  }
+
+  // --- Load state (snapshot + replay) ---
+
+  loadState(sessionId) {
+    const snapshot = this.getLatestSnapshot(sessionId);
+    if (snapshot) {
+      const events = this.getEvents(sessionId, { afterEventId: snapshot.at_event_id });
+      return { snapshot, events };
+    }
+    return { snapshot: null, events: this.getEvents(sessionId) };
+  }
+
+  // --- Outbox ---
+
+  enqueueOutbox({ session_id, kind, payload_json, dedupe_key }) {
+    const id = randomUUID();
+    const ts = Date.now();
+    const payloadStr = typeof payload_json === "string" ? payload_json : JSON.stringify(payload_json);
+
+    this.db.prepare(`
+      insert or ignore into context_outbox (id, session_id, ts, kind, payload_json, dedupe_key, status, attempt_count)
+      values (?, ?, ?, ?, ?, ?, 'pending', 0)
+    `).run(id, session_id, ts, kind, payloadStr, dedupe_key);
+  }
+
+  claimOutboxBatch({ limit = 10, lockTtlMs = 2 * 60 * 1000 } = {}) {
+    const now = Date.now();
+    const staleThreshold = now - lockTtlMs;
+
+    // Claim pending items + stale processing items (lock expired)
+    this.db.exec("begin immediate");
+    try {
+      const rows = this.db.prepare(`
+        select id, session_id, ts, kind, payload_json, dedupe_key, status, locked_at, lock_owner, attempt_count, last_error, done_at
+        from context_outbox
+        where status = 'pending'
+          or (status = 'processing' and locked_at < ?)
+        order by ts asc
+        limit ?
+      `).all(staleThreshold, limit);
+
+      if (rows.length > 0) {
+        const ids = rows.map((r) => r.id);
+        const placeholders = ids.map(() => "?").join(", ");
+        this.db.prepare(
+          `update context_outbox set status = 'processing', locked_at = ? where id in (${placeholders})`
+        ).run(now, ...ids);
+      }
+
+      this.db.exec("commit");
+
+      return rows.map((r) => ({ ...r, status: "processing", locked_at: now }));
+    } catch (err) {
+      try { this.db.exec("rollback"); } catch {}
+      throw err;
+    }
+  }
+
+  markOutboxDone(id) {
+    this.db.prepare(
+      "update context_outbox set status = 'done', done_at = ? where id = ?"
+    ).run(Date.now(), id);
+  }
+
+  markOutboxFailed(id, error) {
+    this.db.prepare(
+      "update context_outbox set status = 'pending', attempt_count = attempt_count + 1, last_error = ?, locked_at = null where id = ?"
+    ).run(error, id);
+  }
+
+  // --- Lifecycle ---
+
+  close() {
+    if (!this.db) return;
+    this.db.close();
+    this.db = null;
+  }
+}

package/src/runtime/cutover-readiness.js

@@ -0,0 +1,42 @@
+export function buildCutoverReadiness({ readiness, pilotStatus } = {}) {
+  const blockers = readiness?.blockers || [];
+  const warnings = readiness?.warnings || [];
+  const tier = readiness?.readinessTier || "not_ready";
+  const healthyRuns = pilotStatus?.healthyProviderRuns || 0;
+  const recommendation = pilotStatus?.recommendation || "not_ready_for_pilot";
+
+  let verdict = "NOT_READY";
+  if (tier === "ready_for_pilot" && healthyRuns >= 2 && warnings.length === 0) {
+    verdict = "READY";
+  } else if (tier === "ready_for_pilot" || (tier === "close" && blockers.length <= 1)) {
+    verdict = "CLOSE";
+  }
+
+  const actions = [];
+  if (blockers.length > 0) {
+    actions.push(`Fix ${blockers.length} blocker(s): ${blockers.join(", ")}`);
+  }
+  if (warnings.length > 0) {
+    actions.push(`Resolve ${warnings.length} warning(s): ${warnings.join(", ")}`);
+  }
+  if (healthyRuns < 2 && tier !== "not_ready") {
+    actions.push(`Run ${2 - healthyRuns} more healthy provider-backed validation(s).`);
+  }
+  if (verdict === "READY") {
+    actions.push("Lane is ready for daily-driver cutover.");
+  }
+
+  return {
+    verdict,
+    readinessTier: tier,
+    recommendation,
+    blockers,
+    warnings,
+    healthyProviderRuns: healthyRuns,
+    actions,
+    checks: readiness?.checks || {},
+    summary: readiness?.summary || {},
+    latestProviderRun: pilotStatus?.latestProviderRun || null,
+    evaluation: pilotStatus?.evaluation || null
+  };
+}