comisai 1.0.42 → 1.0.44

package/node_modules/@comis/memory/dist/schema.js

@@ -1,14 +1,13 @@
-// SPDX-License-Identifier: Apache-2.0
-// @allow-throw: initSchema() embeddingDimensions DDL precondition guard; invalid input prevents schema interpolation; consumed at bootstrap (daemon entry — daemon.ts boundary).
-/**
- * SQLite schema initialization for the @comis/memory package.
- *
- * Creates all tables, virtual tables, indexes, and triggers required
- * by the memory system. Uses better-sqlite3 for synchronous DDL and
- * sqlite-vec for vector search support (with graceful degradation).
- */
 import * as sqliteVec from "sqlite-vec";
-import { initContextSchema } from "./context-schema.js";
+import { ensureLcdTables } from "./schema-lcd.js";
+import { ensurePinnedColumn } from "./schema-pinned.js";
+import { ensureVideoJobTable } from "./schema-video-jobs.js";
+import { ensureDurableRunTable } from "./schema-durable-runs.js";
+import { ensureOutwardLedgerTable } from "./schema-outward-ledger.js";
+import { ensureOutcomeEventsTable } from "./schema-outcome-events.js";
+import { ensureMentalModelsTable } from "./schema-mental-models.js";
+import { ensureUsefulnessFailureColumn } from "./schema-usefulness.js";
+import { ensureObsTokenColumns, ensureObsAuditTable } from "./schema-obs-token.js";
 /** Module-level flag tracking whether sqlite-vec loaded successfully. */
 let vecAvailable = false;
 /**
@@ -19,6 +18,328 @@ let vecAvailable = false;
 export function isVecAvailable() {
     return vecAvailable;
 }
+/**
+ * Additively ensure the `memories` table carries every column the current code
+ * expects, adding any absent. The package's forward-only, additive column-add path
+ * (design §4.1): SQLite has no `ADD COLUMN IF NOT EXISTS`, so each add is guarded by a
+ * `PRAGMA table_info(memories)` presence check. Safe on every boot, including a live
+ * `~/.comis` DB created before a column existed — existing rows get the column NULL (a
+ * nullable add is O(1); no rewrite, no backfill).
+ * Added columns (each documented at its add-site below): `occurred_at`,
+ * the observation set `proof_count`/`source_ids`/`consolidated_at`/
+ * `confidence`/`history`, the typed-observation pair
+ * `observation_kind`/`pattern_type`, and the lifecycle markers
+ * `lifecycle_demoted_at`/`evicted_at`/`strength` (nullable SIDE-columns,
+ * NO PK change). All nullable (NULL = the pre-feature default), NO CHECK (the enums
+ * are the Zod domain type's job).
+ *
+ * @param db - An open better-sqlite3 Database whose `memories` table already exists.
+ */
+export function ensureMemoryColumns(db) {
+    // Object-literal cast (the vec_version() probe style below); the untyped-sqlite rule targets `as Foo[]`, not an object-literal cast.
+    const cols = new Set(db.prepare(`PRAGMA table_info(memories)`).all().map((r) => r.name));
+    // Nullable add → O(1), no table rewrite, no destructive rewrite.
+    if (!cols.has("occurred_at"))
+        db.exec(`ALTER TABLE memories ADD COLUMN occurred_at INTEGER`);
+    // Observation columns. All nullable → O(1) ADD, no rewrite, no backfill
+    // (existing rows get NULL = "raw, never consolidated"). Forward-only.
+    if (!cols.has("proof_count"))
+        db.exec(`ALTER TABLE memories ADD COLUMN proof_count INTEGER`);
+    if (!cols.has("source_ids"))
+        db.exec(`ALTER TABLE memories ADD COLUMN source_ids TEXT`);
+    if (!cols.has("consolidated_at"))
+        db.exec(`ALTER TABLE memories ADD COLUMN consolidated_at INTEGER`);
+    if (!cols.has("confidence"))
+        db.exec(`ALTER TABLE memories ADD COLUMN confidence REAL`);
+    if (!cols.has("history"))
+        db.exec(`ALTER TABLE memories ADD COLUMN history TEXT`);
+    // Typed-observation columns. Both nullable → O(1) ADD, no rewrite,
+    // no backfill (existing rows get NULL: observation_kind NULL = "merge" on read, the
+    // forward-only default). NO CHECK — the enum is enforced in the MemoryEntry Zod type
+    // + the lenient LLM parser, per the occurred_at/proof_count no-CHECK precedent.
+    if (!cols.has("observation_kind"))
+        db.exec(`ALTER TABLE memories ADD COLUMN observation_kind TEXT`);
+    if (!cols.has("pattern_type"))
+        db.exec(`ALTER TABLE memories ADD COLUMN pattern_type TEXT`);
+    // Lifecycle marker columns. Nullable → O(1) ADD, no rewrite/backfill
+    // (existing rows get NULL = "not demoted / not evicted / no strength yet" = byte-
+    // identity for a pre-lifecycle DB). Forward-only, NO CHECK, NO PK CHANGE: nullable SIDE-
+    // columns on the `id`-keyed table, never an identity key (the PK-widening lesson
+    // :150-218 — a side-column over a rebuild). The sweep is SCAFFOLD-DORMANT: it
+    // computes strength/tiers but writes NONE of these markers (the deferred live policy
+    // sets them NON-DESTRUCTIVELY, a marker never a DELETE — the `consolidated_at` :62).
+    if (!cols.has("lifecycle_demoted_at"))
+        db.exec(`ALTER TABLE memories ADD COLUMN lifecycle_demoted_at INTEGER`);
+    if (!cols.has("evicted_at"))
+        db.exec(`ALTER TABLE memories ADD COLUMN evicted_at INTEGER`);
+    if (!cols.has("strength"))
+        db.exec(`ALTER TABLE memories ADD COLUMN strength REAL`);
+}
+/**
+ * Idempotently create the entity-association junction tables:
+ * `memory_entities` (one row per resolved entity, scoped to tenant+agent) and
+ * `memory_entity_links` (the many-to-many memory<->entity edge). Mirrors
+ * `ensureMemoryColumns`'s forward-only, additive contract — all DDL is `CREATE … IF
+ * NOT EXISTS`, safe on every boot including a live `~/.comis` DB created before the
+ * feature existed (no backfill: existing memories have no links until re-extracted).
+ *
+ * ## The UNIQUE index keys on `canonical_key`, NOT a SQL lower() expression
+ *
+ * RESEARCH Pitfall 3: SQLite's built-in `lower()` is ASCII-only (it leaves
+ * `İSTANBUL`/`CAFÉ`/`ПРИВЕТ` unchanged), so the original §4.2 spec's UNIQUE index
+ * over a SQL `lower(...)` of the display name would NOT dedup Turkish/CJK/Cyrillic
+ * case-variants → duplicate entities. Instead the resolver computes a
+ * locale-independent `canonical_key` in TypeScript (`normalizeEntityKey` in
+ * entity-resolver.ts: lower+NFKD+strip-marks) and we UNIQUE-index THAT stored column.
+ *
+ * The index keys on `(tenant_id, agent_id, canonical_key)` so two agents or tenants
+ * NEVER collapse to one entity row even with an identical name — the resolver-side
+ * half of the isolation boundary.
+ *
+ * `ON DELETE CASCADE` on `memory_entity_links.memory_id → memories(id)` is the
+ * ENTIRE link-maintenance story (no orphan-sweep job). It fires
+ * automatically because `openSqliteDatabase` already sets `PRAGMA foreign_keys = ON`
+ * (sqlite-adapter-base.ts:52). NB: the parent `memory_entities` row is intentionally
+ * NOT cascaded by a memory delete (entities are per-concept and may be re-linked;
+ * RESEARCH Pitfall 7), so a stale `mention_count` is by-design, not an orphan bug.
+ *
+ * @param db - An open better-sqlite3 Database whose `memories` table exists (the FK
+ *   target). Call AFTER the base `memories` CREATE.
+ */
+export function ensureEntityTables(db) {
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS memory_entities (
+      id TEXT PRIMARY KEY,
+      tenant_id TEXT NOT NULL,
+      agent_id TEXT NOT NULL,
+      canonical_name TEXT NOT NULL,            -- display form (first-seen casing)
+      canonical_key  TEXT NOT NULL,            -- normalized key (TS lower+NFKD+strip-marks; NOT sqlite lower(), Pitfall 3)
+      mention_count INTEGER NOT NULL DEFAULT 1,
+      first_seen INTEGER NOT NULL,
+      last_seen  INTEGER NOT NULL
+    );
+    CREATE UNIQUE INDEX IF NOT EXISTS idx_mement_uniq
+      ON memory_entities(tenant_id, agent_id, canonical_key);
+
+    CREATE TABLE IF NOT EXISTS memory_entity_links (
+      memory_id TEXT NOT NULL REFERENCES memories(id)        ON DELETE CASCADE,
+      entity_id TEXT NOT NULL REFERENCES memory_entities(id) ON DELETE CASCADE,
+      PRIMARY KEY (memory_id, entity_id)
+    );
+    CREATE INDEX IF NOT EXISTS idx_mentlink_entity ON memory_entity_links(entity_id);
+  `);
+}
+/**
+ * Idempotently create the recall-utility usefulness table (per-intent buckets
+ * were added later): `memory_usefulness` — one row per
+ * (tenant, agent, memory, intent) carrying the durable used/ignored counts +
+ * last-useful-at the recall-utility feedback loop learns from (HINDSIGHT_VS_COMIS.md
+ * #7). Forward-only, idempotent, re-run-safe — safe on every boot incl. a live
+ * `~/.comis` DB predating the feature (no row loss, no corruption).
+ *
+ * ## PRIMARY KEY = isolation boundary + the per-intent upsert key
+ *
+ * `PRIMARY KEY (tenant_id, agent_id, memory_id, intent)` is both the adapter's
+ * `ON CONFLICT` target and the load-bearing isolation scope — two agents/tenants
+ * NEVER share a row even for the same `memory_id` (the adapter also filters every
+ * read/write on `(tenant_id, agent_id)`). `intent` is an ADDITIONAL key, never a
+ * relaxation. `ON DELETE CASCADE` on `memory_id → memories(id)` is the ENTIRE
+ * row-maintenance story (no orphan-sweep): it fires via the `PRAGMA foreign_keys =
+ * ON` already set by `openSqliteDatabase`.
+ *
+ * ## Widening the PK on a pre-intent DB: a transactional REBUILD
+ *
+ * `intent TEXT NOT NULL DEFAULT ''` partitions the signal per query-intent (global
+ * bucket = `''`, the byte-identical pre-intent path). A FRESH DB gets the 4-col PK from
+ * `CREATE TABLE`. An EXISTING (pre-intent) DB has a 3-col PK, and SQLite has NO
+ * `ALTER ADD PRIMARY KEY` — a bare `ADD COLUMN intent` leaves it 3-col, so the
+ * adapter's 4-col `ON CONFLICT(...,intent)` aborts the SECOND intent bucket's
+ * upsert with `UNIQUE constraint failed`. So the pre-intent path runs the
+ * standard SQLite transactional table REBUILD (taken when the 4-col PK is absent
+ * via `PRAGMA table_info`): create a `_new` table with the genuine 4-col PK, copy
+ * EVERY row into the `''` bucket (`COALESCE(intent,'')` — no loss/corruption),
+ * drop, rename. Re-run-safe (4-col PK present → skip) and brackets the rename with
+ * `foreign_keys` OFF so the `memories(id)` FK is not transiently dropped.
+ *
+ * @param db - An open better-sqlite3 Database whose `memories` table already
+ *   exists (the FK target). Call AFTER `ensureEntityTables` in `initSchema`.
+ */
+export function ensureUsefulnessTable(db) {
+    // FRESH DB: the 4-col PK + intent column. EXISTING DB: no-op here (the PK-shape
+    // rebuild below widens it).
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS memory_usefulness (
+      tenant_id      TEXT NOT NULL,
+      agent_id       TEXT NOT NULL,
+      memory_id      TEXT NOT NULL REFERENCES memories(id) ON DELETE CASCADE,
+      intent         TEXT NOT NULL DEFAULT '',
+      used_count     INTEGER NOT NULL DEFAULT 0,
+      ignored_count  INTEGER NOT NULL DEFAULT 0,
+      last_useful_at INTEGER,
+      PRIMARY KEY (tenant_id, agent_id, memory_id, intent)
+    );
+  `);
+    // Detect a pre-intent (or partially-migrated) table by its PK shape (`pk>0` marks a
+    // PK member). The object-literal cast is the sanctioned PRAGMA idiom, NOT `as Foo[]`.
+    const tableInfo = db.prepare(`PRAGMA table_info(memory_usefulness)`).all();
+    const pkHasIntent = tableInfo.some((c) => c.pk > 0 && c.name === "intent");
+    if (!pkHasIntent) {
+        // EXISTING (pre-intent) DB: REBUILD to genuinely widen the PK to 4-col (ADD COLUMN
+        // cannot). A PRISTINE pre-intent table has NO `intent` column (copy the ''
+        // literal); a PARTIALLY-migrated one (column present, PK still 3-col) COALESCEs
+        // it. Toggle foreign_keys OFF around the rename (the pragma is a no-op INSIDE a
+        // txn, so it MUST bracket db.transaction) so the memories(id) FK is not dropped.
+        const intentSelectExpr = tableInfo.some((c) => c.name === "intent") ? "COALESCE(intent, '')" : "''";
+        const fkWasOn = db.pragma("foreign_keys", { simple: true }) === 1;
+        if (fkWasOn)
+            db.pragma("foreign_keys = OFF");
+        const rebuild = db.transaction(() => {
+            db.exec(`
+        CREATE TABLE memory_usefulness_new (
+          tenant_id      TEXT NOT NULL,
+          agent_id       TEXT NOT NULL,
+          memory_id      TEXT NOT NULL REFERENCES memories(id) ON DELETE CASCADE,
+          intent         TEXT NOT NULL DEFAULT '',
+          used_count     INTEGER NOT NULL DEFAULT 0,
+          ignored_count  INTEGER NOT NULL DEFAULT 0,
+          last_useful_at INTEGER,
+          PRIMARY KEY (tenant_id, agent_id, memory_id, intent)
+        );
+        INSERT INTO memory_usefulness_new (tenant_id, agent_id, memory_id, intent, used_count, ignored_count, last_useful_at)
+          SELECT tenant_id, agent_id, memory_id, ${intentSelectExpr}, used_count, ignored_count, last_useful_at FROM memory_usefulness;
+        DROP TABLE memory_usefulness;
+        ALTER TABLE memory_usefulness_new RENAME TO memory_usefulness;
+      `);
+        });
+        rebuild();
+        // Restore the pragma. No foreign_key_check needed: the INSERT…SELECT copies rows
+        // VERBATIM and the FK target (memories.id) is unchanged — no new dangling ref.
+        if (fkWasOn)
+            db.pragma("foreign_keys = ON");
+    }
+    // The explicit named 4-col index for the adapter's per-intent upsert ON CONFLICT target (idempotent; redundant with the now-4-col PK but harmless).
+    db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_usefulness_intent ON memory_usefulness(tenant_id, agent_id, memory_id, intent)`);
+    // The `failure_count` column is part of this table's complete contract (the
+    // adapter's recordFailure upsert references it eagerly at construction) — add it
+    // here so EVERY caller that ensures the usefulness table gets it, not only
+    // initSchema (v2.26 WS4, FORGET-02; distinct from ignored_count).
+    ensureUsefulnessFailureColumn(db);
+}
+/**
+ * Idempotently create the causal-edge table:
+ * `memory_causal_edges` — one row per directed cause→effect edge between two
+ * memories, scoped to a (tenant, agent). Mirrors `ensureEntityTables` /
+ * `ensureUsefulnessTable`'s forward-only, additive contract — the DDL is
+ * `CREATE TABLE IF NOT EXISTS`, so it is safe to run on every boot including a
+ * live `~/.comis` DB created before the feature existed (no backfill: existing
+ * memories simply have no causal edges until re-extracted).
+ *
+ * ## The PRIMARY KEY is the isolation boundary AND the idempotency target
+ *
+ * Comis runs many agents in one DB. `PRIMARY KEY (tenant_id, agent_id,
+ * source_memory_id, target_memory_id)` is both the `INSERT OR IGNORE` conflict
+ * target for the adapter's idempotent edge write and the load-bearing isolation
+ * scope — an edge written under one (tenant, agent) is NEVER returned for another
+ * by memory-id coincidence, and the adapter additionally filters every read/write
+ * on tenant_id + agent_id (belt-and-braces).
+ *
+ * `ON DELETE CASCADE` on BOTH `source_memory_id` and `target_memory_id →
+ * memories(id)` is the ENTIRE edge-maintenance story (no orphan-sweep job — the
+ * same cascade pattern the entity links use): a memory delete drops every edge it participates in
+ * automatically. It fires because `openSqliteDatabase` already sets
+ * `PRAGMA foreign_keys = ON` (sqlite-adapter-base.ts) — no pragma is set here.
+ * The `idx_causal_target` index serves the read lane's effect→cause direction
+ * (the source→effect direction is served by the PK prefix).
+ *
+ * @param db - An open better-sqlite3 Database whose `memories` table already
+ *   exists (the FK target). Call AFTER `ensureUsefulnessTable` in `initSchema`.
+ */
+export function ensureCausalTables(db) {
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS memory_causal_edges (
+      tenant_id        TEXT NOT NULL,
+      agent_id         TEXT NOT NULL,
+      source_memory_id TEXT NOT NULL REFERENCES memories(id) ON DELETE CASCADE,
+      target_memory_id TEXT NOT NULL REFERENCES memories(id) ON DELETE CASCADE,
+      confidence       REAL NOT NULL DEFAULT 1.0,
+      created_at       INTEGER NOT NULL,
+      PRIMARY KEY (tenant_id, agent_id, source_memory_id, target_memory_id)
+    );
+    CREATE INDEX IF NOT EXISTS idx_causal_target
+      ON memory_causal_edges(tenant_id, agent_id, target_memory_id);
+  `);
+}
+/**
+ * Create the segregated bi-temporal knowledge-graph triple table.
+ * Forward-only additive, idempotent (every
+ * `CREATE TABLE/INDEX IF NOT EXISTS`), safe on every boot — the same path serves
+ * a fresh DB and a live DB that predates the table (existing `~/.comis` DBs gain
+ * the empty table with no backfill; mirrors `ensureCausalTables`).
+ *
+ * ## Schema shape (the contract the other knowledge-graph code builds on)
+ *
+ * One row = one S/P/O assertion with the FOUR bi-temporal timestamps (Graphiti's
+ * model, epoch ms): the VALID-time pair `t_valid_start` / `t_valid_end`
+ * (`t_valid_end IS NULL` = currently believed — the default recall filter)
+ * and the TXN-time pair `t_ingested` / `expired_at` (when we learned it / when we
+ * stopped believing it). Plus the world OCCURRED range `t_occurred` /
+ * `t_occurred_end`, the Comis `trust` ladder, optional `source_memory_id`
+ * provenance, and a `confidence`.
+ *
+ * ## The PRIMARY KEY is per-row `id`, NOT the current-truth tuple
+ *
+ * History is NON-DESTRUCTIVE — many superseded versions of a
+ * (tenant, agent, subject, predicate) coexist, so the PK is the row `id`.
+ * "Single current truth" is enforced by the partial index `idx_triples_current`
+ * (on `t_valid_end IS NULL`) + the upsert transaction, NOT a UNIQUE
+ * constraint. The `ON DELETE CASCADE` on `source_memory_id -> memories(id)`
+ * fires via the `PRAGMA foreign_keys = ON` already set by `openSqliteDatabase`
+ * (no pragma is set here) — deleting a source memory drops its derived triples.
+ *
+ * ## Isolation (the §5.2 invariant)
+ *
+ * Every row carries `tenant_id` + `agent_id`, and every index leads with them —
+ * the adapter (sqlite-triple-store.ts) filters every statement on
+ * `(tenant_id, agent_id)`. The `trust` CHECK rejects an out-of-ladder value at
+ * write.
+ *
+ * @param db - An open better-sqlite3 Database whose `memories` table already
+ *   exists (the FK target). Call AFTER `ensureCausalTables` in `initSchema`.
+ */
+export function ensureTripleTable(db) {
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS memory_triples (
+      id               TEXT NOT NULL,
+      tenant_id        TEXT NOT NULL,
+      agent_id         TEXT NOT NULL,
+      subject          TEXT NOT NULL,
+      predicate        TEXT NOT NULL,
+      object           TEXT NOT NULL,
+      trust            TEXT NOT NULL CHECK(trust IN ('system','learned','external')),
+      t_valid_start    INTEGER NOT NULL,
+      t_valid_end      INTEGER,
+      t_ingested       INTEGER NOT NULL,
+      expired_at       INTEGER,
+      t_occurred       INTEGER,
+      t_occurred_end   INTEGER,
+      source_memory_id TEXT REFERENCES memories(id) ON DELETE CASCADE,
+      confidence       REAL,
+      PRIMARY KEY (id)
+    );
+    CREATE INDEX IF NOT EXISTS idx_triples_current
+      ON memory_triples(tenant_id, agent_id, subject, predicate) WHERE t_valid_end IS NULL;
+    CREATE INDEX IF NOT EXISTS idx_triples_validtime
+      ON memory_triples(tenant_id, agent_id, t_valid_start, t_valid_end);
+    CREATE INDEX IF NOT EXISTS idx_triples_subject
+      ON memory_triples(tenant_id, agent_id, subject) WHERE t_valid_end IS NULL;
+  `);
+}
+// (The `relationship` table — the sole storage for the directional social-modeling
+//  subsystem — was DELETED in Phase 226 SIMPLIFY-03 with the rest of that subsystem
+//  (the __SOCIAL_MODELING__ cron, the RelationshipStore port + adapter, the
+//  relationship-block prompt injection). A fresh DB no longer creates it; an existing
+//  DB's orphaned `relationship` table is inert (never read/written) — no migration, no
+//  read path. No alias, I1.)
 /**
  * Initialize the full memory schema on the given SQLite database.
  *
@@ -82,13 +403,6 @@ export function initSchema(db, embeddingDimensions) {
     CREATE INDEX IF NOT EXISTS idx_memories_created ON memories(created_at);
     CREATE INDEX IF NOT EXISTS idx_memories_expires ON memories(expires_at) WHERE expires_at IS NOT NULL;
   `);
-    // --- Migration: add agent_id column for multi-agent memory isolation ---
-    try {
-        db.exec(`ALTER TABLE memories ADD COLUMN agent_id TEXT NOT NULL DEFAULT 'default'`);
-    }
-    catch {
-        // Column already exists -- safe to ignore (SQLite throws on duplicate ADD COLUMN)
-    }
     // Index for agent-scoped queries
     db.exec(`CREATE INDEX IF NOT EXISTS idx_memories_agent ON memories(agent_id);`);
     // --- Vector search table (sqlite-vec) ---
@@ -153,30 +467,35 @@ export function initSchema(db, embeddingDimensions) {
     CREATE INDEX IF NOT EXISTS idx_sessions_tenant ON sessions(tenant_id);
     CREATE INDEX IF NOT EXISTS idx_sessions_updated ON sessions(updated_at);
   `);
-    // --- Archives table (compaction service) ---
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS archives (
-      session_key TEXT NOT NULL,
-      messages TEXT NOT NULL,
-      archived_at INTEGER NOT NULL,
-      expires_at INTEGER NOT NULL
-    );
-    CREATE INDEX IF NOT EXISTS idx_archives_expires ON archives(expires_at);
-  `);
-    // --- Identity links table (cross-platform user recognition) ---
+    // NOTE: the DAG context-store tables (ctx_*) were removed in v2.12 (Phase 126,
+    // LCD reimplementation) — only the schema-create call is gone (no reverse migration; existing DBs keep harmless orphaned tables, design §9).
+    // The calls below run in dependency order AFTER the `memories` table (the FK
+    // target) exists; each is idempotent, and every `ON DELETE CASCADE` fires via
+    // the `PRAGMA foreign_keys = ON` already set by `openSqliteDatabase`. Per-table contracts (schema shape, isolation scope, trust floor) live in each fn JSDoc.
+    ensureMemoryColumns(db); // additive memory columns (forward-only; design §4.1)
+    ensureEntityTables(db); // entity junction tables
+    ensureUsefulnessTable(db); // recall-utility usefulness + intent bucket + failure_count (v2.26 WS4, FORGET-02)
+    ensureCausalTables(db); // causal-edge table
+    ensureTripleTable(db); // bi-temporal KG triples
+    // (ensureRelationshipTable — the directional social-modeling table — was DELETED in
+    //  Phase 226 SIMPLIFY-03 with the rest of that subsystem; orphaned tables on existing
+    //  DBs are inert, no migration.)
+    ensureLcdTables(db); // LCD lossless message + parts store (Phase 127)
+    ensurePinnedColumn(db); // pinned-memory column + partial index (forward-only; design §4.1)
+    ensureVideoJobTable(db); // durable async video-job store (Phase 189, JOB-01/JOB-03)
+    ensureDurableRunTable(db); // durable run checkpoint store (Phase 216, DUR-01)
+    ensureOutwardLedgerTable(db); // exactly-once outward send ledger (Phase 216, ONCE-01)
+    ensureOutcomeEventsTable(db); // outcome_events ledger (v2.26 WS1, OUTCOME-01) — no FK, (tenant,agent)-scoped
+    ensureMentalModelsTable(db, embeddingDimensions, localVecAvailable); // mental_models doc store + FTS/vec/trigram twins (v2.31; generalized from v2.26 WS2 SKILL-01) — trust CHECK IN ('learned'), (tenant,agent)-scoped; copies a pre-existing learned_skills table forward as kind='skill'
+    // --- Observation partial indexes (design §4.1) --- created AFTER ensureMemoryColumns (indexed columns must exist first).
+    // `idx_memories_unconsol` serves the candidate scan (consolidated_at IS NULL); `idx_memories_observations` serves the observation lookup (proof_count IS NOT NULL).
+    // The design's third "live" index (exact-dup-retirement) is OMITTED — deferred to a later phase.
     db.exec(`
-    CREATE TABLE IF NOT EXISTS identity_links (
-      canonical_id TEXT NOT NULL,
-      provider TEXT NOT NULL,
-      provider_user_id TEXT NOT NULL,
-      display_name TEXT,
-      linked_at INTEGER NOT NULL,
-      PRIMARY KEY (provider, provider_user_id)
-    );
-    CREATE INDEX IF NOT EXISTS idx_identity_canonical ON identity_links(canonical_id);
+    CREATE INDEX IF NOT EXISTS idx_memories_unconsol
+      ON memories(agent_id, created_at) WHERE consolidated_at IS NULL;
+    CREATE INDEX IF NOT EXISTS idx_memories_observations
+      ON memories(agent_id) WHERE proof_count IS NOT NULL;
   `);
-    // --- Context store tables (DAG schema) ---
-    initContextSchema(db);
     // --- Observability persistence tables ---
     db.exec(`
     CREATE TABLE IF NOT EXISTS obs_token_usage (
@@ -185,7 +504,6 @@ export function initSchema(db, embeddingDimensions) {
       trace_id TEXT NOT NULL,
       agent_id TEXT NOT NULL,
       channel_id TEXT DEFAULT '',
-      execution_id TEXT DEFAULT '',
       session_key TEXT DEFAULT '',
       provider TEXT NOT NULL,
       model TEXT NOT NULL,
@@ -201,31 +519,25 @@ export function initSchema(db, embeddingDimensions) {
       cost_cache_write REAL NOT NULL DEFAULT 0,
       cache_saved REAL NOT NULL DEFAULT 0,
       latency_ms INTEGER NOT NULL,
-      cache_retention TEXT DEFAULT NULL
+      -- PERSIST-02 cost-correctness columns (fresh DB; ensureObsTokenColumns ALTERs an
+      -- existing DB). The dead cache_retention column is intentionally ABSENT (OQ3 DROP).
+      warmup_turn INTEGER,
+      cache_eligible INTEGER,
+      cost_correction REAL,
+      pending_cache_investment_usd REAL,
+      pricing_state TEXT
     );
     CREATE INDEX IF NOT EXISTS idx_obs_token_timestamp ON obs_token_usage(timestamp);
     CREATE INDEX IF NOT EXISTS idx_obs_token_agent ON obs_token_usage(agent_id, timestamp);
     CREATE INDEX IF NOT EXISTS idx_obs_token_provider ON obs_token_usage(provider, timestamp);
     CREATE INDEX IF NOT EXISTS idx_obs_token_session ON obs_token_usage(session_key, timestamp);
   `);
-    // --- Migration: add cache cost columns to obs_token_usage ---
-    try {
-        db.exec(`ALTER TABLE obs_token_usage ADD COLUMN cost_cache_read REAL NOT NULL DEFAULT 0`);
-    }
-    catch { /* Column already exists */ }
-    try {
-        db.exec(`ALTER TABLE obs_token_usage ADD COLUMN cost_cache_write REAL NOT NULL DEFAULT 0`);
-    }
-    catch { /* Column already exists */ }
-    try {
-        db.exec(`ALTER TABLE obs_token_usage ADD COLUMN cache_saved REAL NOT NULL DEFAULT 0`);
-    }
-    catch { /* Column already exists */ }
-    // --- Migration: add cache_retention column to obs_token_usage ---
-    try {
-        db.exec(`ALTER TABLE obs_token_usage ADD COLUMN cache_retention TEXT DEFAULT NULL`);
-    }
-    catch { /* Column already exists */ }
+    // PERSIST-02 + AUDIT-01/02 (both in schema-obs-token.ts, 800L cap). Called HERE,
+    // after the obs_token_usage CREATE — NOT in the ensure-call sequence above (runs
+    // first). ensureObsTokenColumns: forward-only EXISTING-DB upgrade (drop dead
+    // cache_retention + add the 5 cols). ensureObsAuditTable: obs_audit_events + indexes.
+    ensureObsTokenColumns(db);
+    ensureObsAuditTable(db);
     db.exec(`
     CREATE TABLE IF NOT EXISTS obs_delivery (
       id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -262,7 +574,7 @@ export function initSchema(db, embeddingDimensions) {
     CREATE INDEX IF NOT EXISTS idx_obs_diag_timestamp ON obs_diagnostics(timestamp);
     CREATE INDEX IF NOT EXISTS idx_obs_diag_category ON obs_diagnostics(category, timestamp);
     CREATE INDEX IF NOT EXISTS idx_obs_diag_severity ON obs_diagnostics(severity, timestamp);
-
+    CREATE INDEX IF NOT EXISTS idx_obs_diag_session_cat ON obs_diagnostics(session_key, category, timestamp);
     CREATE TABLE IF NOT EXISTS obs_channel_snapshots (
       id INTEGER PRIMARY KEY AUTOINCREMENT,
       timestamp INTEGER NOT NULL,
@@ -275,6 +587,26 @@ export function initSchema(db, embeddingDimensions) {
     );
     CREATE INDEX IF NOT EXISTS idx_obs_channel_timestamp ON obs_channel_snapshots(timestamp);
     CREATE INDEX IF NOT EXISTS idx_obs_channel_type ON obs_channel_snapshots(channel_type, timestamp);
+
+    -- SystemPromptReport persistence.
+    -- Full JSON payload stored after sanitizeForPersistence pipeline.
+    -- PRIMARY KEY (agent_id, session_id, run_id, generated_at) — run_id is
+    -- nullable; composite key tolerates multiple reports per session.
+    CREATE TABLE IF NOT EXISTS system_prompt_reports (
+      agent_id        TEXT NOT NULL,
+      tenant_id       TEXT,
+      session_id      TEXT NOT NULL,
+      run_id          TEXT,
+      generated_at    INTEGER NOT NULL,
+      provider        TEXT,
+      model           TEXT,
+      system_chars    INTEGER NOT NULL,
+      system_sha256   TEXT NOT NULL,
+      report_json     TEXT NOT NULL,
+      PRIMARY KEY (agent_id, session_id, run_id, generated_at)
+    );
+    CREATE INDEX IF NOT EXISTS idx_spr_session
+      ON system_prompt_reports(agent_id, session_id, generated_at);
   `);
     // --- Delivery queue table ---
     db.exec(`
@@ -286,8 +618,6 @@ export function initSchema(db, embeddingDimensions) {
       tenant_id TEXT NOT NULL DEFAULT 'default',
       options_json TEXT NOT NULL DEFAULT '{}',
       origin TEXT NOT NULL DEFAULT 'unknown',
-      format_applied INTEGER NOT NULL DEFAULT 0,
-      chunking_applied INTEGER NOT NULL DEFAULT 0,
       status TEXT NOT NULL DEFAULT 'pending'
         CHECK(status IN ('pending', 'in_flight', 'delivered', 'failed', 'expired')),
       attempt_count INTEGER NOT NULL DEFAULT 0,
@@ -298,8 +628,6 @@ export function initSchema(db, embeddingDimensions) {
       last_attempt_at INTEGER,
       next_retry_at INTEGER,
       last_error TEXT,
-      markdown_fallback_applied INTEGER NOT NULL DEFAULT 0,
-      delivered_message_id TEXT,
       trace_id TEXT
     );
     CREATE INDEX IF NOT EXISTS idx_dq_status_scheduled

package/node_modules/@comis/memory/dist/secret-store-schema.js

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: Apache-2.0
-// @allow-throw: Canary encrypt / decrypt assertion: master-key mismatch must surface as a hard failure (not a Result.err that the caller might silently swallow). Encryption-correctness boundary per AGENTS.md §2.1 (security-critical assertion zone analog).
+// @allow-throw: Canary encrypt / decrypt assertion: master-key mismatch must surface as a hard failure (not a Result.err that the caller might silently swallow). Encryption-correctness boundary (security-critical assertion zone).
 /**
  * Secret store SQLite schema DDL and canary-based master key validation.
  *
@@ -39,8 +39,6 @@ export function initSecretSchema(db) {
       provider TEXT,
       description TEXT,
       expires_at INTEGER,
-      last_used_at INTEGER,
-      usage_count INTEGER NOT NULL DEFAULT 0,
       created_at INTEGER NOT NULL,
       updated_at INTEGER NOT NULL
     )
@@ -61,22 +59,30 @@ export function initSecretSchema(db) {
  * @throws Error with "DECRYPTION_FAILED" if master key mismatch detected
  */
 export function validateCanary(db, crypto) {
+    // Seed the canary IDEMPOTENTLY. A check-then-INSERT (SELECT, then `if (!row)
+    // INSERT`) is a TOCTOU race: two daemons booting in parallel against a shared
+    // secrets.db (the test harness shares ~/.comis; a restart can race itself)
+    // can both observe the canary as absent and both INSERT, and the loser
+    // crashed with `UNIQUE constraint failed: secrets.name`
+    // (SQLITE_CONSTRAINT_PRIMARYKEY). `ON CONFLICT(name) DO NOTHING` makes a lost
+    // race a harmless no-op; we then read back whatever canary is persisted (ours
+    // or the winner's) and validate it — preserving wrong-key/tamper detection.
+    const result = crypto.encrypt(CANARY_PLAINTEXT);
+    if (!result.ok) {
+        throw new Error(`Failed to encrypt canary: ${result.error.message}`);
+    }
+    const candidate = result.value;
+    const now = systemNowMs();
+    db.prepare(`INSERT INTO secrets (name, ciphertext, iv, auth_tag, salt, created_at, updated_at)
+     VALUES (?, ?, ?, ?, ?, ?, ?)
+     ON CONFLICT(name) DO NOTHING`).run(CANARY_NAME, candidate.ciphertext, candidate.iv, candidate.authTag, candidate.salt, now, now);
+    // The INSERT above guarantees a canary row exists; read + validate it.
     const row = db
         .prepare("SELECT ciphertext, iv, auth_tag, salt FROM secrets WHERE name = ?")
         .get(CANARY_NAME);
     if (!row) {
-        // First initialization: encrypt and store the canary
-        const result = crypto.encrypt(CANARY_PLAINTEXT);
-        if (!result.ok) {
-            throw new Error(`Failed to encrypt canary: ${result.error.message}`);
-        }
-        const encrypted = result.value;
-        const now = systemNowMs();
-        db.prepare(`INSERT INTO secrets (name, ciphertext, iv, auth_tag, salt, created_at, updated_at)
-       VALUES (?, ?, ?, ?, ?, ?, ?)`).run(CANARY_NAME, encrypted.ciphertext, encrypted.iv, encrypted.authTag, encrypted.salt, now, now);
-        return;
+        throw new Error("DECRYPTION_FAILED: secrets canary missing after seed");
     }
-    // Existing canary: reconstruct EncryptedSecret and decrypt
     const encrypted = {
         ciphertext: row.ciphertext,
         iv: row.iv,

package/node_modules/@comis/memory/dist/select-secret-store.d.ts

@@ -0,0 +1,53 @@
+/**
+ * selectSecretStore — discriminated SecretStore factory.
+ * Returns SelectedSecretStore per security.storage mode.
+ * encrypted → SqliteSecretStoreHandle (owns the db handle; caller must close last)
+ * file      → FileSecretStore (sync-atomic secrets.json, plaintext-at-rest)
+ * env       → EnvSecretStore (read-only snapshot, set/delete → err)
+ *
+ * NOTE: dbPath is NOT a parameter. security.secrets.dbPath was a dead config knob
+ * (never read by any caller). The encrypted path uses setupSecrets, whose
+ * "secrets.db" hardcode is now the canonical, no-longer-configurable filename.
+ */
+import type Database from "better-sqlite3";
+import type { Result } from "@comis/shared";
+import type { SecretStorePort, SecretsCrypto, CredentialStorageMode } from "@comis/core";
+import type { SqliteSecretStoreHandle } from "./sqlite-secret-store.js";
+/**
+ * Discriminated return type of selectSecretStore.
+ *
+ * - "encrypted": SqliteSecretStore with exposed db + crypto for adapter sharing.
+ *   secretStore.close() OWNS the shared db handle; OAuth/MCP adapters built on it
+ *   must NOT close it directly.
+ * - "file": FileSecretStore (sync-atomic, plaintext-at-rest).
+ * - "env": Read-only snapshot adapter. set()/delete() return err with actionable hint.
+ */
+export type SelectedSecretStore = {
+    kind: "encrypted";
+    secretStore: SqliteSecretStoreHandle;
+    secretsDb: Database.Database;
+    secretsCrypto: SecretsCrypto;
+} | {
+    kind: "file";
+    secretStore: SecretStorePort;
+} | {
+    kind: "env";
+    secretStore: SecretStorePort;
+};
+/**
+ * Select and construct the appropriate SecretStore based on the configured
+ * credential storage mode.
+ *
+ * @param input.mode     - Credential storage mode from security.storage config
+ * @param input.dataDir  - Data directory (used by file and encrypted modes)
+ * @param input.env      - Environment snapshot (used by encrypted key resolution + env-mode snapshot)
+ * @param input.sensitiveNames - Name set for env-mode scoping (default: empty set)
+ *
+ * @returns ok(SelectedSecretStore) on success, err on encrypted mode key failure.
+ */
+export declare function selectSecretStore(input: {
+    mode: CredentialStorageMode;
+    dataDir: string;
+    env: Record<string, string | undefined>;
+    sensitiveNames?: Set<string>;
+}): Result<SelectedSecretStore, Error>;