@jbarket/loomai 0.5.0 → 0.6.0

package/dist/backends/migrations.js

@@ -1,3 +1,4 @@
+import { rebuildUniqKeys } from './dedup.js';
 function hasColumn(db, table, column) {
     const info = db.pragma(`table_info(${table})`);
     return info.some((c) => c.name === column);
@@ -96,6 +97,24 @@ export const MIGRATIONS = [
             db.prepare('CREATE INDEX idx_memory_revisions_memory ON memory_revisions(memory_id)').run();
         },
     },
+    {
+        id: 'add_sourcing',
+        description: 'Add sourcing column — provenance/trust tier: observed | relayed | inferred | system (t-328)',
+        pending: (db) => !hasColumn(db, 'memories', 'sourcing'),
+        run: (db) => {
+            // NULL on legacy records: "unknown". New writes should supply a value;
+            // the lint-on-write validator will warn (not reject) when omitted.
+            db.prepare('ALTER TABLE memories ADD COLUMN sourcing TEXT').run();
+        },
+    },
+    {
+        id: 'add_provenance',
+        description: 'Add provenance column — free-form origin (surface / person / session) for trust layer (t-328)',
+        pending: (db) => !hasColumn(db, 'memories', 'provenance'),
+        run: (db) => {
+            db.prepare('ALTER TABLE memories ADD COLUMN provenance TEXT').run();
+        },
+    },
     {
         id: 'add_memory_supersessions',
         description: 'Add memory_supersessions table — records that new_ref replaced old_ref, ' +
@@ -119,6 +138,187 @@ export const MIGRATIONS = [
             db.prepare('CREATE INDEX idx_memory_supersessions_new ON memory_supersessions(new_ref)').run();
         },
     },
+    {
+        id: 'add_confidence',
+        description: 'Add confidence column — evidence-backed trust score for feedback memories, ' +
+            'computed in code (t-338)',
+        pending: (db) => !hasColumn(db, 'memories', 'confidence'),
+        run: (db) => {
+            // NULL = not applicable (non-feedback memories never get a confidence
+            // score). remember() seeds it for category='feedback'; update() adjusts
+            // it on a confirm/contradict observation.
+            db.prepare('ALTER TABLE memories ADD COLUMN confidence REAL').run();
+        },
+    },
+    {
+        id: 'add_evidence_count',
+        description: 'Add evidence_count column — number of observations backing a feedback ' +
+            'memory\'s confidence (t-338)',
+        pending: (db) => !hasColumn(db, 'memories', 'evidence_count'),
+        run: (db) => {
+            db.prepare('ALTER TABLE memories ADD COLUMN evidence_count INTEGER NOT NULL DEFAULT 0').run();
+        },
+    },
+    {
+        id: 'add_observed_at',
+        description: 'Add observed_at column — when a feedback memory\'s confidence last took an ' +
+            'observation; the decay clock, separate from `updated` (t-338)',
+        pending: (db) => !hasColumn(db, 'memories', 'observed_at'),
+        run: (db) => {
+            // NULL falls back to `created`. Rows that already carry a confidence
+            // got it from an observation no later than their last update, so
+            // that's the best clock available for them.
+            db.prepare('ALTER TABLE memories ADD COLUMN observed_at TEXT').run();
+            db.prepare('UPDATE memories SET observed_at = COALESCE(updated, created) WHERE confidence IS NOT NULL').run();
+        },
+    },
+    {
+        id: 'add_memory_deletions',
+        description: 'Add memory_deletions table — an append-only journal of every hard delete ' +
+            '(deleteById), so forget()/prune() leave a trace of what was removed and ' +
+            'why (t-436). Standalone, keyed by ref — deliberately NOT a child table of ' +
+            'memories, so it survives the delete it records (that FK-cascade blind spot ' +
+            'is the bug this fixes).',
+        pending: (db) => !hasTable(db, 'memory_deletions'),
+        run: (db) => {
+            db.prepare(`
+        CREATE TABLE memory_deletions (
+          id         INTEGER PRIMARY KEY AUTOINCREMENT,
+          ref        TEXT NOT NULL,
+          category   TEXT NOT NULL,
+          title      TEXT NOT NULL,
+          project    TEXT,
+          content    TEXT NOT NULL,
+          created    TEXT NOT NULL,
+          deleted_at TEXT NOT NULL,
+          op         TEXT NOT NULL
+        )
+      `).run();
+            db.prepare('CREATE INDEX idx_memory_deletions_ref ON memory_deletions(ref)').run();
+        },
+    },
+    {
+        id: 'add_scope',
+        description: 'Add scope column — explicit project|global visibility override, enforced ' +
+            'at recall against the category default (backends/scope.ts, t-337)',
+        pending: (db) => !hasColumn(db, 'memories', 'scope'),
+        run: (db) => {
+            // NULL on legacy records and any write that doesn't set it explicitly:
+            // effectiveScope() falls back to the category default in that case.
+            db.prepare('ALTER TABLE memories ADD COLUMN scope TEXT').run();
+        },
+    },
+    {
+        id: 'add_times_seen',
+        description: 'Add times_seen counter — bumped when remember() reheats an exact restatement instead of inserting (t-335)',
+        pending: (db) => !hasColumn(db, 'memories', 'times_seen'),
+        run: (db) => {
+            // DEFAULT 1 backfills existing rows too: each was "seen" at least once
+            // (the write that created it), even though restatements before t-335
+            // weren't counted.
+            db.prepare('ALTER TABLE memories ADD COLUMN times_seen INTEGER NOT NULL DEFAULT 1').run();
+        },
+    },
+    {
+        id: 'add_uniq',
+        description: 'Add uniq dedup key column + index, backfilled from existing rows, so write-time ' +
+            'remember() can reheat an exact restatement instead of inserting a duplicate (t-335)',
+        pending: (db) => !hasColumn(db, 'memories', 'uniq'),
+        run: (db) => {
+            db.prepare('ALTER TABLE memories ADD COLUMN uniq TEXT').run();
+            db.prepare('CREATE INDEX idx_memories_uniq ON memories(uniq)').run();
+            // Backfill: compute uniq for every existing ACTIVE row so future
+            // restatements of pre-t-335 content also reheat. Archived rows stay
+            // NULL and, within a pre-existing duplicate cluster, only the oldest
+            // active row gets the key — see rebuildUniqKeys().
+            rebuildUniqKeys(db);
+        },
+    },
+    {
+        id: 'rekey_uniq_unicode',
+        description: 'Recompute every uniq key with the Unicode-preserving normalizer — the first ' +
+            'normalizer stripped all non-ASCII text, so distinct non-Latin memories shared ' +
+            'one key (PR120 review)',
+        // Marker: the index this migration creates. Databases that ran add_uniq
+        // under the old normalizer have only idx_memories_uniq.
+        // uniq_active_unique (below) re-keys too and replaces this index.
+        pending: (db) => !hasIndex(db, 'memories', 'idx_memories_uniq_v2') &&
+            !hasIndex(db, 'memories', 'idx_memories_uniq_active') &&
+            !hasIndex(db, 'memories', 'idx_memories_uniq_exact'),
+        run: (db) => {
+            rebuildUniqKeys(db);
+            db.prepare('CREATE INDEX IF NOT EXISTS idx_memories_uniq_v2 ON memories(uniq)').run();
+            db.prepare('DROP INDEX IF EXISTS idx_memories_uniq').run();
+        },
+    },
+    {
+        id: 'uniq_active_unique',
+        description: 'Make uniq UNIQUE among active rows (partial index), so two concurrent ' +
+            'remember() calls of the same content cannot both insert (PR120 review)',
+        // rekey_uniq_exact (below) replaces this index with idx_memories_uniq_exact.
+        pending: (db) => !hasIndex(db, 'memories', 'idx_memories_uniq_active') &&
+            !hasIndex(db, 'memories', 'idx_memories_uniq_exact'),
+        run: (db) => {
+            db.transaction(() => {
+                // Re-key first: any active duplicates written before this index
+                // existed (the race it closes, or stale keys from before update()
+                // refreshed them) would otherwise make CREATE UNIQUE INDEX fail.
+                rebuildUniqKeys(db);
+                db.prepare('CREATE UNIQUE INDEX idx_memories_uniq_active ON memories(uniq) WHERE archived = 0').run();
+                db.prepare('DROP INDEX IF EXISTS idx_memories_uniq_v2').run();
+                db.prepare('DROP INDEX IF EXISTS idx_memories_uniq').run();
+            }).immediate();
+        },
+    },
+    {
+        id: 'rekey_uniq_exact',
+        description: 'Recompute every uniq key with the exact normalizer — the previous one folded ' +
+            'case, punctuation and whitespace, and compared project ids loosely, so ' +
+            '"Offset -5"/"Offset 5", "1.5"/"15" and projects foo-bar/foobar shared a key ' +
+            '(PR120 review, round 2)',
+        // Marker: the unique index this migration creates, under a new name so
+        // databases keyed by the folding normalizer are detectable.
+        pending: (db) => !hasIndex(db, 'memories', 'idx_memories_uniq_exact'),
+        run: (db) => {
+            db.transaction(() => {
+                // Drop the old indexes first so the new unique index is built once,
+                // over the final keys only.
+                db.prepare('DROP INDEX IF EXISTS idx_memories_uniq_active').run();
+                db.prepare('DROP INDEX IF EXISTS idx_memories_uniq_v2').run();
+                db.prepare('DROP INDEX IF EXISTS idx_memories_uniq').run();
+                rebuildUniqKeys(db);
+                db.prepare('CREATE UNIQUE INDEX idx_memories_uniq_exact ON memories(uniq) WHERE archived = 0').run();
+            }).immediate();
+        },
+    }, {
+        id: 'people_cards',
+        description: 'Add people + person_revisions: person cards, the condensed current state of a ' +
+            'human the agent works with (docs/specs/person-cards.md, t-1376). Additive — ' +
+            'dropping both tables is the whole backout',
+        pending: (db) => !hasTable(db, 'people') || !hasTable(db, 'person_revisions'),
+        run: (db) => {
+            db.transaction(() => {
+                db.prepare(`CREATE TABLE IF NOT EXISTS people (
+            slug           TEXT PRIMARY KEY,
+            name           TEXT NOT NULL,
+            card_json      TEXT NOT NULL,
+            archived_at    TEXT,
+            archived_by    TEXT,
+            archive_reason TEXT,
+            updated        TEXT NOT NULL,
+            synthesized_at TEXT
+          )`).run();
+                db.prepare(`CREATE TABLE IF NOT EXISTS person_revisions (
+            id            INTEGER PRIMARY KEY AUTOINCREMENT,
+            slug          TEXT NOT NULL,
+            snapshot_json TEXT NOT NULL,
+            op            TEXT NOT NULL,
+            replaced_at   TEXT NOT NULL
+          )`).run();
+                db.prepare('CREATE INDEX IF NOT EXISTS idx_person_revisions_slug ON person_revisions(slug, id)').run();
+            }).immediate();
+        },
+    },
 ];
 /**
  * Apply pending migrations to a database.

package/dist/backends/mmr.d.ts

@@ -25,6 +25,14 @@ export interface MmrCandidate {
 export interface MmrResult<T> {
     /** Chosen candidates in selection order (first pick = most relevant). */
     selected: T[];
+    /**
+     * The score that actually drove each pick, same order/index as `selected`:
+     * score(c) = λ·relevance(c) − (1−λ)·max_{s ∈ selected} sim(c, s). Equals
+     * the candidate's own `relevance` when MMR made no change (diversity 0 or
+     * pool <= limit) — "blended" only means something once MMR has traded a
+     * pick off against the ones already chosen.
+     */
+    scores: number[];
     /**
      * How many of the top-`limit` candidates by relevance were displaced by
      * a less-relevant-but-different one. 0 means MMR changed nothing.

package/dist/backends/mmr.js

@@ -19,7 +19,18 @@
 /** Default `diversity` (1 − λ) for recall: λ = 0.7. */
 export const DEFAULT_DIVERSITY = 0.3;
 export function cosineSimilarity(a, b) {
-    const n = Math.min(a.length, b.length);
+    // A zero-length vector is the "no embedding available" sentinel (e.g. a
+    // lookup miss in the caller) — treat it as no similarity, same as before.
+    // A mismatch between two REAL vectors means the embedding dimension
+    // changed between model versions somewhere upstream; silently comparing
+    // the overlapping prefix (the old Math.min(a.length, b.length) behavior)
+    // hid that corruption instead of catching it. Raise instead (t-334).
+    if (a.length === 0 || b.length === 0)
+        return 0;
+    if (a.length !== b.length) {
+        throw new Error(`cosineSimilarity: dimension mismatch (${a.length} vs ${b.length}) — embeddings from different model versions can't be compared`);
+    }
+    const n = a.length;
     let dot = 0;
     let na = 0;
     let nb = 0;
@@ -42,13 +53,15 @@ export function cosineSimilarity(a, b) {
  */
 export function mmrSelect(candidates, limit, diversity = DEFAULT_DIVERSITY) {
     if (!(limit > 0))
-        return { selected: [], diversityDrops: 0 };
+        return { selected: [], scores: [], diversityDrops: 0 };
     if (!(diversity > 0) || candidates.length <= limit) {
-        return { selected: candidates.slice(0, limit), diversityDrops: 0 };
+        const picked = candidates.slice(0, limit);
+        return { selected: picked, scores: picked.map((c) => c.relevance), diversityDrops: 0 };
     }
     const lambda = 1 - Math.min(diversity, 1);
     const remaining = candidates.map((_, i) => i);
     const chosen = [];
+    const chosenScores = [];
     // maxSim[i]: similarity of candidate i to the closest already-selected item.
     const maxSim = new Float64Array(candidates.length);
     while (chosen.length < limit && remaining.length > 0) {
@@ -64,6 +77,7 @@ export function mmrSelect(candidates, limit, diversity = DEFAULT_DIVERSITY) {
             }
         }
         chosen.push(best);
+        chosenScores.push(bestScore);
         remaining.splice(remaining.indexOf(best), 1);
         const pickedVec = candidates[best].vector;
         for (const i of remaining) {
@@ -78,5 +92,5 @@ export function mmrSelect(candidates, limit, diversity = DEFAULT_DIVERSITY) {
         .slice(0, limit);
     const chosenSet = new Set(chosen);
     const diversityDrops = topByRelevance.filter((i) => !chosenSet.has(i)).length;
-    return { selected: chosen.map((i) => candidates[i]), diversityDrops };
+    return { selected: chosen.map((i) => candidates[i]), scores: chosenScores, diversityDrops };
 }

package/dist/backends/people.d.ts

@@ -0,0 +1,192 @@
+export declare const SLUG_RE: RegExp;
+export declare const MAX_SLUG = 64;
+export declare const MAX_RENDER = 2000;
+/**
+ * The harness's gating names. Fixed here, not configured: a card that carried
+ * one would look like a gate, and the agent writes its own cards.
+ */
+export declare const RESERVED_KEYS: readonly ['tier', 'field', 'projects', 'scope', 'trust'];
+export type CardValue = string | number | boolean | string[];
+export interface WhereEnvelope {
+    value: CardValue;
+    verified_at?: string;
+    source?: string;
+    stale_after_days?: number;
+}
+export interface MayEntry {
+    text: string;
+    source: string;
+}
+export interface PersonCard {
+    slug: string;
+    name: string;
+    summary?: string;
+    where: Record<string, WhereEnvelope>;
+    how: Record<string, CardValue>;
+    may: MayEntry[];
+}
+export interface PersonWriteInput {
+    slug: string;
+    name?: string;
+    summary?: string;
+    where?: Record<string, WhereEnvelope | null>;
+    how?: Record<string, CardValue | null>;
+    may?: MayEntry[];
+    merge?: boolean;
+}
+export interface PersonRow {
+    slug: string;
+    name: string;
+    card: PersonCard;
+    archived_at: string | null;
+    archived_by: string | null;
+    archive_reason: string | null;
+    updated: string;
+    synthesized_at: string | null;
+}
+export interface MutationResult {
+    card: PersonCard;
+    unchanged: boolean;
+    revision_id: number | null;
+    archived: boolean;
+    updated: string;
+    synthesized_at: string | null;
+}
+export type PersonReadResult = {
+    found: false;
+} | {
+    found: true;
+    archived: true;
+} | {
+    found: true;
+    rendered: string;
+    card: PersonCard;
+    archived: boolean;
+    updated: string;
+    synthesized_at: string | null;
+};
+export interface PersonListEntry {
+    slug: string;
+    name: string;
+    archived: boolean;
+    updated: string;
+    synthesized_at: string | null;
+    stale: number;
+    unverified: number;
+    memories_since_synthesis: number;
+}
+export interface PersonRevisionMeta {
+    id: number;
+    op: RevisionOp;
+    replaced_at: string;
+}
+export type RevisionOp = 'write' | 'verify' | 'archive' | 'restore' | 'revision_restore';
+/** Every rejection the people surface makes. The message names the path. */
+export declare class PersonCardError extends Error {
+    constructor(message: string);
+}
+/** Null when `slug` is a valid person slug, else the reason. Never echoes an over-long slug. */
+export declare function personSlugError(slug: unknown): string | null;
+/**
+ * Validate a complete card: keys, types, lengths and the worst-case render
+ * cap. `nowMs` adds the time checks (no future verified_at); pass null to skip
+ * them, which revision restore does — the snapshot was valid when written.
+ */
+export declare function validateCard(card: unknown, nowMs: number | null): asserts card is PersonCard;
+export type FieldState = {
+    kind: 'ok';
+} | {
+    kind: 'unverified';
+} | {
+    kind: 'stale';
+    days: number;
+};
+/** Stale when now − verified_at > stale_after_days × 86,400 s (strictly greater). */
+export declare function fieldState(env: WhereEnvelope, nowMs: number): FieldState;
+/**
+ * The rendered card — the text identity and `person` return. With
+ * `worstCase`, every where marker is counted at its widest ([STALE 999+d]):
+ * that is the length the write-time cap checks, so time passing can never
+ * push a stored card over it.
+ */
+export declare function renderCard(card: PersonCard, nowMs: number, opts?: {
+    worstCase?: boolean;
+}): string;
+export interface PeopleStoreConfig {
+    dbPath: string;
+    /** Open read-only and never migrate. Reads on an unmigrated store see no cards. */
+    readonly?: boolean;
+    busyTimeoutMs?: number;
+    /** Clock, for tests. */
+    now?: () => number;
+}
+/** Thrown when a stored row cannot be turned back into a card. */
+export declare class UnreadableCardError extends Error {
+    constructor(slug: string, cause: unknown);
+}
+export declare class PeopleStore {
+    private db;
+    private readonly now;
+    /** False on a read-only store whose tables do not exist yet. */
+    readonly migrated: boolean;
+    constructor(config: PeopleStoreConfig);
+    close(): void;
+    private nowIso;
+    private readRow;
+    private snapshot;
+    private store;
+    /** One BEGIN IMMEDIATE transaction around `fn`, retried as a whole on SQLITE_BUSY. */
+    private mutate;
+    private result;
+    private requireActive;
+    write(input: PersonWriteInput): MutationResult;
+    verify(input: {
+        slug: string;
+        fields: string[];
+        verified_at?: string;
+        source?: string;
+    }): MutationResult;
+    archive(input: {
+        slug: string;
+        reason: string;
+        actor: string;
+    }): MutationResult;
+    restore(input: {
+        slug: string;
+    }): MutationResult;
+    revisionRestore(input: {
+        slug: string;
+        revision_id: number;
+    }): MutationResult;
+    /** The stored row, or null. Throws UnreadableCardError for a row that does not parse or validate. */
+    getRow(slug: string): PersonRow | null;
+    read(slug: string, includeArchived?: boolean): PersonReadResult;
+    list(includeArchived?: boolean): PersonListEntry[];
+    history(slug: string): PersonRevisionMeta[];
+    /** A revision's snapshot (the whole previous row), for tests and inspection. */
+    revisionSnapshot(id: number): {
+        slug: string;
+        card: PersonCard;
+        synthesized_at: string | null;
+        archived_at: string | null;
+    } | null;
+}
+/**
+ * Open the people store on a context's memory database. A read-only open
+ * returns null when there is no database file yet; a writable one creates it.
+ */
+export declare function openPeopleStore(contextDir: string, opts?: {
+    readonly?: boolean;
+    now?: () => number;
+}): PeopleStore | null;
+export interface RoomOptions {
+    nowMs?: number;
+    /** Called once per failure (store open or one unreadable row). Never throws through. */
+    onFailure?: (err: unknown) => void;
+}
+/**
+ * The `# In the room` block for identity, or null for no block. Best-effort
+ * like the tape: a store that cannot be opened or is not migrated gives no
+ * block; one bad row gives that slug the unreadable line and the rest render.
+ */
+export declare function roomBlock(contextDir: string, people: readonly string[] | undefined, opts?: RoomOptions): string | null;