@psiclawops/hypermem 0.9.7 → 0.9.9

package/dist/entity-bridge-backfill.d.ts

@@ -0,0 +1,66 @@
+/**
+ * entity-bridge-backfill.ts \u2014 Sprint B
+ *
+ * Operator-run backfill for the entity/facet bridge index.
+ *
+ * Constraints (Sprint B planner):
+ *   - Never runs at startup.
+ *   - Only ever invoked through scripts/backfill-entity-bridge.mjs.
+ *   - Counts only \u2014 never logs message content.
+ *
+ * Usage (from script):
+ *   const summary = await runEntityBridgeBackfill(db, {
+ *     agentId: 'forge',
+ *     batchSize: 200,
+ *     limit: 5000,
+ *     dryRun: false,
+ *     resume: true,
+ *     sinceMessageId: 12345,
+ *     reconcile: false,
+ *   });
+ */
+import type { DatabaseSync } from 'node:sqlite';
+export interface BackfillOptions {
+    agentId?: string;
+    /** Rows fetched per scan tick. Default 200. */
+    batchSize?: number;
+    /** Hard cap on total messages scanned. Default Infinity. */
+    limit?: number;
+    /** When true, do not write any rows. Default false. */
+    dryRun?: boolean;
+    /** When true, skip messages already in entity_bridge_message_index. Default true. */
+    resume?: boolean;
+    /** Lower bound on message id (exclusive of \u2018already done\u2019 unless reconcile). */
+    sinceMessageId?: number;
+    /**
+     * When true, reprocess messages that already have an index row. Defaults to
+     * false. Reconcile mode still respects `limit`.
+     */
+    reconcile?: boolean;
+    /** Optional progress sink. Receives counts only. */
+    onProgress?: (s: BackfillProgress) => void;
+}
+export interface BackfillProgress {
+    scanned: number;
+    written: number;
+    skipped: number;
+    failed: number;
+    zeroMention: number;
+    highestMessageId: number | null;
+}
+export interface BackfillSummary extends BackfillProgress {
+    dryRun: boolean;
+    reconcile: boolean;
+    agentId?: string;
+    durationMs: number;
+    bridgeTablesPresent: boolean;
+}
+/**
+ * Run the backfill. Returns a counts-only summary suitable for logging.
+ *
+ * On any per-message extraction error we record a 'failed' index row (when
+ * not in dry-run mode) and continue. The whole run never throws unless the
+ * DB itself is broken.
+ */
+export declare function runEntityBridgeBackfill(db: DatabaseSync, opts?: BackfillOptions): Promise<BackfillSummary>;
+//# sourceMappingURL=entity-bridge-backfill.d.ts.map

package/dist/entity-bridge-backfill.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"entity-bridge-backfill.d.ts","sourceRoot":"","sources":["../src/entity-bridge-backfill.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAIhD,MAAM,WAAW,eAAe;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,+CAA+C;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,4DAA4D;IAC5D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,uDAAuD;IACvD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,qFAAqF;IACrF,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,0FAA0F;IAC1F,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,oDAAoD;IACpD,UAAU,CAAC,EAAE,CAAC,CAAC,EAAE,gBAAgB,KAAK,IAAI,CAAC;CAC5C;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;CACjC;AAED,MAAM,WAAW,eAAgB,SAAQ,gBAAgB;IACvD,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,EAAE,OAAO,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,mBAAmB,EAAE,OAAO,CAAC;CAC9B;AAED;;;;;;GAMG;AACH,wBAAsB,uBAAuB,CAC3C,EAAE,EAAE,YAAY,EAChB,IAAI,GAAE,eAAoB,GACzB,OAAO,CAAC,eAAe,CAAC,CAwH1B"}

package/dist/entity-bridge-backfill.js

@@ -0,0 +1,145 @@
+/**
+ * entity-bridge-backfill.ts \u2014 Sprint B
+ *
+ * Operator-run backfill for the entity/facet bridge index.
+ *
+ * Constraints (Sprint B planner):
+ *   - Never runs at startup.
+ *   - Only ever invoked through scripts/backfill-entity-bridge.mjs.
+ *   - Counts only \u2014 never logs message content.
+ *
+ * Usage (from script):
+ *   const summary = await runEntityBridgeBackfill(db, {
+ *     agentId: 'forge',
+ *     batchSize: 200,
+ *     limit: 5000,
+ *     dryRun: false,
+ *     resume: true,
+ *     sinceMessageId: 12345,
+ *     reconcile: false,
+ *   });
+ */
+import { EntityBridgeStore } from './entity-bridge-store.js';
+import { extractEntityFacetMentions } from './entity-extractor.js';
+/**
+ * Run the backfill. Returns a counts-only summary suitable for logging.
+ *
+ * On any per-message extraction error we record a 'failed' index row (when
+ * not in dry-run mode) and continue. The whole run never throws unless the
+ * DB itself is broken.
+ */
+export async function runEntityBridgeBackfill(db, opts = {}) {
+    const start = Date.now();
+    const batchSize = Math.max(1, Math.min(5000, opts.batchSize ?? 200));
+    const limit = opts.limit && opts.limit > 0 ? opts.limit : Infinity;
+    const dryRun = Boolean(opts.dryRun);
+    const reconcile = Boolean(opts.reconcile);
+    const resume = opts.resume ?? !reconcile;
+    const sinceMessageId = opts.sinceMessageId ?? 0;
+    const agentId = opts.agentId;
+    const store = new EntityBridgeStore(db);
+    const tablesPresent = store.tablesExist();
+    const progress = {
+        scanned: 0,
+        written: 0,
+        skipped: 0,
+        failed: 0,
+        zeroMention: 0,
+        highestMessageId: null,
+    };
+    if (!tablesPresent) {
+        return {
+            ...progress,
+            dryRun,
+            reconcile,
+            agentId,
+            durationMs: Date.now() - start,
+            bridgeTablesPresent: false,
+        };
+    }
+    let cursor = sinceMessageId;
+    let remaining = limit;
+    // SQL: select messages above cursor in batches, optionally filtered by agent.
+    // We deliberately use COALESCE(text_content,'') length>0 so we don't index
+    // pure tool/heartbeat rows.
+    const baseSelect = `
+    SELECT m.id, m.conversation_id, m.agent_id, m.text_content
+    FROM messages m
+    ${reconcile ? '' : 'LEFT JOIN entity_bridge_message_index idx ON idx.message_id = m.id'}
+    WHERE m.id > ?
+      AND COALESCE(m.text_content, '') != ''
+      AND m.is_heartbeat = 0
+      ${agentId ? 'AND m.agent_id = ?' : ''}
+      ${resume && !reconcile ? 'AND idx.message_id IS NULL' : ''}
+    ORDER BY m.id ASC
+    LIMIT ?
+  `;
+    const stmt = db.prepare(baseSelect);
+    while (remaining > 0) {
+        const fetch = Math.min(batchSize, remaining);
+        const params = [cursor];
+        if (agentId)
+            params.push(agentId);
+        params.push(fetch);
+        const rows = stmt.all(...params);
+        if (rows.length === 0)
+            break;
+        for (const row of rows) {
+            progress.scanned++;
+            cursor = row.id;
+            progress.highestMessageId = row.id;
+            try {
+                const mentions = extractEntityFacetMentions(row.text_content);
+                if (dryRun) {
+                    if (mentions.entities.length === 0 && mentions.facets.length === 0) {
+                        progress.zeroMention++;
+                    }
+                    else {
+                        progress.written++;
+                    }
+                }
+                else {
+                    const r = store.recordMentions({
+                        messageId: row.id,
+                        agentId: row.agent_id,
+                        threadRef: row.conversation_id,
+                        mentions,
+                        source: 'backfill',
+                    });
+                    if (r.entityCount === 0 && r.facetCount === 0) {
+                        progress.zeroMention++;
+                    }
+                    else {
+                        progress.written++;
+                    }
+                }
+            }
+            catch (err) {
+                progress.failed++;
+                if (!dryRun) {
+                    store.recordIndexFailure({
+                        messageId: row.id,
+                        agentId: row.agent_id,
+                        threadRef: row.conversation_id,
+                        error: err,
+                        source: 'backfill',
+                    });
+                }
+            }
+        }
+        remaining -= rows.length;
+        if (opts.onProgress)
+            opts.onProgress({ ...progress });
+        if (rows.length < fetch)
+            break;
+    }
+    return {
+        ...progress,
+        dryRun,
+        reconcile,
+        agentId,
+        durationMs: Date.now() - start,
+        bridgeTablesPresent: true,
+    };
+}
+//# sourceMappingURL=entity-bridge-backfill.js.map

package/dist/entity-bridge-store.d.ts

@@ -0,0 +1,164 @@
+/**
+ * entity-bridge-store.ts \u2014 Sprint B
+ *
+ * Metadata-only CRUD/query helpers for the entity/facet bridge tables.
+ *
+ * Hard rule: this module never logs message content. Mention rows store
+ * a `match_term` (the surface form of the matched span) and offsets, but
+ * full message text remains in the `messages` table only.
+ *
+ * Tables (created by schema v12 migration):
+ *   - memory_entities                  (agent_id, entity_key, display_name, ...)
+ *   - memory_facets                    (agent_id, facet_key, ...)
+ *   - message_entity_mentions          (message_id, entity_id, ...)
+ *   - message_facet_mentions           (message_id, facet_id, ...)
+ *   - entity_bridge_message_index      (message_id, entity_count, facet_count, status, ...)
+ *
+ * The store is created on-demand. If the v12 tables are absent (older DB),
+ * `tablesExist()` returns false and all writes/reads are no-ops/empty.
+ */
+import type { DatabaseSync } from 'node:sqlite';
+import type { EntityFacetMentions } from './entity-extractor.js';
+export interface BridgeIndexState {
+    exists: boolean;
+    status?: string;
+    source?: string;
+    entityCount?: number;
+    facetCount?: number;
+    indexedAt?: string;
+    lastError?: string | null;
+}
+export interface BridgeWatermarkDiagnostics {
+    totalMessages: number;
+    indexedMessages: number;
+    failedMessages: number;
+    zeroMentionMessages: number;
+    highestIndexedMessageId: number | null;
+}
+export interface BridgeCandidateMessage {
+    messageId: number;
+    threadRef: number | null;
+    matchedEntities: string[];
+    matchedFacets: string[];
+}
+export interface BridgeGraphNeighbor {
+    /** Entity or facet key. */
+    key: string;
+    /** Edge weight (mention count or co-occurrence count). */
+    weight: number;
+}
+export interface BridgeGraphSnapshot {
+    /** Map of entityKey -> messageIds[] (capped). */
+    entityMessages: Map<string, number[]>;
+    /** Map of facetKey -> messageIds[] (capped). */
+    facetMessages: Map<string, number[]>;
+    /** Map of messageId -> entity keys. */
+    messageEntities: Map<number, string[]>;
+    /** Map of messageId -> facet keys. */
+    messageFacets: Map<number, string[]>;
+    /** Diagnostics on cap behavior. */
+    diagnostics: {
+        nodeCount: number;
+        edgeCount: number;
+        seedExpanded: number;
+        nodesCapped: boolean;
+        edgesCapped: boolean;
+    };
+}
+export declare class EntityBridgeStore {
+    private readonly db;
+    private _tablesChecked;
+    private _tablesExist;
+    private _stmtUpsertEntity?;
+    private _stmtTouchEntity?;
+    private _stmtUpsertFacet?;
+    private _stmtTouchFacet?;
+    private _stmtInsertEntityMention?;
+    private _stmtInsertFacetMention?;
+    private _stmtUpsertIndex?;
+    private _stmtGetIndex?;
+    constructor(db: DatabaseSync);
+    /**
+     * Check whether all v12 bridge tables exist in this DB.
+     * Cached after the first call. Cheap when cached.
+     */
+    tablesExist(): boolean;
+    getIndexState(messageId: number): BridgeIndexState;
+    /**
+     * Watermark diagnostics: counts of indexed/failed/zero-mention messages
+     * and the highest indexed message id, scoped to a single agent.
+     */
+    getWatermarkDiagnostics(agentId: string): BridgeWatermarkDiagnostics;
+    /**
+     * Record entity/facet mentions for a single message. Always writes a
+     * row into `entity_bridge_message_index` even when there are zero mentions
+     * so that callers can distinguish "never indexed" from "indexed, no mentions".
+     *
+     * Wraps all writes in a single transaction. Returns whether any write
+     * occurred. On failure, records a 'failed' index row when possible and
+     * rethrows the underlying error so the caller can decide whether to surface.
+     */
+    recordMentions(input: {
+        messageId: number;
+        agentId: string;
+        threadRef?: number | null;
+        mentions: EntityFacetMentions;
+        source?: 'live' | 'backfill';
+    }): {
+        wrote: boolean;
+        entityCount: number;
+        facetCount: number;
+    };
+    /**
+     * Emit a metadata-only failure marker without attempting any mention writes.
+     * Used by callers (e.g. message-store live indexing) when extraction itself
+     * threw before reaching the store, or to record an index attempt that never
+     * produced mentions due to disabled tables.
+     */
+    recordIndexFailure(input: {
+        messageId: number;
+        agentId: string;
+        threadRef?: number | null;
+        error: unknown;
+        source?: 'live' | 'backfill';
+    }): boolean;
+    private recordIndexFailureEvent;
+    /**
+     * Look up the internal entity rows for a list of normalized entity keys.
+     * Missing keys are silently dropped.
+     */
+    lookupEntityIds(agentId: string, keys: string[]): Map<string, number>;
+    lookupFacetIds(agentId: string, keys: string[]): Map<string, number>;
+    /**
+     * Build a metadata-only graph snapshot for PPR. Bounded by node/edge caps.
+     *
+     * Algorithm:
+     *  - Resolve seed entity/facet keys to internal ids.
+     *  - For each seed, fetch up to `perSeedMessageLimit` mention rows.
+     *  - Aggregate into message \u2194 entity \u2194 facet adjacency lists, capped.
+     */
+    buildGraphSnapshot(opts: {
+        agentId: string;
+        seedEntityKeys: string[];
+        seedFacetKeys: string[];
+        maxNodes: number;
+        maxEdges: number;
+        perSeedMessageLimit: number;
+    }): BridgeGraphSnapshot;
+    /**
+     * Resolve a list of message ids into candidate rows joined back to
+     * `messages` for compose-lane consumption. Returns the matched entity/facet
+     * keys per message as a side-channel diagnostic.
+     *
+     * Caller is expected to gate further rendering on whether a message's
+     * full text should be hydrated.
+     */
+    fetchCandidates(opts: {
+        agentId: string;
+        messageIds: number[];
+    }): BridgeCandidateMessage[];
+    private ensureUpsertStmts;
+    private upsertEntityRow;
+    private upsertFacetRow;
+}
+//# sourceMappingURL=entity-bridge-store.d.ts.map

package/dist/entity-bridge-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"entity-bridge-store.d.ts","sourceRoot":"","sources":["../src/entity-bridge-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAiB,MAAM,aAAa,CAAC;AAC/D,OAAO,KAAK,EAA+B,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE9F,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAED,MAAM,WAAW,0BAA0B;IACzC,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,uBAAuB,EAAE,MAAM,GAAG,IAAI,CAAC;CACxC;AAED,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,MAAM,EAAE,CAAC;CACzB;AAED,MAAM,WAAW,mBAAmB;IAClC,2BAA2B;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,0DAA0D;IAC1D,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,mBAAmB;IAClC,iDAAiD;IACjD,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACtC,gDAAgD;IAChD,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACrC,uCAAuC;IACvC,eAAe,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACvC,sCAAsC;IACtC,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACrC,mCAAmC;IACnC,WAAW,EAAE;QACX,SAAS,EAAE,MAAM,CAAC;QAClB,SAAS,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,MAAM,CAAC;QACrB,WAAW,EAAE,OAAO,CAAC;QACrB,WAAW,EAAE,OAAO,CAAC;KACtB,CAAC;CACH;AAcD,qBAAa,iBAAiB;IAchB,OAAO,CAAC,QAAQ,CAAC,EAAE;IAb/B,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,YAAY,CAAS;IAG7B,OAAO,CAAC,iBAAiB,CAAC,CAAgB;IAC1C,OAAO,CAAC,gBAAgB,CAAC,CAAgB;IACzC,OAAO,CAAC,gBAAgB,CAAC,CAAgB;IACzC,OAAO,CAAC,eAAe,CAAC,CAAgB;IACxC,OAAO,CAAC,wBAAwB,CAAC,CAAgB;IACjD,OAAO,CAAC,uBAAuB,CAAC,CAAgB;IAChD,OAAO,CAAC,gBAAgB,CAAC,CAAgB;IACzC,OAAO,CAAC,aAAa,CAAC,CAAgB;gBAET,EAAE,EAAE,YAAY;IAE7C;;;OAGG;IACH,WAAW,IAAI,OAAO;IAiBtB,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,gBAAgB;IAsBlD;;;OAGG;IACH,uBAAuB,CAAC,OAAO,EAAE,MAAM,GAAG,0BAA0B;IAwCpE;;;;;;;;OAQG;IACH,cAAc,CAAC,KAAK,EAAE;QACpB,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC1B,QAAQ,EAAE,mBAAmB,CAAC;QAC9B,MAAM,CAAC,EAAE,MAAM,GAAG,UAAU,CAAC;KAC9B,GAAG;QAAE,KAAK,EAAE,OAAO,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE;IAiF/D;;;;;OAKG;IACH,kBAAkB,CAAC,KAAK,EAAE;QACxB,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QAC1B,KAAK,EAAE,OAAO,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,GAAG,UAAU,CAAC;KAC9B,GAAG,OAAO;IAwBX,OAAO,CAAC,uBAAuB;IA0B/B;;;OAGG;IACH,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;IAWrE,cAAc,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;IAWpE;;;;;;;OAOG;IACH,kBAAkB,CAAC,IAAI,EAAE;QACvB,OAAO,EAAE,MAAM,CAAC;QAChB,cAAc,EAAE,MAAM,EAAE,CAAC;QACzB,aAAa,EAAE,MAAM,EAAE,CAAC;QACxB,QAAQ,EAAE,MAAM,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,mBAAmB,EAAE,MAAM,CAAC;KAC7B,GAAG,mBAAmB;IA0HvB;;;;;;;OAOG;IACH,eAAe,CAAC,IAAI,EAAE;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,UAAU,EAAE,MAAM,EAAE,CAAC;KACtB,GAAG,sBAAsB,EAAE;IAgD5B,OAAO,CAAC,iBAAiB;IA+CzB,OAAO,CAAC,eAAe;IAMvB,OAAO,CAAC,cAAc;CAKvB"}