@psiclawops/hypermem 0.5.0 → 0.5.1

package/dist/background-indexer.d.ts

@@ -0,0 +1,132 @@
+/**
+ * hypermem Background Indexer
+ *
+ * Processes message history to extract structured knowledge:
+ *   - Facts: atomic pieces of learned information
+ *   - Episodes: significant events worth remembering
+ *   - Topics: conversation threads and their lifecycle
+ *   - Knowledge: durable structured entries (domain + key)
+ *
+ * Runs as a periodic background task, processing unindexed messages
+ * in batches. Each batch is scored, classified, and stored in L4 (library.db).
+ *
+ * Design principles:
+ *   - No LLM dependency: extraction uses pattern matching + heuristics
+ *   - Idempotent: tracks watermarks per agent to avoid reprocessing
+ *   - Bounded: processes N messages per tick to avoid blocking
+ *   - Observable: logs extraction stats for monitoring
+ */
+import type { DatabaseSync } from 'node:sqlite';
+import type { IndexerConfig, SessionCursor } from './types.js';
+import { type DreamerConfig } from './dreaming-promoter.js';
+import type { VectorStore } from './vector-store.js';
+export interface IndexerStats {
+    agentId: string;
+    messagesProcessed: number;
+    factsExtracted: number;
+    episodesRecorded: number;
+    topicsUpdated: number;
+    knowledgeUpserted: number;
+    /** Number of superseded fact vectors tombstoned from the vector index this tick. */
+    tombstoned: number;
+    elapsedMs: number;
+    /** Number of messages that were post-cursor (unseen by model, high-signal priority). */
+    postCursorMessages: number;
+}
+/**
+ * Optional callback to fetch the session cursor for an agent+session.
+ * When provided, the indexer uses the cursor to prioritize unseen messages.
+ * The cursor boundary separates "model has seen this" from "new since last compose".
+ */
+export type CursorFetcher = (agentId: string, sessionKey: string) => Promise<SessionCursor | null>;
+export interface WatermarkState {
+    agentId: string;
+    lastMessageId: number;
+    lastRunAt: string;
+}
+export declare class BackgroundIndexer {
+    private getMessageDb?;
+    private getLibraryDb?;
+    private listAgents?;
+    private getCursor?;
+    private readonly config;
+    private readonly dreamerConfig;
+    private intervalHandle;
+    private running;
+    private vectorStore;
+    private synthesizer;
+    private tickCount;
+    constructor(config?: Partial<IndexerConfig>, getMessageDb?: ((agentId: string) => DatabaseSync) | undefined, getLibraryDb?: (() => DatabaseSync) | undefined, listAgents?: (() => string[]) | undefined, getCursor?: CursorFetcher | undefined, dreamerConfig?: Partial<DreamerConfig>);
+    /**
+     * Set the vector store for embedding new facts/episodes at index time.
+     * Optional — if not set, indexer runs without embedding (FTS5-only mode).
+     */
+    setVectorStore(vs: VectorStore): void;
+    /**
+     * Start periodic indexing.
+     */
+    start(): void;
+    /**
+     * Stop periodic indexing.
+     */
+    stop(): void;
+    /**
+     * Run one indexing pass across all agents.
+     */
+    tick(): Promise<IndexerStats[]>;
+    /**
+     * Process a single agent's unindexed messages.
+     *
+     * When a cursor fetcher is available, messages are split into two tiers:
+     *   - Post-cursor (id > cursor.lastSentId): "unseen" by the model, high-signal priority
+     *   - Pre-cursor (id <= cursor.lastSentId): already in the model's context window, lower priority
+     * Post-cursor messages are processed first. This ensures the indexer prioritizes
+     * content the model hasn't seen yet — decisions, incidents, and discoveries that
+     * happened between context windows.
+     */
+    private processAgent;
+    /**
+     * Fetch unindexed messages for an agent.
+     */
+    private getUnindexedMessages;
+    /**
+     * Get the session key for a conversation ID.
+     */
+    private getSessionKeyForMessage;
+    /**
+     * Get the indexing watermark for an agent.
+     */
+    private getWatermark;
+    /**
+     * Set the indexing watermark for an agent.
+     */
+    private setWatermark;
+    /**
+     * Apply time-based decay to facts.
+     * Increases decay_score for older facts, making them less relevant.
+     */
+    private applyDecay;
+    /**
+     * Parse a duration string like "24h", "7d" into seconds.
+     */
+    private parseDuration;
+    /**
+     * One-time backfill: embed episodes with sig>=0.5 that were missed by the
+     * old >=0.7 vectorization threshold.
+     *
+     * Gated by a system_state flag 'indexer:episode_backfill_v1' so it runs
+     * exactly once even across gateway restarts. Safe to re-run manually
+     * (delete the flag row first) if re-backfill is ever needed.
+     */
+    backfillEpisodeVectors(): Promise<void>;
+    /**
+     * Get current watermarks for all agents.
+     */
+    getWatermarks(libraryDb: DatabaseSync): WatermarkState[];
+}
+/**
+ * Create and start a background indexer connected to hypermem databases.
+ * Used by the hook or a standalone daemon.
+ */
+export declare function createIndexer(getMessageDb: (agentId: string) => DatabaseSync, getLibraryDb: () => DatabaseSync, listAgents: () => string[], config?: Partial<IndexerConfig>, getCursor?: CursorFetcher, vectorStore?: VectorStore, dreamerConfig?: Partial<DreamerConfig>): BackgroundIndexer;
+//# sourceMappingURL=background-indexer.d.ts.map

package/dist/background-indexer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"background-indexer.d.ts","sourceRoot":"","sources":["../src/background-indexer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,KAAK,EAAiB,aAAa,EAAe,aAAa,EAAE,MAAM,YAAY,CAAC;AAK3F,OAAO,EAA2B,KAAK,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAOrF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAuCrD,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,MAAM,CAAC;IACzB,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,oFAAoF;IACpF,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,wFAAwF;IACxF,kBAAkB,EAAE,MAAM,CAAC;CAC5B;AAED;;;;GAIG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,KAAK,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC;AAEnG,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB;AA+XD,qBAAa,iBAAiB;IAW1B,OAAO,CAAC,YAAY,CAAC;IACrB,OAAO,CAAC,YAAY,CAAC;IACrB,OAAO,CAAC,UAAU,CAAC;IACnB,OAAO,CAAC,SAAS,CAAC;IAbpB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IACvC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAyB;IACvD,OAAO,CAAC,cAAc,CAA+C;IACrE,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,WAAW,CAA4B;IAC/C,OAAO,CAAC,WAAW,CAAiC;IACpD,OAAO,CAAC,SAAS,CAAa;gBAG5B,MAAM,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,EACvB,YAAY,CAAC,GAAE,CAAC,OAAO,EAAE,MAAM,KAAK,YAAY,aAAA,EAChD,YAAY,CAAC,GAAE,MAAM,YAAY,aAAA,EACjC,UAAU,CAAC,GAAE,MAAM,MAAM,EAAE,aAAA,EAC3B,SAAS,CAAC,EAAE,aAAa,YAAA,EACjC,aAAa,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC;IA8BxC;;;OAGG;IACH,cAAc,CAAC,EAAE,EAAE,WAAW,GAAG,IAAI;IAIrC;;OAEG;IACH,KAAK,IAAI,IAAI;IA0Bb;;OAEG;IACH,IAAI,IAAI,IAAI;IAOZ;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;IAsIrC;;;;;;;;;OASG;YACW,YAAY;IA4M1B;;OAEG;IACH,OAAO,CAAC,oBAAoB;IA+B5B;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAK/B;;OAEG;IACH,OAAO,CAAC,YAAY;IAsBpB;;OAEG;IACH,OAAO,CAAC,YAAY;IAWpB;;;OAGG;IACH,OAAO,CAAC,UAAU;IA8ClB;;OAEG;IACH,OAAO,CAAC,aAAa;IAarB;;;;;;;OAOG;IACG,sBAAsB,IAAI,OAAO,CAAC,IAAI,CAAC;IAgF7C;;OAEG;IACH,aAAa,CAAC,SAAS,EAAE,YAAY,GAAG,cAAc,EAAE;CAezD;AAID;;;GAGG;AACH,wBAAgB,aAAa,CAC3B,YAAY,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,YAAY,EAC/C,YAAY,EAAE,MAAM,YAAY,EAChC,UAAU,EAAE,MAAM,MAAM,EAAE,EAC1B,MAAM,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,EAC/B,SAAS,CAAC,EAAE,aAAa,EACzB,WAAW,CAAC,EAAE,WAAW,EACzB,aAAa,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,GACrC,iBAAiB,CAInB"}