hippo-memory 1.60.0 → 1.61.0

package/dist/transcript-tail.js

@@ -0,0 +1,48 @@
+// Transcript reads shared by capture.ts and compaction-record.ts; a leaf so neither has to import the other.
+import * as fs from 'fs';
+/** Never read the whole transcript: PreCompact fires exactly when it's largest. */
+export const PRE_COMPACT_TAIL_BYTES = 256 * 1024;
+/** Cut to `maxChars` UTF-16 units, backing off one unit rather than leave half a surrogate pair in a stored field. */
+export function truncateCodePointSafe(text, maxChars) {
+    if (text.length <= maxChars)
+        return text;
+    let end = maxChars;
+    if (end > 0) {
+        const code = text.charCodeAt(end - 1);
+        if (code >= 0xd800 && code <= 0xdbff)
+            end -= 1;
+    }
+    return text.slice(0, end);
+}
+/** The last `capBytes` of a JSONL transcript, read at an offset and starting on a whole line so every line parses. */
+export function readTranscriptTail(transcriptPath, capBytes = PRE_COMPACT_TAIL_BYTES) {
+    const size = fs.statSync(transcriptPath).size;
+    const start = Math.max(0, size - capBytes);
+    const length = size - start;
+    if (length <= 0)
+        return '';
+    const fd = fs.openSync(transcriptPath, 'r');
+    try {
+        // A seek that lands just after '\n' already starts a whole line and must keep it.
+        let onLineBoundary = start === 0;
+        if (!onLineBoundary) {
+            const prevByte = Buffer.alloc(1);
+            fs.readSync(fd, prevByte, 0, 1, start - 1);
+            onLineBoundary = prevByte[0] === 0x0a; // '\n'
+        }
+        const buf = Buffer.alloc(length);
+        fs.readSync(fd, buf, 0, length, start);
+        let text = buf.toString('utf8');
+        if (!onLineBoundary) {
+            // Landed mid-line — drop the partial first line so every remaining
+            // line parses as complete JSON.
+            const nl = text.indexOf('\n');
+            text = nl === -1 ? '' : text.slice(nl + 1);
+        }
+        return text;
+    }
+    finally {
+        fs.closeSync(fd);
+    }
+}
+//# sourceMappingURL=transcript-tail.js.map

package/dist/vector-store.d.ts

@@ -0,0 +1,27 @@
+/** Embedding vectors in the `memory_vectors` table: one Float32 BLOB per memory id, plus the one-time import of the old JSON index. */
+import type { DatabaseSyncLike } from './db.js';
+export declare const EMBEDDING_MODEL_META_KEY = "embedding_model";
+export declare const MEMORY_VECTORS_DDL = "\n  CREATE TABLE IF NOT EXISTS memory_vectors (\n    memory_id TEXT PRIMARY KEY,\n    model     TEXT NOT NULL,\n    dim       INTEGER NOT NULL,\n    vector    BLOB NOT NULL\n  );\n  CREATE TRIGGER IF NOT EXISTS trg_memory_vectors_delete AFTER DELETE ON memories BEGIN\n    DELETE FROM memory_vectors WHERE memory_id = old.id;\n  END;\n";
+export declare function encodeVector(vector: readonly number[]): Uint8Array;
+export declare function decodeVector(blob: Uint8Array): Float32Array;
+/** Insert or replace vectors, skipping empty or non-finite ones; the caller owns any surrounding transaction. */
+export declare function upsertVectors(db: DatabaseSyncLike, rows: Iterable<readonly [string, readonly number[]]>, model: string): number;
+/** Replace the whole table with `index` in one transaction, as a full rebuild does. */
+export declare function replaceAllVectors(db: DatabaseSyncLike, index: Readonly<Record<string, readonly number[]>>, model: string): void;
+/** Drop vectors whose memory row is gone; returns how many went. */
+export declare function deleteOrphanVectors(db: DatabaseSyncLike): number;
+/** Vectors for `ids`, or every stored vector when `ids` is omitted. */
+export declare function loadVectors(db: DatabaseSyncLike, ids?: readonly string[]): Map<string, number[]>;
+export declare function storedVectorIds(db: DatabaseSyncLike): Set<string>;
+export declare function hasStoredVectors(db: DatabaseSyncLike): boolean;
+export interface VectorMatch {
+    id: string;
+    score: number;
+}
+/** The `k` stored vectors closest to `query` among `memories m` rows passing `where` (SQL starting with ` AND`).
+ *  Filters run before the cut, so rows a caller may not see can never push admitted rows out of the top k. */
+export declare function topVectorMatches(db: DatabaseSyncLike, query: readonly number[], k: number, where: string, params: readonly (string | number)[]): VectorMatch[];
+/** Move a legacy `embeddings.json` into `memory_vectors`, then keep the file as `embeddings.json.imported-<stamp>`.
+ *  Runs after the migration commits, so the rename never outlives rolled-back rows; a file an older binary writes later imports the same way. */
+export declare function importLegacyEmbeddingIndex(db: DatabaseSyncLike, hippoRoot: string): void;
+//# sourceMappingURL=vector-store.d.ts.map

package/dist/vector-store.js

@@ -0,0 +1,210 @@
+/** Embedding vectors in the `memory_vectors` table: one Float32 BLOB per memory id, plus the one-time import of the old JSON index. */
+import * as fs from 'fs';
+import * as path from 'path';
+import { log } from './log.js';
+/** Legacy whole-file index; imported once into `memory_vectors`, then kept beside the store as a renamed backup. */
+const LEGACY_EMBEDDINGS_FILE = 'embeddings.json';
+export const EMBEDDING_MODEL_META_KEY = 'embedding_model';
+// Never equal to a real index identity, so the next embed run treats it as a model change and rebuilds every vector.
+const QUARANTINED_INDEX_IDENTITY = 'quarantined-corrupt-index';
+// No FK to memories: a vector may outlive its row until `hippo embed` prunes it, and the trigger covers deletes.
+export const MEMORY_VECTORS_DDL = `
+  CREATE TABLE IF NOT EXISTS memory_vectors (
+    memory_id TEXT PRIMARY KEY,
+    model     TEXT NOT NULL,
+    dim       INTEGER NOT NULL,
+    vector    BLOB NOT NULL
+  );
+  CREATE TRIGGER IF NOT EXISTS trg_memory_vectors_delete AFTER DELETE ON memories BEGIN
+    DELETE FROM memory_vectors WHERE memory_id = old.id;
+  END;
+`;
+const ID_CHUNK = 500;
+export function encodeVector(vector) {
+    return new Uint8Array(Float32Array.from(vector).buffer);
+}
+export function decodeVector(blob) {
+    // A view needs 4-byte alignment; copy when SQLite hands back an unaligned slice.
+    const bytes = blob.byteOffset % 4 === 0 ? blob : blob.slice();
+    return new Float32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 4);
+}
+/** Insert or replace vectors, skipping empty or non-finite ones; the caller owns any surrounding transaction. */
+export function upsertVectors(db, rows, model) {
+    const stmt = db.prepare('INSERT OR REPLACE INTO memory_vectors (memory_id, model, dim, vector) VALUES (?, ?, ?, ?)');
+    let written = 0;
+    for (const [id, vector] of rows) {
+        if (vector.length === 0 || !vector.every(Number.isFinite))
+            continue;
+        stmt.run(id, model, vector.length, encodeVector(vector));
+        written++;
+    }
+    return written;
+}
+function inTransaction(db, fn) {
+    db.exec('BEGIN IMMEDIATE');
+    try {
+        const out = fn();
+        db.exec('COMMIT');
+        return out;
+    }
+    catch (err) {
+        try {
+            db.exec('ROLLBACK');
+        }
+        catch { /* keep the original error */ }
+        throw err;
+    }
+}
+/** Replace the whole table with `index` in one transaction, as a full rebuild does. */
+export function replaceAllVectors(db, index, model) {
+    inTransaction(db, () => {
+        db.exec('DELETE FROM memory_vectors');
+        upsertVectors(db, Object.entries(index), model);
+    });
+}
+/** Drop vectors whose memory row is gone; returns how many went. */
+export function deleteOrphanVectors(db) {
+    return Number(db.prepare('DELETE FROM memory_vectors WHERE memory_id NOT IN (SELECT id FROM memories)').run().changes ?? 0);
+}
+/** Vectors for `ids`, or every stored vector when `ids` is omitted. */
+export function loadVectors(db, ids) {
+    const out = new Map();
+    const read = (rows) => {
+        // SAFETY: both SELECTs below name exactly these two columns; node:sqlite returns BLOBs as Uint8Array.
+        for (const row of rows)
+            out.set(row.memory_id, Array.from(decodeVector(row.vector)));
+    };
+    if (ids === undefined) {
+        read(db.prepare('SELECT memory_id, vector FROM memory_vectors').iterate());
+        return out;
+    }
+    const unique = [...new Set(ids)];
+    for (let i = 0; i < unique.length; i += ID_CHUNK) {
+        const chunk = unique.slice(i, i + ID_CHUNK);
+        read(db.prepare(`SELECT memory_id, vector FROM memory_vectors WHERE memory_id IN (${chunk.map(() => '?').join(', ')})`).all(...chunk));
+    }
+    return out;
+}
+export function storedVectorIds(db) {
+    // SAFETY: the SELECT names exactly the one column read.
+    return new Set(db.prepare('SELECT memory_id FROM memory_vectors').all().map((r) => r.memory_id));
+}
+export function hasStoredVectors(db) {
+    return db.prepare('SELECT 1 FROM memory_vectors LIMIT 1').get() !== undefined;
+}
+/** The `k` stored vectors closest to `query` among `memories m` rows passing `where` (SQL starting with ` AND`).
+ *  Filters run before the cut, so rows a caller may not see can never push admitted rows out of the top k. */
+export function topVectorMatches(db, query, k, where, params) {
+    if (k <= 0 || query.length === 0)
+        return [];
+    const q = Float32Array.from(query);
+    let qNorm = 0;
+    for (let i = 0; i < q.length; i++)
+        qNorm += q[i] * q[i];
+    qNorm = Math.sqrt(qNorm);
+    if (qNorm < 1e-10)
+        return [];
+    // SHORTCUT: brute-force cosine over every vector of matching dim, fine to ~100k rows; an ANN index (sqlite-vec, HNSW) is the upgrade.
+    const rows = db.prepare(`
+    SELECT v.memory_id AS id, v.vector AS vector
+    FROM memory_vectors v JOIN memories m ON m.id = v.memory_id
+    WHERE v.dim = ?${where}
+  `).iterate(q.length, ...params);
+    const top = [];
+    // SAFETY: the SELECT above names exactly these two columns; node:sqlite returns BLOBs as Uint8Array.
+    for (const row of rows) {
+        const v = decodeVector(row.vector);
+        let dot = 0;
+        let norm = 0;
+        for (let i = 0; i < v.length; i++) {
+            dot += q[i] * v[i];
+            norm += v[i] * v[i];
+        }
+        if (norm < 1e-20)
+            continue;
+        const score = dot / (qNorm * Math.sqrt(norm));
+        if (top.length === k && !beats(score, row.id, top[k - 1]))
+            continue;
+        insertSorted(top, { id: row.id, score }, k);
+    }
+    return top;
+}
+// Ties break on id so the cut is the same on every run.
+function beats(score, id, other) {
+    return score > other.score || (score === other.score && id < other.id);
+}
+function insertSorted(top, m, k) {
+    let i = top.length;
+    while (i > 0 && beats(m.score, m.id, top[i - 1]))
+        i--;
+    top.splice(i, 0, m);
+    if (top.length > k)
+        top.pop();
+}
+function stamp() {
+    return `${new Date().toISOString().replace(/[:.]/g, '-')}-${process.pid}`;
+}
+/** Rename `fp` to `aside`; a rename blocked by an open handle (Windows) falls back to a copy, so the bytes are kept either way. */
+function moveAside(fp, aside) {
+    try {
+        fs.renameSync(fp, aside);
+        return true;
+    }
+    catch (err) {
+        if (err instanceof Error && 'code' in err && err.code === 'ENOENT')
+            return false;
+        fs.copyFileSync(fp, aside, fs.constants.COPYFILE_EXCL);
+        fs.rmSync(fp, { force: true });
+        return true;
+    }
+}
+// The old writer only wrote a plain object of id to number[]; any other shape is corrupt, and non-numeric entries are dropped.
+function parseLegacyIndex(raw) {
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch {
+        return null; // the caller quarantines the corrupt file and logs it
+    }
+    if (!(parsed instanceof Object) || Array.isArray(parsed))
+        return null;
+    const index = new Map();
+    for (const [id, value] of Object.entries(parsed)) {
+        if (Array.isArray(value) && value.every(Number.isFinite))
+            index.set(id, value);
+    }
+    return index;
+}
+/** Move a legacy `embeddings.json` into `memory_vectors`, then keep the file as `embeddings.json.imported-<stamp>`.
+ *  Runs after the migration commits, so the rename never outlives rolled-back rows; a file an older binary writes later imports the same way. */
+export function importLegacyEmbeddingIndex(db, hippoRoot) {
+    const fp = path.join(hippoRoot, LEGACY_EMBEDDINGS_FILE);
+    let raw;
+    try {
+        raw = fs.readFileSync(fp, 'utf8');
+    }
+    catch (err) {
+        if (err instanceof Error && 'code' in err && err.code === 'ENOENT')
+            return;
+        throw err;
+    }
+    const index = parseLegacyIndex(raw);
+    if (!index) {
+        const aside = `${fp}.corrupt-${stamp()}`;
+        if (!moveAside(fp, aside))
+            return;
+        log.error(`${LEGACY_EMBEDDINGS_FILE} could not be parsed; kept it as ${path.basename(aside)} and the next embed rebuilds the index`, { hippoRoot });
+        db.prepare(`INSERT INTO meta(key, value) VALUES(?, ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`)
+            .run(EMBEDDING_MODEL_META_KEY, QUARANTINED_INDEX_IDENTITY);
+        return;
+    }
+    // SAFETY: the SELECT names exactly the one column read.
+    const model = db.prepare(`SELECT value FROM meta WHERE key = ?`).get(EMBEDDING_MODEL_META_KEY)?.value ?? '';
+    const written = inTransaction(db, () => upsertVectors(db, index, model));
+    const backup = `${fp}.imported-${stamp()}`;
+    if (moveAside(fp, backup)) {
+        log.info(`moved ${written} vectors from ${LEGACY_EMBEDDINGS_FILE} into the store; the file is kept as ${path.basename(backup)}`, { hippoRoot });
+    }
+}
+//# sourceMappingURL=vector-store.js.map

package/dist/version.d.ts

@@ -6,7 +6,7 @@
  * hand. scripts/check-manifest-versions.mjs gates publish on all seven sites.
  *
  * Used by:
- *   - src/db.ts rollback-safety guard (refuses to open a DB stamped with
+ *   - src/db/migrate.ts rollback-safety guard (refuses to open a DB stamped with
  *     min_compatible_binary newer than this).
  *   - src/server.ts HTTP /health.
  *   - src/mcp/server.ts MCP serverInfo.
@@ -16,7 +16,7 @@
  * an ESM `import` can resolve cleanly, and a hardcoded constant survives
  * any packager that drops .json files.
  */
-export declare const PACKAGE_VERSION = "1.60.0";
+export declare const PACKAGE_VERSION = "1.61.0";
 /** Compares plain x.y.z versions, positive if a > b; tags throw so the rollback guard never misfires silently. */
 export declare function compareSemver(a: string, b: string): number;
 //# sourceMappingURL=version.d.ts.map

package/dist/version.js

@@ -6,7 +6,7 @@
  * hand. scripts/check-manifest-versions.mjs gates publish on all seven sites.
  *
  * Used by:
- *   - src/db.ts rollback-safety guard (refuses to open a DB stamped with
+ *   - src/db/migrate.ts rollback-safety guard (refuses to open a DB stamped with
  *     min_compatible_binary newer than this).
  *   - src/server.ts HTTP /health.
  *   - src/mcp/server.ts MCP serverInfo.
@@ -16,7 +16,7 @@
  * an ESM `import` can resolve cleanly, and a hardcoded constant survives
  * any packager that drops .json files.
  */
-export const PACKAGE_VERSION = '1.60.0';
+export const PACKAGE_VERSION = '1.61.0';
 /** Compares plain x.y.z versions, positive if a > b; tags throw so the rollback guard never misfires silently. */
 export function compareSemver(a, b) {
     const parse = (v) => {

package/dist/working-memory.js

@@ -6,7 +6,7 @@
  * exceeds WM_MAX_ENTRIES per scope.
  */
 import { closeHippoDb } from './db.js';
-import { openStore } from './store.js';
+import { openStore } from './store/open.js';
 export const WM_MAX_ENTRIES = 20;
 function rowToItem(row) {
     let metadata = {};

package/dist/yaml.js

@@ -3,13 +3,42 @@
  * No external deps. Handles simple key-value + arrays (inline only).
  * Sufficient for the MemoryEntry frontmatter schema.
  */
+const SCALAR_LOOKALIKE = /^(null|true|false|-?\d+(\.\d+)?)$/;
+const UNESCAPES = new Map([['\\', '\\'], ['"', '"'], ['n', '\n'], ['r', '\r']]);
 function escapeString(s) {
-    // If string contains special chars, quote it
-    if (/[:#\[\]{},\n\r"']/.test(s) || s.trim() !== s || s === '') {
-        return `"${s.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`;
+    // A string that reads as null, a boolean or a number is quoted so it comes back a string.
+    if (/[:#\[\]{},\n\r"']/.test(s) || s.trim() !== s || s === '' || SCALAR_LOOKALIKE.test(s)) {
+        const escaped = s.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n').replace(/\r/g, '\\r');
+        return `"${escaped}"`;
     }
     return s;
 }
+// An unknown escape stays as written, so a hand-written Windows path still reads back.
+function unescapeQuoted(body) {
+    return body.replace(/\\([\s\S])/g, (all, ch) => UNESCAPES.get(ch) ?? all);
+}
+function parseString(raw) {
+    const t = raw.trim();
+    return t.length >= 2 && t.startsWith('"') && t.endsWith('"') ? unescapeQuoted(t.slice(1, -1)) : t;
+}
+function splitInlineList(inner) {
+    const items = [];
+    let start = 0;
+    let inQuotes = false;
+    for (let i = 0; i < inner.length; i++) {
+        const ch = inner[i];
+        if (inQuotes && ch === '\\')
+            i++;
+        else if (ch === '"')
+            inQuotes = !inQuotes;
+        else if (ch === ',' && !inQuotes) {
+            items.push(inner.slice(start, i));
+            start = i + 1;
+        }
+    }
+    items.push(inner.slice(start));
+    return items.map(parseString);
+}
 function isYamlBoolean(val) {
     return typeof val === 'boolean';
 }
@@ -53,15 +82,11 @@ function parseValue(raw) {
         const inner = s.slice(1, -1).trim();
         if (inner === '')
             return [];
-        return inner.split(',').map((item) => {
-            const t = item.trim();
-            return t.startsWith('"') && t.endsWith('"') ? t.slice(1, -1) : t;
-        });
+        return splitInlineList(inner);
     }
     // Quoted string
-    if (s.startsWith('"') && s.endsWith('"')) {
-        return s.slice(1, -1).replace(/\\"/g, '"').replace(/\\\\/g, '\\');
-    }
+    if (s.startsWith('"') && s.endsWith('"'))
+        return parseString(s);
     // Number
     if (/^-?\d+(\.\d+)?$/.test(s)) {
         return parseFloat(s);
@@ -91,7 +116,7 @@ export function parseFrontmatter(raw) {
                 const listLine = frontLines[j].trim();
                 if (!listLine.startsWith('- '))
                     break;
-                items.push(listLine.slice(2).trim());
+                items.push(parseString(listLine.slice(2)));
                 j++;
             }
             if (items.length > 0) {