hippo-memory 2.0.0 → 2.1.0

package/dist/store/entry-reads.js

@@ -1,4 +1,4 @@
-import { closeHippoDb } from '../db/index.js';
+import { closeHippoDb, ftsRowCounts, isFtsAvailable } from '../db/index.js';
 import { MEMORY_SELECT_COLUMNS, rowToEntry, parseJsonArray } from './rows.js';
 import { onHandle, openStore } from './open.js';
 import { escapeLike } from '../util/escape.js';
@@ -8,9 +8,11 @@ import { chunked } from '../util/chunked.js';
 import { DEFAULT_TENANT_ID } from '../util/env.js';
 import { placeholders } from '../util/sql.js';
 const MAX_IDS_PER_READ = 500;
-// The plus keeps SQLite on the primary key for an id list: with a bare tenant_id it walks every row of the tenant instead.
+// The plus keeps SQLite on the primary key for an id list: with
+// a bare tenant_id it walks every row of the tenant instead.
 export const TENANT_IS = '+tenant_id = ?';
-/** Read a memory entry by ID. With `tenantId` the read is tenant-scoped (cross-tenant returns null); omitted, no tenant filter is applied. */
+/** Read a memory entry by ID. With `tenantId` the read is tenant-scoped (cross-tenant returns null); omitted, no
+ * tenant filter is applied. */
 export function readEntry(hippoRoot, id, tenantId) {
     return onHandle(hippoRoot, (db) => {
         // SAFETY: both branches select exactly MEMORY_SELECT_COLUMNS, matching
@@ -21,30 +23,8 @@ export function readEntry(hippoRoot, id, tenantId) {
         return row ? rowToEntry(row) : null;
     }, openStore);
 }
-/** Answers whether a tenant's rows in this store hold an id: `ids` are looked up now, one query per chunk, and any other id on its first ask. */
-export function heldIdLookup(hippoRoot, tenantId, ids) {
-    const held = new Map();
-    const lookUp = (asked) => {
-        onHandle(hippoRoot, (db) => {
-            for (const chunk of chunked([...new Set(asked)])) {
-                for (const id of chunk)
-                    held.set(id, false);
-                // SAFETY: rows' shape matches the single `id` column selected.
-                const rows = db.prepare(`SELECT id FROM memories WHERE id IN (${placeholders(chunk.length)}) AND ${TENANT_IS}`).all(...chunk, tenantId);
-                for (const row of rows)
-                    held.set(row.id, true);
-            }
-        }, openStore);
-    };
-    if (ids.length > 0)
-        lookUp(ids);
-    return (id) => {
-        if (!held.has(id))
-            lookUp([id]);
-        return held.get(id) === true;
-    };
-}
-/** Rows by id on the caller's handle, one query per chunk; an id missing or in another tenant is absent from the map. */
+/** Rows by id on the caller's handle, one query per chunk; an
+ * id missing or in another tenant is absent from the map. */
 export function selectEntriesByIds(db, ids, tenantId) {
     const byId = new Map();
     const tenantClause = tenantId !== undefined ? ` AND ${TENANT_IS}` : '';
@@ -66,7 +46,10 @@ export function selectChildrenByParent(db, parentIds, tenantId) {
     for (const chunk of chunked([...new Set(parentIds)])) {
         const marks = placeholders(chunk.length);
         // SAFETY: selects exactly MEMORY_SELECT_COLUMNS, matching MemoryRow's field set.
-        const rows = db.prepare(`SELECT ${MEMORY_SELECT_COLUMNS} FROM memories WHERE dag_parent_id IN (${marks})${tenantClause} ORDER BY created ASC, id ASC`).all(...chunk, ...tenantArgs);
+        const rows = db
+            .prepare(`SELECT ${MEMORY_SELECT_COLUMNS} FROM memories ` +
+            `WHERE dag_parent_id IN (${marks})${tenantClause} ORDER BY created ASC, id ASC`)
+            .all(...chunk, ...tenantArgs);
         for (const row of rows) {
             const entry = rowToEntry(row);
             const parentId = entry.dag_parent_id ?? '';
@@ -79,7 +62,8 @@ export function selectChildrenByParent(db, parentIds, tenantId) {
     }
     return byParent;
 }
-/** Batched lookup, capped at 500 ids per call to keep IN(?,?,...) within SQLite limits; tenant filter enforced when `tenantId` is passed. */
+/** Batched lookup, capped at 500 ids per call to keep IN(?,?,...) within SQLite limits; tenant filter enforced when
+ * `tenantId` is passed. */
 export function loadEntriesByIds(hippoRoot, ids, tenantId) {
     if (ids.length === 0)
         return [];
@@ -89,8 +73,14 @@ export function loadEntriesByIds(hippoRoot, ids, tenantId) {
         // Without ORDER BY, rows follow SQLite's IN(...) scan order, undefined w.r.t. `ids`.
         // SAFETY: both branches select exactly MEMORY_SELECT_COLUMNS, matching MemoryRow's field set.
         const rows = tenantId !== undefined
-            ? db.prepare(`SELECT ${MEMORY_SELECT_COLUMNS} FROM memories WHERE id IN (${marks}) AND ${TENANT_IS} ORDER BY created ASC, content ASC, id ASC`).all(...capped, tenantId)
-            : db.prepare(`SELECT ${MEMORY_SELECT_COLUMNS} FROM memories WHERE id IN (${marks}) ORDER BY created ASC, content ASC, id ASC`).all(...capped);
+            ? db
+                .prepare(`SELECT ${MEMORY_SELECT_COLUMNS} FROM memories WHERE id IN (${marks}) ` +
+                `AND ${TENANT_IS} ORDER BY created ASC, content ASC, id ASC`)
+                .all(...capped, tenantId)
+            : db
+                .prepare(`SELECT ${MEMORY_SELECT_COLUMNS} FROM memories ` +
+                `WHERE id IN (${marks}) ORDER BY created ASC, content ASC, id ASC`)
+                .all(...capped);
         return rows.map(rowToEntry);
     }, openStore);
 }
@@ -98,13 +88,15 @@ function originClause(origins) {
     return origins === undefined ? '' : ` AND (origin_project = '' OR ${originInSql(origins)})`;
 }
 /** All `kind='raw'` rows for a session, tenant-scoped, oldest-first, excluding superseded; `[]` for an empty sessionId.
- * With `cap`, the NEWEST rows are loaded (DESC LIMIT, reversed client-side); ASC + LIMIT would drop the newest and break fresh-tail in assemble. */
+ * With `cap`, the NEWEST rows are loaded (DESC LIMIT, reversed client-side); ASC + LIMIT would drop the newest and
+ * break fresh-tail in assemble. */
 export function loadSessionRawMemories(hippoRoot, sessionId, tenantId, cap, origins) {
     if (!sessionId)
         return [];
     return onHandle(hippoRoot, (db) => {
         const params = [];
-        let sql = `SELECT ${MEMORY_SELECT_COLUMNS} FROM memories WHERE kind = 'raw' AND source_session_id = ? AND superseded_by IS NULL`;
+        let sql = `SELECT ${MEMORY_SELECT_COLUMNS} FROM memories WHERE kind = 'raw' ` +
+            `AND source_session_id = ? AND superseded_by IS NULL`;
         params.push(sessionId);
         if (tenantId !== undefined) {
             sql += ' AND tenant_id = ?';
@@ -125,14 +117,17 @@ export function loadSessionRawMemories(hippoRoot, sessionId, tenantId, cap, orig
         return rows.map(rowToEntry);
     }, openStore);
 }
-/** Pre-cap, scope-aware row count for a session, so `assemble` reports the full size without leaking rows the caller could not load. Mirrors
- * `passesScopeFilterForRecall`'s default-deny: explicit scope matches exactly, none admits only `ownScope`. `tenantId` undefined counts cross-tenant. */
+/** Pre-cap, scope-aware row count for a session, so `assemble` reports the full size without leaking rows the caller
+ * could not load. Mirrors
+ * `passesScopeFilterForRecall`'s default-deny: explicit scope matches exactly, none admits only `ownScope`.
+ * `tenantId` undefined counts cross-tenant. */
 export function countSessionRawMemories(hippoRoot, sessionId, tenantId, scope, ownScope, origins) {
     if (!sessionId)
         return 0;
     return onHandle(hippoRoot, (db) => {
         const params = [];
-        let sql = `SELECT COUNT(*) AS c FROM memories WHERE kind = 'raw' AND source_session_id = ? AND superseded_by IS NULL`;
+        let sql = `SELECT COUNT(*) AS c FROM memories WHERE kind = 'raw' ` +
+            `AND source_session_id = ? AND superseded_by IS NULL`;
         params.push(sessionId);
         if (tenantId !== undefined) {
             sql += ' AND tenant_id = ?';
@@ -154,8 +149,10 @@ export function countSessionRawMemories(hippoRoot, sessionId, tenantId, scope, o
         return Number(row?.c ?? 0);
     }, openStore);
 }
-/** Last N kind='raw' memories by `created` desc, tenant-scoped, max 200; pass `sessionId` or concurrent sessions show up in each other's fresh tail.
- * The tenant-wide shape is discouraged: `api.recall` throws `RecallContractError` for it when `HIPPO_REQUIRE_SESSION_SCOPED_FRESH_TAIL=1`. */
+/** Last N kind='raw' memories by `created` desc, tenant-scoped, max 200; pass
+ * `sessionId` or concurrent sessions show up in each other's fresh tail. The
+ * tenant-wide shape is discouraged: `api.recall` throws `RecallContractError`
+ * for it when `HIPPO_REQUIRE_SESSION_SCOPED_FRESH_TAIL=1`. */
 export function loadFreshRawMemories(hippoRoot, count, tenantId, sessionId, origins) {
     if (count <= 0)
         return [];
@@ -182,13 +179,15 @@ export function loadFreshRawMemories(hippoRoot, count, tenantId, sessionId, orig
         return rows.map(rowToEntry);
     }, openStore);
 }
-/** Load all entries from SQLite; omitting `tenantId` yields all rows (legacy). Recall paths that surface results to a user MUST pass a resolved tenant. */
+/** Load all entries from SQLite; omitting `tenantId` yields all rows (legacy). Recall paths that surface results to
+ * a user MUST pass a resolved tenant. */
 export function loadAllEntries(hippoRoot, tenantId) {
     return onHandle(hippoRoot, (db) => {
         return selectAllEntries(db, tenantId);
     }, openStore);
 }
-/** Every tenant's memory ids in loadAllEntries' order, for a caller that reads the rows a page at a time instead of holding them all. */
+/** Every tenant's memory ids in loadAllEntries' order, for a caller that reads the rows a page at a time instead of
+ * holding them all. */
 export function loadAllEntryIds(hippoRoot) {
     return onHandle(hippoRoot, (db) => {
         // SAFETY: the SELECT names exactly the one column read.
@@ -204,15 +203,46 @@ export function countAllEntries(hippoRoot) {
     }, openStore);
 }
 // The only code points whose JS toLowerCase yields ASCII, which SQLite's ASCII-only LIKE folding cannot see.
-const FOLDS_TO_ASCII = ['\u0130', '\u212A'];
-/** A superset, in loadAllEntries' order, of the rows whose lowercased content holds any of `tokens` or whose lowercased tags hold `tag`.
+const FOLDS_TO_ASCII = ['İ', 'K'];
+// Chars or bytes differ only when a non-ASCII char is present; the check never decodes the value.
+const hasNonAscii = (column) => `length(${column}) != length(CAST(${column} AS BLOB))`;
+/** The FTS5 phrase for `text`: its ASCII alphanumeric runs in order, which is how unicode61 splits it; null when it
+ * holds none (the phrase would be empty) or any non-ASCII char. */
+function ftsPhrase(text) {
+    if (!/^[ -~]*$/.test(text))
+        return null;
+    const pieces = text.match(/[A-Za-z0-9]+/g);
+    return pieces === null ? null : `"${pieces.join(' ')}"`;
+}
+/** MATCH expression whose hits include every row the LIKE path would keep, or null when some token or the tag has no
+ * FTS form. Rows with non-ASCII text are added by the caller: JS tokenizers drop those chars, unicode61 keeps them. */
+function ftsMentionQuery(tokens, tag) {
+    const parts = [];
+    for (const t of new Set(tokens)) {
+        const phrase = ftsPhrase(t);
+        if (phrase === null)
+            return null;
+        parts.push(phrase);
+    }
+    if (tag !== null) {
+        const phrase = ftsPhrase(tag);
+        if (phrase === null)
+            return null;
+        parts.push(`tags : ${phrase}`);
+    }
+    return parts.length === 0 ? null : parts.join(' OR ');
+}
+/** A superset, in loadAllEntries' order, of the rows whose lowercased content holds any of `tokens` or whose lowercased
+ * tags hold `tag`.
  * The caller re-checks each row exactly; this only spares decoding rows that cannot match. */
 export function loadEntriesMentioning(hippoRoot, tokens, tag, tenantId) {
     const likes = [...new Set(tokens)].map((t) => `%${escapeLike(t)}%`);
     const clauses = likes.map(() => `content LIKE ? ESCAPE '\\'`);
     const params = [...likes];
-    // A tag needing JSON escapes or case folding past ASCII is not found by a substring of tags_json, so every tagged row stays in.
-    if (/^[\x20-\x7e]*$/.test(tag) && !/["\\]/.test(tag)) {
+    // A tag needing JSON escapes or case folding past ASCII is not
+    // found by a substring of tags_json, so every tagged row stays in.
+    const plainTag = /^[ -~]*$/.test(tag) && !/["\\]/.test(tag);
+    if (plainTag) {
         clauses.push(`tags_json LIKE ? ESCAPE '\\'`);
         params.push(`%${escapeLike(tag)}%`);
     }
@@ -223,23 +253,43 @@ export function loadEntriesMentioning(hippoRoot, tokens, tag, tenantId) {
         params.push(ch, ch);
     }
     const tenantClause = tenantId !== undefined ? 'tenant_id = ? AND ' : '';
+    const tenantParams = tenantId !== undefined ? [tenantId] : [];
     return onHandle(hippoRoot, (db) => {
+        const ftsQuery = plainTag && isFtsAvailable(db) && ftsInSync(db) ? ftsMentionQuery(tokens, tag) : null;
+        const where = ftsQuery === null
+            ? clauses.join(' OR ')
+            : `id IN (SELECT id FROM memories_fts WHERE memories_fts MATCH ?) OR ${hasNonAscii('content')} OR ` +
+                hasNonAscii('tags_json');
         // SAFETY: selects exactly MEMORY_SELECT_COLUMNS, matching MemoryRow's field set.
-        const rows = db.prepare(`SELECT ${MEMORY_SELECT_COLUMNS} FROM memories WHERE ${tenantClause}(${clauses.join(' OR ')}) ORDER BY created ASC, id ASC`).all(...(tenantId !== undefined ? [tenantId] : []), ...params);
+        const rows = db
+            .prepare(`SELECT ${MEMORY_SELECT_COLUMNS} FROM memories ` +
+            `WHERE ${tenantClause}(${where}) ORDER BY created ASC, id ASC`)
+            .all(...tenantParams, ...(ftsQuery === null ? params : [ftsQuery]));
         return rows.map(rowToEntry);
     }, openStore);
 }
-/** The rows in loadAllEntries' order (every tenant unless one is named), with only the columns that say whether another store already holds each. */
+function ftsInSync(db) {
+    const counts = ftsRowCounts(db);
+    return counts !== null && counts.memories === counts.fts;
+}
+/** The rows in loadAllEntries' order (every tenant unless one is named), with only the columns that say whether
+ * another store already holds each. */
 export function loadEntryTextKeys(hippoRoot, tenantId) {
     return onHandle(hippoRoot, (db) => {
         const where = tenantId !== undefined ? 'WHERE tenant_id = ? ' : '';
         // SAFETY: rows' shape matches the four columns named in the SELECT.
         const rows = db.prepare(`SELECT id, tenant_id, content, source FROM memories ${where}ORDER BY created ASC, id ASC`)
             .all(...(tenantId !== undefined ? [tenantId] : []));
-        return rows.map((row) => ({ id: row.id, tenant_id: row.tenant_id ?? DEFAULT_TENANT_ID, content: row.content, source: row.source }));
+        return rows.map((row) => ({
+            id: row.id,
+            tenant_id: row.tenant_id ?? DEFAULT_TENANT_ID,
+            content: row.content,
+            source: row.source,
+        }));
     }, openStore);
 }
-/** Every entry plus a by-id lookup of each as loaded, rebuilt from its row on first use, so no deep copy of the store is kept. */
+/** Every entry plus a by-id lookup of each as loaded, rebuilt from its row on first use, so no deep copy of the
+ * store is kept. */
 export function loadAllEntriesWithBase(hippoRoot) {
     const db = openStore(hippoRoot);
     let rows;
@@ -284,7 +334,9 @@ export function loadEntriesPage(hippoRoot, tenantId, page) {
     params.push(page.limit + 1);
     return onHandle(hippoRoot, (db) => {
         // SAFETY: selects exactly MEMORY_SELECT_COLUMNS, matching MemoryRow's field set.
-        const rows = db.prepare(`SELECT ${MEMORY_SELECT_COLUMNS} FROM memories ${clause}ORDER BY created ASC, id ASC LIMIT ?`).all(...params);
+        const rows = db
+            .prepare(`SELECT ${MEMORY_SELECT_COLUMNS} FROM memories ${clause}ORDER BY created ASC, id ASC LIMIT ?`)
+            .all(...params);
         const entries = rows.slice(0, page.limit).map(rowToEntry);
         const last = entries[entries.length - 1];
         return { entries, next: rows.length > page.limit && last ? { created: last.created, id: last.id } : null };
@@ -294,29 +346,42 @@ export function loadEntriesPage(hippoRoot, tenantId, page) {
 export function selectAllEntries(db, tenantId) {
     return selectAllRows(db, tenantId).map(rowToEntry);
 }
-/** The columns a text match needs, for every row of a tenant in loadAllEntries' order: callers read whole rows only for the ids that match. */
+/** The columns a text match needs, for every row of a tenant in loadAllEntries' order: callers read whole rows only
+ * for the ids that match. */
 export function selectTenantTexts(db, tenantId) {
     // SAFETY: rows' shape matches the four columns named in the SELECT.
-    return db.prepare('SELECT id, content, source, scope FROM memories WHERE tenant_id = ? ORDER BY created ASC, id ASC').all(tenantId);
+    return db
+        .prepare('SELECT id, content, source, scope FROM memories WHERE tenant_id = ? ORDER BY created ASC, id ASC')
+        .all(tenantId);
 }
-/** Live rows whose source starts with `prefix`, on the caller's handle; LIKE folds case, so the prefix is checked again exactly. */
+/** Live rows whose source starts with `prefix`, on the caller's
+ * handle; LIKE folds case, so the prefix is checked again exactly. */
 export function selectLiveEntriesBySourcePrefix(db, tenantId, prefix) {
     // SAFETY: selects exactly MEMORY_SELECT_COLUMNS, matching MemoryRow's field set.
-    const rows = db.prepare(`SELECT ${MEMORY_SELECT_COLUMNS} FROM memories WHERE tenant_id = ? AND superseded_by IS NULL AND source LIKE ? ESCAPE '\\'`).all(tenantId, `${escapeLike(prefix)}%`);
+    const rows = db
+        .prepare(`SELECT ${MEMORY_SELECT_COLUMNS} FROM memories WHERE tenant_id = ? ` +
+        `AND superseded_by IS NULL AND source LIKE ? ESCAPE '\\'`)
+        .all(tenantId, `${escapeLike(prefix)}%`);
     return rows.map(rowToEntry).filter((entry) => entry.source.startsWith(prefix));
 }
-// Content of every team-visible tenant row tagged `tag`, without reading the rest of the store; `origins` keeps one project's rows and user-global ones.
-// No owner: a personal or connector-private row must never answer `duplicate` for, or stop, a team copy. `instr` prefilters; `includes` re-checks.
+// Content of every team-visible tenant row tagged `tag`, without reading the rest of the store; `origins` keeps one
+// project's rows and user-global ones.
+// No owner: a personal or connector-private row must never answer `duplicate` for, or stop, a team copy. `instr`
+// prefilters; `includes` re-checks.
 export function loadContentsWithTag(hippoRoot, tenantId, tag, origins) {
     return onHandle(hippoRoot, (db) => {
         const inProject = origins === undefined ? '' : ` AND (origin_project = '' OR ${originInSql(origins)})`;
         const admit = scopeAdmitSql('');
         /** SAFETY: rows' shape matches the two columns named in the SELECT below. */
-        const rows = db.prepare(`SELECT content, tags_json FROM memories WHERE tenant_id = ? AND instr(tags_json, ?) > 0${inProject} AND ${admit.sql}`).all(tenantId, JSON.stringify(tag), ...(origins ?? []), ...admit.params);
+        const rows = db
+            .prepare(`SELECT content, tags_json FROM memories WHERE tenant_id = ? ` +
+            `AND instr(tags_json, ?) > 0${inProject} AND ${admit.sql}`)
+            .all(tenantId, JSON.stringify(tag), ...(origins ?? []), ...admit.params);
         return rows.filter((r) => parseJsonArray(r.tags_json).includes(tag)).map((r) => r.content);
     }, openStore);
 }
-/** Live raw rows whose artifact_ref matches a LIKE pattern, for one tenant; the store is set up first when it is new. */
+/** Live raw rows whose artifact_ref matches a LIKE pattern,
+ * for one tenant; the store is set up first when it is new. */
 export function loadVaultRawRows(hippoRoot, likeParam, tenantId) {
     return onHandle(hippoRoot, (db) => {
         // SAFETY: query selects exactly the columns of VaultRawRow, in the same
@@ -331,7 +396,8 @@ export function loadVaultRawRows(hippoRoot, likeParam, tenantId) {
 export function selectPreviewRows(db, columns, tenantId) {
     const tenant = columns.has('tenant_id') ? ' WHERE tenant_id = ?' : '';
     const column = (name, fallback) => (columns.has(name) ? name : `${fallback} AS ${name}`);
-    const statement = db.prepare(`SELECT id, content, ${column('source', "''")}, ${column('confidence', 'NULL')}, ${column('extracted_from', 'NULL')}, ` +
+    const statement = db.prepare(`SELECT id, content, ${column('source', "''")}, ` +
+        `${column('confidence', 'NULL')}, ${column('extracted_from', 'NULL')}, ` +
         `${column('dag_level', '0')}, ${column('tags_json', "'[]'")} FROM memories${tenant}`);
     // SAFETY: the SELECT names every PreviewRow field, each a literal fallback when its column is missing.
     return (tenant ? statement.all(tenantId) : statement.all());
@@ -343,12 +409,18 @@ export function entryIdTakenAt(db, id) {
 /** The text, tenant and scope of memory `id`, or undefined when no row holds it. */
 export function entryRejectRowAt(db, id) {
     // SAFETY: row's shape matches the three columns named in the SELECT.
-    return db
-        .prepare(`SELECT content, tenant_id, scope FROM memories WHERE id = ?`)
-        .get(id);
+    return db.prepare(`SELECT content, tenant_id, scope FROM memories WHERE id = ?`).get(id);
 }
 /** Memory ids held by each typed-object table, with the status of the object that holds them. */
-const OBJECT_TABLES = ['decisions', 'incidents', 'processes', 'policies', 'skills', 'project_briefs', 'customer_notes'];
+const OBJECT_TABLES = [
+    'decisions',
+    'incidents',
+    'processes',
+    'policies',
+    'skills',
+    'project_briefs',
+    'customer_notes',
+];
 export function objectMemoryRowsAt(db) {
     const out = [];
     for (const table of OBJECT_TABLES) {
@@ -357,7 +429,8 @@ export function objectMemoryRowsAt(db) {
     }
     return out;
 }
-/** Live rows of a tenant whose source does not start with `sourcePrefix`; `origins` (global store only) limits them to user-global rows and those projects. */
+/** Live rows of a tenant whose source does not start with `sourcePrefix`; `origins` (global store only) limits them
+ * to user-global rows and those projects. */
 export function selectRowsOutsideSourcePrefixAt(db, tenantId, sourcePrefix, origins) {
     const visible = origins !== null ? ` AND (origin_project = '' OR ${originInSql(origins)})` : '';
     const params = origins !== null ? [tenantId, sourcePrefix, ...origins] : [tenantId, sourcePrefix];
@@ -370,15 +443,18 @@ export function selectRowsOutsideSourcePrefixAt(db, tenantId, sourcePrefix, orig
 export function loadLiveContentsBySourceAndTag(hippoRoot, tenantId, source, tag, exceptSessionId) {
     return onHandle(hippoRoot, (db) => {
         /** SAFETY: rows' shape matches the two columns named in the SELECT below. */
-        const rows = db.prepare(`SELECT content, tags_json FROM memories
+        const rows = db
+            .prepare(`SELECT content, tags_json FROM memories
        WHERE tenant_id = ? AND source = ? AND instr(tags_json, ?) > 0
          AND COALESCE(extracted_from, '') = '' AND COALESCE(superseded_by, '') = ''
          AND (source_session_id IS NULL OR source_session_id != ?)
-       ORDER BY created ASC, id ASC`).all(tenantId, source, JSON.stringify(tag), exceptSessionId);
+       ORDER BY created ASC, id ASC`)
+            .all(tenantId, source, JSON.stringify(tag), exceptSessionId);
         return rows.filter((r) => parseJsonArray(r.tags_json).includes(tag)).map((r) => r.content);
     }, openStore);
 }
-/** Every tenant's distilled rows nothing has superseded, in loadAllEntries' order: the rows that can be each other's duplicate. */
+/** Every tenant's distilled rows nothing has superseded, in
+ * loadAllEntries' order: the rows that can be each other's duplicate. */
 export function loadCurrentDistilledEntries(hippoRoot) {
     return onHandle(hippoRoot, (db) => {
         // SAFETY: selects exactly MEMORY_SELECT_COLUMNS, matching MemoryRow's field set.
@@ -388,4 +464,43 @@ export function loadCurrentDistilledEntries(hippoRoot) {
         return rows.map(rowToEntry);
     }, openStore);
 }
+/** Rows holding at least one signal, in loadAllEntries' order; omitting `tenantId` reads every tenant. A superset:
+ * callers re-check each row. */
+export function loadEntriesWithSignals(hippoRoot, signals, tenantId) {
+    // A backslash may hide an escaped tag from instr, so such rows always pass; LIKE folds case, so the prefix match is
+    // wider than startsWith.
+    const anyOf = [
+        'pinned != 0', 'retrieval_count >= ?', `emotional_valence IN (${placeholders(signals.valences.length)})`,
+        "source LIKE ? ESCAPE '\\'", "instr(tags_json, '\\') > 0", ...signals.tags.map(() => 'instr(tags_json, ?) > 0'),
+    ];
+    const params = [
+        signals.minRetrievals,
+        ...signals.valences,
+        `${escapeLike(signals.sourcePrefix)}%`,
+        ...signals.tags.map((t) => JSON.stringify(t)),
+    ];
+    const tenant = tenantId !== undefined ? 'tenant_id = ? AND ' : '';
+    return onHandle(hippoRoot, (db) => {
+        // SAFETY: selects exactly MEMORY_SELECT_COLUMNS, matching MemoryRow's field set.
+        const rows = db
+            .prepare(`SELECT ${MEMORY_SELECT_COLUMNS} FROM memories ` +
+            `WHERE ${tenant}(${anyOf.join(' OR ')}) ORDER BY created ASC, id ASC`)
+            .all(...(tenantId !== undefined ? [tenantId] : []), ...params);
+        return rows.map(rowToEntry);
+    }, openStore);
+}
+/** A tenant's rows from `origins` that nothing superseded and
+ * that are neither raw nor archived, in loadAllEntries' order. */
+export function loadCurrentEntriesFromOrigins(hippoRoot, tenantId, origins) {
+    return onHandle(hippoRoot, (db) => {
+        // SAFETY: selects exactly MEMORY_SELECT_COLUMNS, matching MemoryRow's field set.
+        const rows = db
+            .prepare(`SELECT ${MEMORY_SELECT_COLUMNS} FROM memories
+       WHERE tenant_id = ? AND ${originInSql(origins)} AND COALESCE(superseded_by, '') = ''
+         AND COALESCE(kind, 'distilled') NOT IN ('raw', 'archived')
+       ORDER BY created ASC, id ASC`)
+            .all(tenantId, ...origins);
+        return rows.map(rowToEntry);
+    }, openStore);
+}
 //# sourceMappingURL=entry-reads.js.map

package/dist/store/entry-row.d.ts

@@ -7,17 +7,21 @@ export declare function upsertMemoryRow(db: ReturnType<typeof openHippoDb>, entr
 export declare function syncFtsRow(db: ReturnType<typeof openHippoDb>, entry: MemoryEntry, isNewRow?: boolean): void;
 export declare function ftsRowExists(db: ReturnType<typeof openHippoDb>, id: string): boolean;
 export declare function deleteFtsRow(db: ReturnType<typeof openHippoDb>, id: string): void;
-/** One set delete for `staleIds`, because `id` is UNINDEXED and each delete by id scans the whole index; then one insert per row. */
+/** One set delete for `staleIds`, because `id` is UNINDEXED and each delete by id scans the whole index; then one
+ * insert per row. */
 export declare function replaceFtsRows(db: ReturnType<typeof openHippoDb>, rows: readonly MemoryEntry[], staleIds: readonly string[]): void;
-/** Row counts of a full-text index that has drifted from `memories`, re-synced after the count when `repair`; null when the two agree or there is no index. */
+/** Row counts of a full-text index that has drifted from `memories`, re-synced after the count when `repair`; null when
+ * the two agree or there is no index. */
 export declare function ftsDrift(hippoRoot: string, repair: boolean): {
     memories: number;
     fts: number;
 } | null;
-/** Stamp origin_project from the store's own location (parent of `<project>/.hippo`; the home store is '' = user-global) when never stamped; returns a copy.
- * NULL is PRESERVED: it means no known project and ambient context denies it, so a writeback must not launder it. */
+/** Stamp origin_project from the store's own location (parent of `<project>/.hippo`; the home store is '' =
+ * user-global) when never stamped; returns a copy. NULL is PRESERVED: it means no known project and ambient context
+ * denies it, so a writeback must not launder it. */
 export declare function stampOriginProject(hippoRoot: string, entry: MemoryEntry): MemoryEntry;
-/** Import-time variant for a mirror with no origin field (an explicit null stays null), for rows predating the origin column.
- * Like the v39 backfill, the provenance source (`shared:<project>:` / `promoted:<localRoot>`) wins over the destination store's location. */
+/** Import-time variant for a mirror with no origin field (an explicit null stays null), for rows predating the origin
+ * column. Like the v39 backfill, the provenance source (`shared:<project>:` / `promoted:<localRoot>`) wins over the
+ * destination store's location. */
 export declare function stampOriginProjectForImport(hippoRoot: string, entry: MemoryEntry): MemoryEntry;
 //# sourceMappingURL=entry-row.d.ts.map

package/dist/store/entry-row.js

@@ -2,7 +2,7 @@ import { DEFAULT_TENANT_ID } from '../util/env.js';
 import { closeHippoDb, ftsRowCounts, openHippoDb, isFtsAvailable, repairFtsDrift } from '../db/index.js';
 import { fallbackOrigin, originFromSource } from '../core/project-identity.js';
 import { checkRejectionGuard } from './rejection.js';
-import { errorMessage, log } from '../util/log.js';
+import { errorMessage, log, errorFields } from '../util/log.js';
 /** A 1.x JS caller's `tenantId` would otherwise fall through to the default tenant without a sound. */
 function rejectLegacyTenantKey(entry) {
     if (Object.hasOwn(entry, 'tenantId') && entry.tenant_id === undefined) {
@@ -37,7 +37,8 @@ const UPSERT_MEMORY_SQL = `
       descendant_count, earliest_at, latest_at,
       dag_level_3_built_at,
       updated_at
-    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))
+    ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
+    ?, ?, datetime('now'))
     ON CONFLICT(id) DO UPDATE SET
       created = excluded.created,
       last_retrieved = excluded.last_retrieved,
@@ -129,7 +130,7 @@ export function syncFtsRow(db, entry, isNewRow = false) {
     }
     catch (err) {
         // The memories table stays authoritative; a stale FTS row only costs recall quality, so the write goes on.
-        log.warnThenDebug('fts-sync', `FTS index update failed for ${entry.id}; keyword recall may miss it: ${errorMessage(err)}`);
+        log.warnThenSample('fts-sync', `FTS index update failed for ${entry.id}; keyword recall may miss it: ${errorMessage(err)}`, errorFields(err));
     }
 }
 export function ftsRowExists(db, id) {
@@ -142,10 +143,11 @@ export function deleteFtsRow(db, id) {
         db.prepare(`DELETE FROM memories_fts WHERE id = ?`).run(id);
     }
     catch (err) {
-        log.warnThenDebug('fts-delete', `FTS index delete failed for ${id}; recall may return a stale hit: ${errorMessage(err)}`);
+        log.warnThenSample('fts-delete', `FTS index delete failed for ${id}; recall may return a stale hit: ${errorMessage(err)}`, errorFields(err));
     }
 }
-/** One set delete for `staleIds`, because `id` is UNINDEXED and each delete by id scans the whole index; then one insert per row. */
+/** One set delete for `staleIds`, because `id` is UNINDEXED and each delete by id scans the whole index; then one
+ * insert per row. */
 export function replaceFtsRows(db, rows, staleIds) {
     if (!isFtsAvailable(db))
         return;
@@ -157,7 +159,7 @@ export function replaceFtsRows(db, rows, staleIds) {
         catch (err) {
             // Their old rows are still indexed, so inserting them again would index an id twice; only new ids go in.
             kept = new Set(staleIds);
-            log.warnThenDebug('fts-delete', `FTS index delete failed for ${staleIds.length} row(s); recall may return a stale hit: ${errorMessage(err)}`);
+            log.warnThenSample('fts-delete', `FTS index delete failed for ${staleIds.length} row(s); recall may return a stale hit: ${errorMessage(err)}`, errorFields(err));
         }
     }
     const toInsert = rows.filter((row) => !kept.has(row.id));
@@ -169,11 +171,12 @@ export function replaceFtsRows(db, rows, staleIds) {
             insert.run(row.id, row.content, row.tags.join(' '));
         }
         catch (err) {
-            log.warnThenDebug('fts-sync', `FTS index update failed for ${row.id}; keyword recall may miss it: ${errorMessage(err)}`);
+            log.warnThenSample('fts-sync', `FTS index update failed for ${row.id}; keyword recall may miss it: ${errorMessage(err)}`, errorFields(err));
         }
     }
 }
-/** Row counts of a full-text index that has drifted from `memories`, re-synced after the count when `repair`; null when the two agree or there is no index. */
+/** Row counts of a full-text index that has drifted from `memories`, re-synced after the count when `repair`; null when
+ * the two agree or there is no index. */
 export function ftsDrift(hippoRoot, repair) {
     const db = openHippoDb(hippoRoot);
     try {
@@ -190,15 +193,17 @@ export function ftsDrift(hippoRoot, repair) {
         closeHippoDb(db);
     }
 }
-/** Stamp origin_project from the store's own location (parent of `<project>/.hippo`; the home store is '' = user-global) when never stamped; returns a copy.
- * NULL is PRESERVED: it means no known project and ambient context denies it, so a writeback must not launder it. */
+/** Stamp origin_project from the store's own location (parent of `<project>/.hippo`; the home store is '' =
+ * user-global) when never stamped; returns a copy. NULL is PRESERVED: it means no known project and ambient context
+ * denies it, so a writeback must not launder it. */
 export function stampOriginProject(hippoRoot, entry) {
     if (entry.origin_project !== undefined)
         return entry;
     return { ...entry, origin_project: fallbackOrigin(hippoRoot) };
 }
-/** Import-time variant for a mirror with no origin field (an explicit null stays null), for rows predating the origin column.
- * Like the v39 backfill, the provenance source (`shared:<project>:` / `promoted:<localRoot>`) wins over the destination store's location. */
+/** Import-time variant for a mirror with no origin field (an explicit null stays null), for rows predating the origin
+ * column. Like the v39 backfill, the provenance source (`shared:<project>:` / `promoted:<localRoot>`) wins over the
+ * destination store's location. */
 export function stampOriginProjectForImport(hippoRoot, entry) {
     if (entry.origin_project !== undefined)
         return entry;

package/dist/store/entry-writes.d.ts

@@ -10,24 +10,34 @@ export interface WriteEntryInternalOptions extends WriteEntryOptions {
     /** Runs in the same SAVEPOINT as the INSERT, so a throw rolls the row back. */
     afterWrite?: (db: DatabaseSyncLike, memoryId: string) => void;
 }
-/** Write an entry to SQLite and refresh mirrors. `opts.actor` defaults to 'cli'; the api layer passes the resolved actor,
- * so the audit log gets one row per write. */
+/** Write an entry to SQLite and refresh mirrors. `opts.actor` defaults to 'cli'; the api layer passes the resolved
+ * actor, so the audit log gets one row per write. */
 export declare function writeEntry(hippoRoot: string, entry: MemoryEntry, opts?: WriteEntryOptions): void;
-/** writeEntry for store code that needs the write's own handle: `opts.afterWrite` shares the INSERT's SAVEPOINT. */
-export declare function writeEntryWithHook(hippoRoot: string, entry: MemoryEntry, opts?: WriteEntryInternalOptions): void;
+/** Writes a promoted copy; with `auditAs` the `promote` audit row joins the INSERT's SAVEPOINT, so both or neither. */
+export declare function writePromotedEntry(hippoRoot: string, entry: MemoryEntry, opts: {
+    readonly actor?: string;
+    readonly sourceId: string;
+    readonly auditAs?: {
+        tenantId: string;
+        actor: string;
+    };
+}): void;
 /** writeEntry for each of `entries` in one transaction, so all land or none; each
  * keeps its audit row, and mirrors follow the commit. Returns how many were written; an empty list opens no store. */
 export declare function writeEntriesTogether(hippoRoot: string, entries: readonly MemoryEntry[]): number;
 /** Runs `fill` in one transaction; each `put` writes one entry and returns false when the store rejects its value,
- * leaving that row's refusal audit and the rest of the batch. Mirrors follow the commit. Returns how many were written. */
+ * leaving that row's refusal audit and the rest of the batch. Mirrors follow the commit. Returns how many were
+ * written. */
 export declare function writeEntriesSkippingRejected(hippoRoot: string, fill: (put: (entry: MemoryEntry) => boolean) => void, actor?: string): number;
 /** Adds `tag` to each of a tenant's rows that lacks it, in `ids` order, each read fresh
  * on one open store and committed alone; an id the tenant does not hold is skipped. */
 export declare function addTagToEntries(hippoRoot: string, tenantId: string, ids: readonly string[], tag: string): void;
-/** DB-only write path: upsert, afterWrite hook and audit row inside one withWriteScope; the caller owns `db` and must call `writeEntryMirrors` after any
- * outer tx commits, so a rolled-back tx never leaves orphan markdown. The audit row commits atomically with the INSERT. */
+/** DB-only write path: upsert, afterWrite hook and audit row inside one withWriteScope; the caller owns `db` and must
+ * call `writeEntryMirrors` after any outer tx commits, so a rolled-back tx never leaves orphan markdown. The audit row
+ * commits atomically with the INSERT. */
 export declare function writeEntryDbOnly(db: DatabaseSyncLike, entry: MemoryEntry, opts?: Pick<WriteEntryInternalOptions, 'actor' | 'afterWrite'>): void;
-/** What follows a row's upsert inside its write scope: the remember row, last of the write's audit rows, then the summary parent's dirty mark. */
+/** What follows a row's upsert inside its write scope: the remember row, last of the write's audit rows, then the
+ * summary parent's dirty mark. */
 export declare function auditEntryWrite(db: DatabaseSyncLike, entry: MemoryEntry, actor: string): void;
 /** Markdown mirror path, invoked AFTER commit (a rolled-back tx must leave no orphan markdown). */
 export declare function writeEntryMirrors(hippoRoot: string, entry: MemoryEntry): void;
@@ -39,13 +49,15 @@ export interface StrengthenOptions {
 /** Strengthen what a read returned: update only the four retrieval columns on the live row, never a stale copy.
  *  Best effort: a failure logs and never fails the read. Returns the ids found in this store. */
 export declare function strengthenRetrieved(hippoRoot: string, ids: readonly string[], opts: StrengthenOptions): Set<string>;
-/** strengthenRetrieved on an open handle in its own write scope, a savepoint inside a caller's transaction, so a recall's last writes share one handle. */
+/** strengthenRetrieved on an open handle in its own write scope, a savepoint inside a caller's transaction, so a
+ * recall's last writes share one handle. */
 export declare function strengthenRetrievedInOwnTx(db: DatabaseSyncLike, ids: readonly string[], opts: StrengthenOptions): Set<string>;
 /** strengthenRetrieved on the caller's handle, inside the caller's transaction. Throws; the caller decides. */
 export declare function strengthenRetrievedOn(db: DatabaseSyncLike, ids: readonly string[], opts: StrengthenOptions): Set<string>;
 /** Rewrites a live row's tags and its full-text row on the caller's transaction, with no audit row. */
 export declare function setEntryTagsInTx(db: DatabaseSyncLike, entry: MemoryEntry): void;
-/** Removes a row from `memories` and full-text search on the caller's transaction, as sleep's dormant move does, and marks its summary parent dirty. */
+/** Removes a row from `memories` and full-text search on the caller's transaction, as sleep's dormant move does, and
+ * marks its summary parent dirty. */
 export declare function deleteEntryRowInTx(db: DatabaseSyncLike, entry: MemoryEntry, actor: string): void;
 /** One half-life per memory id, on the caller's transaction. */
 export declare function setHalfLivesAt(db: DatabaseSyncLike, rows: readonly {