@intentic/iq 1.326.0 → 1.328.0

package/dist/recall/ingest/ingest.js

@@ -0,0 +1,180 @@
+import { readdirSync, statSync } from "node:fs";
+import { basename, isAbsolute, join, relative } from "node:path";
+import { isMissing } from "@intentic/base/errors";
+import { stripInjectedPreamble } from "@intentic/constants";
+import { readLines } from "../transcript/line-reader.js";
+import { aiTitleOf, assistantTextOf, fileTouchesOf, parseLine, timestampOf, typedPromptOf, uuidOf } from "../transcript/lines.js";
+const RESPONSE_CAP = 4000;
+const asString = (value) => (typeof value === "string" ? value : undefined);
+const parseDelta = async (path, fromByte, lastOrdinal, root) => {
+    const delta = {
+        slug: undefined,
+        version: undefined,
+        gitBranch: undefined,
+        title: undefined,
+        minTs: undefined,
+        maxTs: undefined,
+        byteOffset: fromByte,
+        newTurns: [],
+        touches: new Map(),
+        responses: new Map(),
+    };
+    let ordinal = lastOrdinal;
+    let lineStart = fromByte;
+    for await (const { json, endByte } of readLines(path, fromByte)) {
+        const startByte = lineStart;
+        lineStart = endByte;
+        delta.byteOffset = endByte;
+        const line = parseLine(json);
+        if (line === undefined) {
+            continue;
+        }
+        const ts = timestampOf(line);
+        if (ts !== undefined) {
+            delta.minTs = Math.min(delta.minTs ?? ts, ts);
+            delta.maxTs = Math.max(delta.maxTs ?? ts, ts);
+        }
+        delta.slug ??= asString(line["slug"]);
+        delta.version ??= asString(line["version"]);
+        delta.gitBranch ??= asString(line["gitBranch"]);
+        const title = aiTitleOf(line);
+        if (title !== undefined) {
+            delta.title = title;
+        }
+        const typed = typedPromptOf(line);
+        const uuid = uuidOf(line);
+        if (typed !== undefined && uuid !== undefined && ts !== undefined) {
+            ordinal += 1;
+            delta.newTurns.push({ uuid, ordinal, ts, prompt: stripInjectedPreamble(typed), startByte });
+            continue;
+        }
+        if (ordinal < 0) {
+            continue;
+        }
+        const text = assistantTextOf(line);
+        if (text !== undefined) {
+            delta.responses.set(ordinal, text.slice(0, RESPONSE_CAP));
+        }
+        for (const { path: touched, modified } of fileTouchesOf(line)) {
+            const rel = relative(root, touched);
+            if (rel === "" || rel.startsWith("..") || isAbsolute(rel)) {
+                continue;
+            }
+            const byPath = delta.touches.get(ordinal) ?? new Map();
+            delta.touches.set(ordinal, byPath);
+            const touch = byPath.get(rel);
+            if (touch === undefined) {
+                byPath.set(rel, { modified, lastTs: ts ?? delta.maxTs ?? 0 });
+            }
+            else {
+                touch.modified ||= modified;
+                touch.lastTs = Math.max(touch.lastTs, ts ?? 0);
+            }
+        }
+    }
+    return delta;
+};
+const applyDelta = (db, transcriptPath, sessionId, delta, stat) => {
+    db.run(`INSERT INTO sessions (session_id, slug, title, version, git_branch, first_ts, last_ts) VALUES (?, ?, ?, ?, ?, ?, ?)
+         ON CONFLICT(session_id) DO UPDATE SET
+             slug = coalesce(sessions.slug, excluded.slug),
+             title = coalesce(excluded.title, sessions.title),
+             version = coalesce(sessions.version, excluded.version),
+             git_branch = coalesce(sessions.git_branch, excluded.git_branch),
+             first_ts = min(sessions.first_ts, excluded.first_ts),
+             last_ts = max(sessions.last_ts, excluded.last_ts)`, sessionId, delta.slug ?? null, delta.title ?? null, delta.version ?? null, delta.gitBranch ?? null, delta.minTs ?? Date.now(), delta.maxTs ?? 0);
+    const sessionRowId = db.get("SELECT id FROM sessions WHERE session_id = ?", sessionId)?.["id"];
+    const newOrdinals = new Set(delta.newTurns.map((turn) => turn.ordinal));
+    for (const turn of delta.newTurns) {
+        db.run(`INSERT INTO turns (session_id, uuid, ordinal, ts, prompt, response, start_byte) VALUES (?, ?, ?, ?, ?, ?, ?)
+             ON CONFLICT(session_id, uuid) DO NOTHING`, sessionRowId, turn.uuid, turn.ordinal, turn.ts, turn.prompt, delta.responses.get(turn.ordinal) ?? "", turn.startByte);
+    }
+    for (const [ordinal, response] of delta.responses) {
+        if (!newOrdinals.has(ordinal)) {
+            db.run("UPDATE turns SET response = ? WHERE session_id = ? AND ordinal = ?", response, sessionRowId, ordinal);
+        }
+    }
+    for (const [ordinal, byPath] of delta.touches) {
+        const turnId = db.get("SELECT id FROM turns WHERE session_id = ? AND ordinal = ?", sessionRowId, ordinal)?.["id"];
+        if (turnId === undefined) {
+            continue;
+        }
+        for (const [path, touch] of byPath) {
+            db.run(`INSERT INTO turn_files (turn_id, path, modified, last_ts) VALUES (?, ?, ?, ?)
+                 ON CONFLICT(turn_id, path) DO UPDATE SET
+                     modified = max(turn_files.modified, excluded.modified),
+                     last_ts = max(turn_files.last_ts, excluded.last_ts)`, turnId, path, touch.modified ? 1 : 0, touch.lastTs);
+        }
+    }
+    db.run(`INSERT INTO transcripts (path, session_id, byte_offset, mtime_ms, size) VALUES (?, ?, ?, ?, ?)
+         ON CONFLICT(path) DO UPDATE SET byte_offset = excluded.byte_offset, mtime_ms = excluded.mtime_ms, size = excluded.size`, transcriptPath, sessionId, delta.byteOffset, stat.mtimeMs, stat.size);
+};
+export const ingest = async (db, options) => {
+    const onDisk = new Map();
+    let entries;
+    try {
+        entries = readdirSync(options.projectsDir);
+    }
+    catch (error) {
+        if (!isMissing(error)) {
+            throw error;
+        }
+        entries = [];
+    }
+    for (const entry of entries) {
+        if (!entry.endsWith(".jsonl")) {
+            continue;
+        }
+        const path = join(options.projectsDir, entry);
+        const stat = statSync(path);
+        if (stat.isFile()) {
+            onDisk.set(path, { mtimeMs: Math.trunc(stat.mtimeMs), size: stat.size });
+        }
+    }
+    const known = new Map(db.all("SELECT path, session_id, byte_offset, mtime_ms, size FROM transcripts").map((row) => [row["path"], row]));
+    for (const [path, row] of known) {
+        if (onDisk.has(path)) {
+            continue;
+        }
+        db.transaction(() => {
+            db.run("DELETE FROM sessions WHERE session_id = ?", row["session_id"]);
+            db.run("DELETE FROM transcripts WHERE path = ?", path);
+        });
+    }
+    const pending = [...onDisk]
+        .filter(([path, stat]) => {
+        const row = known.get(path);
+        return row === undefined || Number(row["mtime_ms"]) !== stat.mtimeMs || Number(row["size"]) !== stat.size;
+    })
+        .sort(([, a], [, b]) => b.mtimeMs - a.mtimeMs);
+    for (const [path, stat] of pending) {
+        const row = known.get(path);
+        if (options.deadlineMs !== undefined && Date.now() >= options.deadlineMs) {
+            break;
+        }
+        const sessionId = basename(path, ".jsonl");
+        let fromByte = row === undefined ? 0 : Number(row["byte_offset"]);
+        if (fromByte > stat.size) {
+            db.transaction(() => {
+                db.run("DELETE FROM sessions WHERE session_id = ?", sessionId);
+                db.run("DELETE FROM transcripts WHERE path = ?", path);
+            });
+            fromByte = 0;
+        }
+        const lastOrdinal = fromByte === 0
+            ? -1
+            : Number(db.get("SELECT max(t.ordinal) AS n FROM turns t JOIN sessions s ON s.id = t.session_id WHERE s.session_id = ?", sessionId)?.["n"] ?? -1);
+        const delta = await parseDelta(path, fromByte, lastOrdinal, options.root);
+        db.transaction(() => {
+            applyDelta(db, path, sessionId, delta, stat);
+        });
+    }
+    const count = (sql) => Number(db.get(sql)?.["n"] ?? 0);
+    return {
+        transcripts: count("SELECT COUNT(*) AS n FROM transcripts"),
+        sessions: count("SELECT COUNT(*) AS n FROM sessions"),
+        turns: count("SELECT COUNT(*) AS n FROM turns"),
+        files: count("SELECT COUNT(DISTINCT path) AS n FROM turn_files"),
+    };
+};
+//# sourceMappingURL=ingest.js.map

package/dist/recall/ingest/ingest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../../../src/recall/ingest/ingest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAChD,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACjE,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAClD,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAC;AACzD,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,wBAAwB,CAAC;AAwBlI,MAAM,YAAY,GAAG,IAAI,CAAC;AAE1B,MAAM,QAAQ,GAAG,CAAC,KAAc,EAAsB,EAAE,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;AAmBzG,MAAM,UAAU,GAAG,KAAK,EAAE,IAAY,EAAE,QAAgB,EAAE,WAAmB,EAAE,IAAY,EAAkB,EAAE;IAC3G,MAAM,KAAK,GAAU;QACjB,IAAI,EAAE,SAAS;QACf,OAAO,EAAE,SAAS;QAClB,SAAS,EAAE,SAAS;QACpB,KAAK,EAAE,SAAS;QAChB,KAAK,EAAE,SAAS;QAChB,KAAK,EAAE,SAAS;QAChB,UAAU,EAAE,QAAQ;QACpB,QAAQ,EAAE,EAAE;QACZ,OAAO,EAAE,IAAI,GAAG,EAAE;QAClB,SAAS,EAAE,IAAI,GAAG,EAAE;KACvB,CAAC;IACF,IAAI,OAAO,GAAG,WAAW,CAAC;IAC1B,IAAI,SAAS,GAAG,QAAQ,CAAC;IACzB,IAAI,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC;QAC9D,MAAM,SAAS,GAAG,SAAS,CAAC;QAC5B,SAAS,GAAG,OAAO,CAAC;QACpB,KAAK,CAAC,UAAU,GAAG,OAAO,CAAC;QAC3B,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,SAAS;QACb,CAAC;QACD,MAAM,EAAE,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;YACnB,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;YAC9C,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QAClD,CAAC;QACD,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QACtC,KAAK,CAAC,OAAO,KAAK,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;QAC5C,KAAK,CAAC,SAAS,KAAK,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACtB,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC;QACxB,CAAC;QACD,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAC1B,IAAI,KAAK,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS,IAAI,EAAE,KAAK,SAAS,EAAE,CAAC;YAChE,OAAO,IAAI,CAAC,CAAC;YAGb,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,qBAAqB,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;YAC5F,SAAS;QACb,CAAC;QACD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YACd,SAAS;QACb,CAAC;QACD,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC,CAAC;QAC9D,CAAC;QACD,KAAK,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5D,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YACpC,IAAI,GAAG,KAAK,EAAE,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACxD,SAAS;YACb,CAAC;YACD,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,GAAG,EAAiB,CAAC;YACtE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACnC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACtB,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,IAAI,KAAK,CAAC,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;YAClE,CAAC;iBAAM,CAAC;gBACJ,KAAK,CAAC,QAAQ,KAAK,QAAQ,CAAC;gBAC5B,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC;YACnD,CAAC;QACL,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,EAAY,EAAE,cAAsB,EAAE,SAAiB,EAAE,KAAY,EAAE,IAAuC,EAAQ,EAAE;IACxI,EAAE,CAAC,GAAG,CACF;;;;;;;+DAOuD,EACvD,SAAS,EACT,KAAK,CAAC,IAAI,IAAI,IAAI,EAClB,KAAK,CAAC,KAAK,IAAI,IAAI,EACnB,KAAK,CAAC,OAAO,IAAI,IAAI,EACrB,KAAK,CAAC,SAAS,IAAI,IAAI,EAEvB,KAAK,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,EAAE,EACzB,KAAK,CAAC,KAAK,IAAI,CAAC,CACnB,CAAC;IACF,MAAM,YAAY,GAAG,EAAE,CAAC,GAAG,CAAC,8CAA8C,EAAE,SAAS,CAAC,EAAE,CAAC,IAAI,CAAW,CAAC;IACzG,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;IACxE,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;QAChC,EAAE,CAAC,GAAG,CAEF;sDAC0C,EAC1C,YAAY,EACZ,IAAI,CAAC,IAAI,EACT,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,EAAE,EACP,IAAI,CAAC,MAAM,EACX,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EACvC,IAAI,CAAC,SAAS,CACjB,CAAC;IACN,CAAC;IACD,KAAK,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QAChD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5B,EAAE,CAAC,GAAG,CAAC,oEAAoE,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;QAClH,CAAC;IACL,CAAC;IACD,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAC5C,MAAM,MAAM,GAAG,EAAE,CAAC,GAAG,CAAC,2DAA2D,EAAE,YAAY,EAAE,OAAO,CAAC,EAAE,CAAC,IAAI,CAAuB,CAAC;QACxI,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACvB,SAAS;QACb,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;YACjC,EAAE,CAAC,GAAG,CACF;;;yEAGyD,EACzD,MAAM,EACN,IAAI,EACJ,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EACtB,KAAK,CAAC,MAAM,CACf,CAAC;QACN,CAAC;IACL,CAAC;IACD,EAAE,CAAC,GAAG,CACF;gIACwH,EACxH,cAAc,EACd,SAAS,EACT,KAAK,CAAC,UAAU,EAChB,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,IAAI,CACZ,CAAC;AACN,CAAC,CAAC;AAIF,MAAM,CAAC,MAAM,MAAM,GAAG,KAAK,EAAE,EAAY,EAAE,OAAmE,EAAwB,EAAE;IACpI,MAAM,MAAM,GAAG,IAAI,GAAG,EAA6C,CAAC;IAGpE,IAAI,OAAiB,CAAC;IACtB,IAAI,CAAC;QACD,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAC/C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACb,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;YACpB,MAAM,KAAK,CAAC;QAChB,CAAC;QACD,OAAO,GAAG,EAAE,CAAC;IACjB,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC1B,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC5B,SAAS;QACb,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC5B,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;YAChB,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7E,CAAC;IACL,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,uEAAuE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,MAAM,CAAW,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IAClJ,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,KAAK,EAAE,CAAC;QAC9B,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACnB,SAAS;QACb,CAAC;QACD,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAChB,EAAE,CAAC,GAAG,CAAC,2CAA2C,EAAE,GAAG,CAAC,YAAY,CAAW,CAAC,CAAC;YACjF,EAAE,CAAC,GAAG,CAAC,wCAAwC,EAAE,IAAI,CAAC,CAAC;QAC3D,CAAC,CAAC,CAAC;IACP,CAAC;IAED,MAAM,OAAO,GAAG,CAAC,GAAG,MAAM,CAAC;SACtB,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE;QACrB,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC5B,OAAO,GAAG,KAAK,SAAS,IAAI,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,KAAK,IAAI,CAAC,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC;IAC9G,CAAC,CAAC;SACD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;IACnD,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAE5B,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;YACvE,MAAM;QACV,CAAC;QACD,MAAM,SAAS,GAAG,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC3C,IAAI,QAAQ,GAAG,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC;QAClE,IAAI,QAAQ,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAEvB,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;gBAChB,EAAE,CAAC,GAAG,CAAC,2CAA2C,EAAE,SAAS,CAAC,CAAC;gBAC/D,EAAE,CAAC,GAAG,CAAC,wCAAwC,EAAE,IAAI,CAAC,CAAC;YAC3D,CAAC,CAAC,CAAC;YACH,QAAQ,GAAG,CAAC,CAAC;QACjB,CAAC;QACD,MAAM,WAAW,GACb,QAAQ,KAAK,CAAC;YACV,CAAC,CAAC,CAAC,CAAC;YACJ,CAAC,CAAC,MAAM,CACF,EAAE,CAAC,GAAG,CAAC,uGAAuG,EAAE,SAAS,CAAC,EAAE,CACxH,GAAG,CACN,IAAI,CAAC,CAAC,CACV,CAAC;QACZ,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAC1E,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YAChB,UAAU,CAAC,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;IACP,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IACvE,OAAO;QACH,WAAW,EAAE,KAAK,CAAC,uCAAuC,CAAC;QAC3D,QAAQ,EAAE,KAAK,CAAC,oCAAoC,CAAC;QACrD,KAAK,EAAE,KAAK,CAAC,iCAAiC,CAAC;QAC/C,KAAK,EAAE,KAAK,CAAC,kDAAkD,CAAC;KACnE,CAAC;AACN,CAAC,CAAC"}

package/dist/recall/rank/files.d.ts

@@ -0,0 +1,26 @@
+import type { SqliteDb } from "@intentic/base/sqlite";
+export declare const decayOf: (ts: number, now: number) => number;
+export declare const fileIdf: (db: SqliteDb, paths: readonly string[]) => Map<string, number>;
+export interface MatchingTurn {
+    readonly turnId: number;
+    readonly sessionRowId: number;
+    readonly ts: number;
+    readonly score: number;
+}
+export declare const TURN_BM25 = "-bm25(turns_fts, 1.0, 0.5)";
+export declare const matchingTurns: (db: SqliteDb, fts: string, sinceTs: number) => MatchingTurn[];
+export declare const matchingTitles: (db: SqliteDb, fts: string) => Map<number, number>;
+export interface TopicFile {
+    readonly path: string;
+    readonly score: number;
+    readonly sessions: number;
+    readonly lastTouched: number;
+    readonly sampleTitle: string | undefined;
+}
+export interface TopicOptions {
+    readonly days?: number;
+    readonly limit?: number;
+}
+export declare const rankFilesForTopic: (db: SqliteDb, query: string, options?: TopicOptions) => TopicFile[];
+export declare const sessionFiles: (db: SqliteDb, sessionRowId: number) => Set<string>;
+//# sourceMappingURL=files.d.ts.map

package/dist/recall/rank/files.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"files.d.ts","sourceRoot":"","sources":["../../../src/recall/rank/files.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAQtD,eAAO,MAAM,OAAO,OAAQ,MAAM,OAAO,MAAM,KAAG,MAAmE,CAAC;AAatH,eAAO,MAAM,OAAO,OAAQ,QAAQ,SAAS,SAAS,MAAM,EAAE,KAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAelF,CAAC;AAEF,MAAM,WAAW,YAAY;IACzB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CAC1B;AAGD,eAAO,MAAM,SAAS,+BAA+B,CAAC;AAGtD,eAAO,MAAM,aAAa,OAAQ,QAAQ,OAAO,MAAM,WAAW,MAAM,KAAG,YAAY,EASyD,CAAC;AAGjJ,eAAO,MAAM,cAAc,OAAQ,QAAQ,OAAO,MAAM,KAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAUxE,CAAC;AAEN,MAAM,WAAW,SAAS;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;CAC5C;AAED,MAAM,WAAW,YAAY;IACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;CAC3B;AAID,eAAO,MAAM,iBAAiB,OAAQ,QAAQ,SAAS,MAAM,YAAW,YAAY,KAAQ,SAAS,EA0DpG,CAAC;AAGF,eAAO,MAAM,YAAY,OAAQ,QAAQ,gBAAgB,MAAM,KAAG,GAAG,CAAC,MAAM,CAKvE,CAAC"}

package/dist/recall/rank/files.js

@@ -0,0 +1,88 @@
+import { ftsQueryOf } from "./terms.js";
+const HALF_LIFE_DAYS = 14;
+const DAY_MS = 24 * 60 * 60 * 1000;
+export const decayOf = (ts, now) => 2 ** (-Math.max(0, now - ts) / (HALF_LIFE_DAYS * DAY_MS));
+const chunked = (items, size) => {
+    const chunks = [];
+    for (let i = 0; i < items.length; i += size) {
+        chunks.push(items.slice(i, i + size));
+    }
+    return chunks;
+};
+const inList = (n) => Array.from({ length: n }, () => "?").join(", ");
+export const fileIdf = (db, paths) => {
+    const total = Number(db.get("SELECT COUNT(*) AS n FROM sessions")?.["n"] ?? 0);
+    const idf = new Map();
+    for (const chunk of chunked(paths, 500)) {
+        const rows = db.all(`SELECT tf.path AS path, COUNT(DISTINCT t.session_id) AS n
+             FROM turn_files tf JOIN turns t ON t.id = tf.turn_id
+             WHERE tf.path IN (${inList(chunk.length)}) GROUP BY tf.path`, ...chunk);
+        for (const row of rows) {
+            idf.set(row["path"], Math.log((total + 1) / (1 + Number(row["n"]))));
+        }
+    }
+    return idf;
+};
+export const TURN_BM25 = "-bm25(turns_fts, 1.0, 0.5)";
+export const matchingTurns = (db, fts, sinceTs) => db
+    .all(`SELECT t.id AS id, t.session_id AS session, t.ts AS ts, ${TURN_BM25} AS score
+             FROM turns_fts JOIN turns t ON t.id = turns_fts.rowid
+             WHERE turns_fts MATCH ? AND t.ts >= ?`, fts, sinceTs)
+    .map((row) => ({ turnId: Number(row["id"]), sessionRowId: Number(row["session"]), ts: Number(row["ts"]), score: Number(row["score"]) }));
+export const matchingTitles = (db, fts) => new Map(db
+    .all(`SELECT s.id AS id, -bm25(sessions_fts) AS score
+                 FROM sessions_fts JOIN sessions s ON s.id = sessions_fts.rowid
+                 WHERE sessions_fts MATCH ?`, fts)
+    .map((row) => [Number(row["id"]), Number(row["score"])]));
+export const rankFilesForTopic = (db, query, options = {}) => {
+    const fts = ftsQueryOf(query);
+    if (fts === undefined) {
+        return [];
+    }
+    const now = Date.now();
+    const sinceTs = now - (options.days ?? 90) * DAY_MS;
+    const turns = matchingTurns(db, fts, sinceTs);
+    if (turns.length === 0) {
+        return [];
+    }
+    const titles = matchingTitles(db, fts);
+    const byPath = new Map();
+    for (const chunk of chunked(turns, 500)) {
+        const byTurnId = new Map(chunk.map((turn) => [turn.turnId, turn]));
+        const rows = db.all(`SELECT turn_id, path, modified, last_ts FROM turn_files WHERE turn_id IN (${inList(chunk.length)})`, ...chunk.map((turn) => turn.turnId));
+        for (const row of rows) {
+            const turn = byTurnId.get(Number(row["turn_id"]));
+            const contribution = (turn.score + 0.5 * (titles.get(turn.sessionRowId) ?? 0)) * decayOf(turn.ts, now) * (Number(row["modified"]) === 1 ? 1.25 : 1);
+            const path = row["path"];
+            const accumulated = byPath.get(path) ?? { score: 0, sessions: new Set(), lastTouched: 0, bestContribution: 0, bestSession: 0 };
+            byPath.set(path, accumulated);
+            accumulated.score += contribution;
+            accumulated.sessions.add(turn.sessionRowId);
+            accumulated.lastTouched = Math.max(accumulated.lastTouched, Number(row["last_ts"]));
+            if (contribution > accumulated.bestContribution) {
+                accumulated.bestContribution = contribution;
+                accumulated.bestSession = turn.sessionRowId;
+            }
+        }
+    }
+    const idf = fileIdf(db, [...byPath.keys()]);
+    const titleOf = (sessionRowId) => {
+        const title = db.get("SELECT title FROM sessions WHERE id = ?", sessionRowId)?.["title"];
+        return typeof title === "string" ? title : undefined;
+    };
+    return [...byPath.entries()]
+        .map(([path, accumulated]) => ({
+        path,
+        score: accumulated.score * (idf.get(path) ?? 0),
+        sessions: accumulated.sessions.size,
+        lastTouched: accumulated.lastTouched,
+        sampleTitle: titleOf(accumulated.bestSession),
+    }))
+        .filter((file) => file.score > 0)
+        .toSorted((a, b) => b.score - a.score || a.path.localeCompare(b.path))
+        .slice(0, options.limit ?? 20);
+};
+export const sessionFiles = (db, sessionRowId) => new Set(db
+    .all("SELECT DISTINCT tf.path AS path FROM turn_files tf JOIN turns t ON t.id = tf.turn_id WHERE t.session_id = ?", sessionRowId)
+    .map((row) => row["path"]));
+//# sourceMappingURL=files.js.map

package/dist/recall/rank/files.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"files.js","sourceRoot":"","sources":["../../../src/recall/rank/files.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAGxC,MAAM,cAAc,GAAG,EAAE,CAAC;AAE1B,MAAM,MAAM,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAEnC,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,EAAU,EAAE,GAAW,EAAU,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,EAAE,CAAC,GAAG,CAAC,cAAc,GAAG,MAAM,CAAC,CAAC,CAAC;AAEtH,MAAM,OAAO,GAAG,CAAI,KAAmB,EAAE,IAAY,EAAS,EAAE;IAC5D,MAAM,MAAM,GAAU,EAAE,CAAC;IACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;QAC1C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAGtF,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,EAAY,EAAE,KAAwB,EAAuB,EAAE;IACnF,MAAM,KAAK,GAAG,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,oCAAoC,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/E,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtC,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,EAAE,CAAC,GAAG,CACf;;iCAEqB,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,oBAAoB,EAC7D,GAAG,KAAK,CACX,CAAC;QACF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACrB,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnF,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACf,CAAC,CAAC;AAUF,MAAM,CAAC,MAAM,SAAS,GAAG,4BAA4B,CAAC;AAGtD,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,EAAY,EAAE,GAAW,EAAE,OAAe,EAAkB,EAAE,CACxF,EAAE;KACG,GAAG,CACA,2DAA2D,SAAS;;mDAE7B,EACvC,GAAG,EACH,OAAO,CACV;KACA,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAGjJ,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,EAAY,EAAE,GAAW,EAAuB,EAAE,CAC7E,IAAI,GAAG,CACH,EAAE;KACG,GAAG,CACA;;4CAE4B,EAC5B,GAAG,CACN;KACA,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAC/D,CAAC;AAiBN,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,EAAY,EAAE,KAAa,EAAE,OAAO,GAAiB,EAAE,EAAe,EAAE;IACtG,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAC9B,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACpB,OAAO,EAAE,CAAC;IACd,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,OAAO,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC;IACpD,MAAM,KAAK,GAAG,aAAa,CAAC,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;IAC9C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrB,OAAO,EAAE,CAAC;IACd,CAAC;IACD,MAAM,MAAM,GAAG,cAAc,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAQvC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC9C,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;QACnE,MAAM,IAAI,GAAG,EAAE,CAAC,GAAG,CACf,6EAA6E,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,EACpG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CACtC,CAAC;QACF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAE,CAAC;YACnD,MAAM,YAAY,GACd,CAAC,IAAI,CAAC,KAAK,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACnI,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAW,CAAC;YACnC,MAAM,WAAW,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,GAAG,EAAU,EAAE,WAAW,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;YACvI,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;YAC9B,WAAW,CAAC,KAAK,IAAI,YAAY,CAAC;YAClC,WAAW,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC5C,WAAW,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;YACpF,IAAI,YAAY,GAAG,WAAW,CAAC,gBAAgB,EAAE,CAAC;gBAC9C,WAAW,CAAC,gBAAgB,GAAG,YAAY,CAAC;gBAC5C,WAAW,CAAC,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC;YAChD,CAAC;QACL,CAAC;IACL,CAAC;IACD,MAAM,GAAG,GAAG,OAAO,CAAC,EAAE,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAG,CAAC,YAAoB,EAAsB,EAAE;QACzD,MAAM,KAAK,GAAG,EAAE,CAAC,GAAG,CAAC,yCAAyC,EAAE,YAAY,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;QACzF,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;IACzD,CAAC,CAAC;IACF,OAAO,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;SACvB,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,WAAW,CAAC,EAAa,EAAE,CAAC,CAAC;QACtC,IAAI;QACJ,KAAK,EAAE,WAAW,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/C,QAAQ,EAAE,WAAW,CAAC,QAAQ,CAAC,IAAI;QACnC,WAAW,EAAE,WAAW,CAAC,WAAW;QACpC,WAAW,EAAE,OAAO,CAAC,WAAW,CAAC,WAAW,CAAC;KAChD,CAAC,CAAC;SACF,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;SAChC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;SACrE,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;AACvC,CAAC,CAAC;AAGF,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,EAAY,EAAE,YAAoB,EAAe,EAAE,CAC5E,IAAI,GAAG,CACH,EAAE;KACG,GAAG,CAAC,6GAA6G,EAAE,YAAY,CAAC;KAChI,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAW,CAAC,CAC3C,CAAC"}

package/dist/recall/rank/grab.d.ts

@@ -0,0 +1,25 @@
+import type { SqliteDb } from "@intentic/base/sqlite";
+export interface TurnExcerpt {
+    readonly sessionId: string;
+    readonly title: string | undefined;
+    readonly ts: number;
+    readonly ordinal: number;
+    readonly turnUuid: string;
+    readonly score: number;
+    readonly prompt: string;
+    readonly fragment: string;
+    readonly repeats: number;
+    readonly bookends: SessionBookends | undefined;
+}
+export interface SessionBookends {
+    readonly first: string;
+    readonly last: string;
+    readonly turns: number;
+}
+export interface GrabOptions {
+    readonly days?: number;
+    readonly limit?: number;
+    readonly excludeSessionId?: string;
+}
+export declare const grabExcerpts: (db: SqliteDb, query: string, options?: GrabOptions) => TurnExcerpt[];
+//# sourceMappingURL=grab.d.ts.map

package/dist/recall/rank/grab.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"grab.d.ts","sourceRoot":"","sources":["../../../src/recall/rank/grab.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAMtD,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IAExB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAE1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAEzB,QAAQ,CAAC,QAAQ,EAAE,eAAe,GAAG,SAAS,CAAC;CAClD;AAID,MAAM,WAAW,eAAe;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;CACtC;AAcD,eAAO,MAAM,YAAY,OAAQ,QAAQ,SAAS,MAAM,YAAW,WAAW,KAAQ,WAAW,EA8BhG,CAAC"}

package/dist/recall/rank/grab.js

@@ -0,0 +1,74 @@
+import { decayOf, TURN_BM25 } from "./files.js";
+import { ftsQueryOf } from "./terms.js";
+const DAY_MS = 24 * 60 * 60 * 1000;
+const BOOKEND_CHARS = 200;
+const repeatKey = (prompt) => prompt.toLowerCase().replaceAll(/\d+/g, "#").replaceAll(/\s+/g, " ").trim();
+export const grabExcerpts = (db, query, options = {}) => {
+    const fts = ftsQueryOf(query);
+    if (fts === undefined) {
+        return [];
+    }
+    const now = Date.now();
+    const sinceTs = now - (options.days ?? 90) * DAY_MS;
+    const ranked = db
+        .all(`SELECT s.session_id AS sid, s.title AS title, t.ts AS ts, t.ordinal AS ordinal, t.uuid AS uuid,
+                    t.prompt AS prompt, ${TURN_BM25} AS bm25, snippet(turns_fts, 1, '', '', '…', 48) AS fragment
+             FROM turns_fts JOIN turns t ON t.id = turns_fts.rowid JOIN sessions s ON s.id = t.session_id
+             WHERE turns_fts MATCH ? AND t.ts >= ?`, fts, sinceTs)
+        .filter((row) => row["sid"] !== options.excludeSessionId)
+        .map((row) => ({
+        sessionId: row["sid"],
+        title: typeof row["title"] === "string" ? row["title"] : undefined,
+        ts: Number(row["ts"]),
+        ordinal: Number(row["ordinal"]),
+        turnUuid: row["uuid"],
+        score: Number(row["bm25"]) * decayOf(Number(row["ts"]), now),
+        prompt: row["prompt"],
+        fragment: row["fragment"],
+    }))
+        .toSorted((a, b) => b.score - a.score || a.sessionId.localeCompare(b.sessionId) || a.ordinal - b.ordinal);
+    const collapsed = collapseRepeats(ranked).slice(0, options.limit ?? 10);
+    return withBookends(db, collapsed);
+};
+const collapseRepeats = (ranked) => {
+    const best = new Map();
+    for (const hit of ranked) {
+        const key = repeatKey(hit.prompt);
+        const seen = best.get(key);
+        if (seen === undefined) {
+            best.set(key, { ...hit, repeats: 0 });
+            continue;
+        }
+        seen.repeats += 1;
+    }
+    return [...best.values()];
+};
+const withBookends = (db, hits) => {
+    const ids = [...new Set(hits.map((hit) => hit.sessionId))];
+    if (ids.length === 0) {
+        return [];
+    }
+    const rows = db.all(`SELECT s.session_id AS sid,
+                (SELECT prompt FROM turns WHERE session_id = s.id ORDER BY ordinal ASC LIMIT 1) AS first_prompt,
+                (SELECT prompt FROM turns WHERE session_id = s.id ORDER BY ordinal DESC LIMIT 1) AS last_prompt,
+                (SELECT COUNT(*) FROM turns WHERE session_id = s.id) AS turns
+         FROM sessions s WHERE s.session_id IN (${ids.map(() => "?").join(",")})`, ...ids);
+    const bySession = new Map();
+    for (const row of rows) {
+        const first = row["first_prompt"];
+        const last = row["last_prompt"];
+        if (typeof first !== "string" || typeof last !== "string") {
+            continue;
+        }
+        bySession.set(row["sid"], {
+            first: first.slice(0, BOOKEND_CHARS),
+            last: last.slice(0, BOOKEND_CHARS),
+            turns: Number(row["turns"]),
+        });
+    }
+    return hits.map((hit) => {
+        const bookends = bySession.get(hit.sessionId);
+        return { ...hit, bookends: bookends === undefined || bookends.turns <= 1 ? undefined : bookends };
+    });
+};
+//# sourceMappingURL=grab.js.map

package/dist/recall/rank/grab.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"grab.js","sourceRoot":"","sources":["../../../src/recall/rank/grab.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,MAAM,MAAM,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAoCnC,MAAM,aAAa,GAAG,GAAG,CAAC;AAI1B,MAAM,SAAS,GAAG,CAAC,MAAc,EAAU,EAAE,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAI1H,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,EAAY,EAAE,KAAa,EAAE,OAAO,GAAgB,EAAE,EAAiB,EAAE;IAClG,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAC9B,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;QACpB,OAAO,EAAE,CAAC;IACd,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,OAAO,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC;IACpD,MAAM,MAAM,GAAG,EAAE;SACZ,GAAG,CACA;0CAC8B,SAAS;;mDAEA,EACvC,GAAG,EACH,OAAO,CACV;SACA,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,OAAO,CAAC,gBAAgB,CAAC;SACxD,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACX,SAAS,EAAE,GAAG,CAAC,KAAK,CAAW;QAC/B,KAAK,EAAE,OAAO,GAAG,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS;QAClE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACrB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC/B,QAAQ,EAAE,GAAG,CAAC,MAAM,CAAW;QAC/B,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,CAAC;QAC5D,MAAM,EAAE,GAAG,CAAC,QAAQ,CAAW;QAC/B,QAAQ,EAAE,GAAG,CAAC,UAAU,CAAW;KACtC,CAAC,CAAC;SACF,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;IAC9G,MAAM,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IACxE,OAAO,YAAY,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;AACvC,CAAC,CAAC;AAIF,MAAM,eAAe,GAAG,CAAC,MAA6B,EAAwC,EAAE;IAC5F,MAAM,IAAI,GAAG,IAAI,GAAG,EAA4C,CAAC;IACjE,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAClC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC3B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACrB,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,GAAG,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;YACtC,SAAS;QACb,CAAC;QACD,IAAI,CAAC,OAAO,IAAI,CAAC,CAAC;IACtB,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAC9B,CAAC,CAAC;AAIF,MAAM,YAAY,GAAG,CAAC,EAAY,EAAE,IAA8C,EAAiB,EAAE;IACjG,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IAC3D,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnB,OAAO,EAAE,CAAC;IACd,CAAC;IACD,MAAM,IAAI,GAAG,EAAE,CAAC,GAAG,CACf;;;;kDAI0C,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EACzE,GAAG,GAAG,CACT,CAAC;IACF,MAAM,SAAS,GAAG,IAAI,GAAG,EAA2B,CAAC;IACrD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,KAAK,GAAG,GAAG,CAAC,cAAc,CAAC,CAAC;QAClC,MAAM,IAAI,GAAG,GAAG,CAAC,aAAa,CAAC,CAAC;QAChC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YACxD,SAAS;QACb,CAAC;QACD,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAW,EAAE;YAChC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC;YACpC,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC;YAClC,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;SAC9B,CAAC,CAAC;IACP,CAAC;IAED,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACpB,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC9C,OAAO,EAAE,GAAG,GAAG,EAAE,QAAQ,EAAE,QAAQ,KAAK,SAAS,IAAI,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;IACtG,CAAC,CAAC,CAAC;AACP,CAAC,CAAC"}

package/dist/recall/rank/match.d.ts

@@ -0,0 +1,18 @@
+import type { SqliteDb } from "@intentic/base/sqlite";
+export interface SessionMatch {
+    readonly sessionId: string;
+    readonly title: string | undefined;
+    readonly lastTs: number;
+    readonly promptCount: number;
+    readonly score: number;
+    readonly bm25: number;
+    readonly coverage: number;
+    readonly strong: boolean;
+}
+export interface MatchOptions {
+    readonly days?: number;
+    readonly excludeSessionId?: string;
+    readonly files?: readonly string[];
+}
+export declare const matchSessions: (db: SqliteDb, prompt: string, options?: MatchOptions) => SessionMatch[];
+//# sourceMappingURL=match.d.ts.map

package/dist/recall/rank/match.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"match.d.ts","sourceRoot":"","sources":["../../../src/recall/rank/match.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AActD,MAAM,WAAW,YAAY;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,YAAY;IACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAEnC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACtC;AAyDD,eAAO,MAAM,aAAa,OAAQ,QAAQ,UAAU,MAAM,YAAW,YAAY,KAAQ,YAAY,EAuFpG,CAAC"}

package/dist/recall/rank/match.js

@@ -0,0 +1,122 @@
+import { decayOf, matchingTitles, matchingTurns, sessionFiles } from "./files.js";
+import { contentTermsOf, ftsMatchOf } from "./terms.js";
+const DAY_MS = 24 * 60 * 60 * 1000;
+const MIN_CONTENT_TERMS = 2;
+const STRONG_COVERAGE = 0.5;
+const coverageBySession = (db, terms, sinceTs) => {
+    const turnCount = Number(db.get("SELECT COUNT(*) AS n FROM turns WHERE ts >= ?", sinceTs)?.["n"] ?? 0);
+    const postings = terms.map((term) => db.all(`SELECT t.id AS id, t.session_id AS session
+             FROM turns_fts JOIN turns t ON t.id = turns_fts.rowid
+             WHERE turns_fts MATCH ? AND t.ts >= ?`, ftsMatchOf([term]), sinceTs));
+    const weights = postings.map((rows) => Math.log(1 + (turnCount - rows.length + 0.5) / (rows.length + 0.5)));
+    const total = weights.reduce((sum, weight) => sum + weight, 0);
+    const titleTerms = new Map();
+    const titleWeight = new Map();
+    terms.forEach((term, index) => {
+        for (const row of db.all("SELECT rowid AS id FROM sessions_fts WHERE sessions_fts MATCH ?", ftsMatchOf([term]))) {
+            const session = Number(row["id"]);
+            const seen = titleTerms.get(session) ?? new Set();
+            titleTerms.set(session, seen.add(index));
+            titleWeight.set(session, (titleWeight.get(session) ?? 0) + weights[index]);
+        }
+    });
+    const turnWeight = new Map();
+    postings.forEach((rows, index) => {
+        for (const row of rows) {
+            const session = Number(row["session"]);
+            if (titleTerms.get(session)?.has(index) === true) {
+                continue;
+            }
+            const turn = Number(row["id"]);
+            const entry = turnWeight.get(turn) ?? { session, weight: 0 };
+            entry.weight += weights[index];
+            turnWeight.set(turn, entry);
+        }
+    });
+    const bestTurn = new Map();
+    for (const { session, weight } of turnWeight.values()) {
+        bestTurn.set(session, Math.max(bestTurn.get(session) ?? 0, weight));
+    }
+    const coverage = new Map();
+    for (const session of new Set([...bestTurn.keys(), ...titleWeight.keys()])) {
+        coverage.set(session, ((bestTurn.get(session) ?? 0) + (titleWeight.get(session) ?? 0)) / total);
+    }
+    return coverage;
+};
+export const matchSessions = (db, prompt, options = {}) => {
+    const terms = contentTermsOf(prompt);
+    if (terms.length === 0) {
+        return [];
+    }
+    const fts = ftsMatchOf(terms);
+    const now = Date.now();
+    const sinceTs = now - (options.days ?? 45) * DAY_MS;
+    const titles = matchingTitles(db, fts);
+    const bestTurn = new Map();
+    for (const turn of matchingTurns(db, fts, sinceTs)) {
+        bestTurn.set(turn.sessionRowId, Math.max(bestTurn.get(turn.sessionRowId) ?? 0, turn.score));
+    }
+    const candidateIds = new Set([...bestTurn.keys(), ...titles.keys()]);
+    if (candidateIds.size === 0) {
+        return [];
+    }
+    const candidates = [];
+    for (const sessionRowId of candidateIds) {
+        const row = db.get(`SELECT s.session_id AS sid, s.title AS title, s.last_ts AS last_ts, COUNT(t.id) AS prompts
+             FROM sessions s LEFT JOIN turns t ON t.session_id = s.id
+             WHERE s.id = ? GROUP BY s.id`, sessionRowId);
+        if (row === undefined || Number(row["prompts"]) === 0 || Number(row["last_ts"]) < sinceTs) {
+            continue;
+        }
+        const sessionId = row["sid"];
+        if (sessionId === options.excludeSessionId) {
+            continue;
+        }
+        candidates.push({
+            sessionRowId,
+            sessionId,
+            title: typeof row["title"] === "string" ? row["title"] : undefined,
+            lastTs: Number(row["last_ts"]),
+            promptCount: Number(row["prompts"]),
+            bm25: (bestTurn.get(sessionRowId) ?? 0) + 0.5 * (titles.get(sessionRowId) ?? 0),
+        });
+    }
+    const maxBm25 = candidates.reduce((max, candidate) => (candidate.bm25 > max ? candidate.bm25 : max), 0);
+    if (maxBm25 === 0) {
+        return [];
+    }
+    const coverage = coverageBySession(db, terms, sinceTs);
+    const overlapTarget = options.files !== undefined && options.files.length > 0 ? new Set(options.files) : undefined;
+    return candidates
+        .map((candidate) => {
+        const normalized = candidate.bm25 / maxBm25;
+        const recency = decayOf(candidate.lastTs, now);
+        let score;
+        if (overlapTarget === undefined) {
+            score = 0.75 * normalized + 0.25 * recency;
+        }
+        else {
+            const touched = sessionFiles(db, candidate.sessionRowId);
+            let shared = 0;
+            for (const path of overlapTarget) {
+                if (touched.has(path)) {
+                    shared += 1;
+                }
+            }
+            score = 0.6 * normalized + 0.2 * recency + 0.2 * (shared / overlapTarget.size);
+        }
+        const covered = coverage.get(candidate.sessionRowId) ?? 0;
+        return {
+            sessionId: candidate.sessionId,
+            title: candidate.title,
+            lastTs: candidate.lastTs,
+            promptCount: candidate.promptCount,
+            score,
+            bm25: candidate.bm25,
+            coverage: covered,
+            strong: terms.length >= MIN_CONTENT_TERMS && covered >= STRONG_COVERAGE,
+        };
+    })
+        .toSorted((a, b) => b.score - a.score || a.sessionId.localeCompare(b.sessionId));
+};
+//# sourceMappingURL=match.js.map

package/dist/recall/rank/match.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"match.js","sourceRoot":"","sources":["../../../src/recall/rank/match.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAClF,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExD,MAAM,MAAM,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAOnC,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAC5B,MAAM,eAAe,GAAG,GAAG,CAAC;AAyB5B,MAAM,iBAAiB,GAAG,CAAC,EAAY,EAAE,KAAwB,EAAE,OAAe,EAAuB,EAAE;IACvG,MAAM,SAAS,GAAG,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,+CAA+C,EAAE,OAAO,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IACvG,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAChC,EAAE,CAAC,GAAG,CACF;;mDAEuC,EACvC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,EAClB,OAAO,CACV,CACJ,CAAC;IACF,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IAC5G,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,EAAE,CAAC,CAAC,CAAC;IAC/D,MAAM,UAAU,GAAG,IAAI,GAAG,EAAuB,CAAC;IAClD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC9C,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC1B,KAAK,MAAM,GAAG,IAAI,EAAE,CAAC,GAAG,CAAC,iEAAiE,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9G,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;YAClC,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,GAAG,EAAU,CAAC;YAC1D,UAAU,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YACzC,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,OAAO,CAAC,KAAK,CAAE,CAAC,CAAC;QAChF,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,IAAI,GAAG,EAA+C,CAAC;IAC1E,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;YACvC,IAAI,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC/C,SAAS;YACb,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;YAC/B,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;YAC7D,KAAK,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK,CAAE,CAAC;YAChC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAChC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,KAAK,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;QACpD,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,KAAK,MAAM,OAAO,IAAI,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,EAAE,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;QACzE,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC;IACpG,CAAC;IACD,OAAO,QAAQ,CAAC;AACpB,CAAC,CAAC;AAKF,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,EAAY,EAAE,MAAc,EAAE,OAAO,GAAiB,EAAE,EAAkB,EAAE;IACtG,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC;IACrC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrB,OAAO,EAAE,CAAC;IACd,CAAC;IACD,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;IAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,OAAO,GAAG,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC;IACpD,MAAM,MAAM,GAAG,cAAc,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACvC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC3C,KAAK,MAAM,IAAI,IAAI,aAAa,CAAC,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC;QACjD,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAChG,CAAC;IACD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,EAAE,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACrE,IAAI,YAAY,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,CAAC;IACd,CAAC;IASD,MAAM,UAAU,GAAgB,EAAE,CAAC;IACnC,KAAK,MAAM,YAAY,IAAI,YAAY,EAAE,CAAC;QACtC,MAAM,GAAG,GAAG,EAAE,CAAC,GAAG,CACd;;0CAE8B,EAC9B,YAAY,CACf,CAAC;QACF,IAAI,GAAG,KAAK,SAAS,IAAI,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,OAAO,EAAE,CAAC;YACxF,SAAS;QACb,CAAC;QACD,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,CAAW,CAAC;QACvC,IAAI,SAAS,KAAK,OAAO,CAAC,gBAAgB,EAAE,CAAC;YACzC,SAAS;QACb,CAAC;QACD,UAAU,CAAC,IAAI,CAAC;YACZ,YAAY;YACZ,SAAS;YACT,KAAK,EAAE,OAAO,GAAG,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS;YAClE,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAC9B,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACnC,IAAI,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;SAClF,CAAC,CAAC;IACP,CAAC;IAGD,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACxG,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;QAChB,OAAO,EAAE,CAAC;IACd,CAAC;IACD,MAAM,QAAQ,GAAG,iBAAiB,CAAC,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IACvD,MAAM,aAAa,GAAG,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACnH,OAAO,UAAU;SACZ,GAAG,CAAC,CAAC,SAAS,EAAgB,EAAE;QAC7B,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,GAAG,OAAO,CAAC;QAC5C,MAAM,OAAO,GAAG,OAAO,CAAC,SAAS,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC/C,IAAI,KAAa,CAAC;QAClB,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;YAC9B,KAAK,GAAG,IAAI,GAAG,UAAU,GAAG,IAAI,GAAG,OAAO,CAAC;QAC/C,CAAC;aAAM,CAAC;YACJ,MAAM,OAAO,GAAG,YAAY,CAAC,EAAE,EAAE,SAAS,CAAC,YAAY,CAAC,CAAC;YACzD,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;gBAC/B,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;oBACpB,MAAM,IAAI,CAAC,CAAC;gBAChB,CAAC;YACL,CAAC;YACD,KAAK,GAAG,GAAG,GAAG,UAAU,GAAG,GAAG,GAAG,OAAO,GAAG,GAAG,GAAG,CAAC,MAAM,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;QACnF,CAAC;QACD,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAC1D,OAAO;YACH,SAAS,EAAE,SAAS,CAAC,SAAS;YAC9B,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,MAAM,EAAE,SAAS,CAAC,MAAM;YACxB,WAAW,EAAE,SAAS,CAAC,WAAW;YAClC,KAAK;YACL,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,QAAQ,EAAE,OAAO;YACjB,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,iBAAiB,IAAI,OAAO,IAAI,eAAe;SAC1E,CAAC;IACN,CAAC,CAAC;SACD,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;AACzF,CAAC,CAAC"}

package/dist/recall/rank/terms.d.ts

@@ -0,0 +1,4 @@
+export declare const contentTermsOf: (text: string) => string[];
+export declare const ftsMatchOf: (terms: readonly string[]) => string;
+export declare const ftsQueryOf: (text: string) => string | undefined;
+//# sourceMappingURL=terms.d.ts.map

package/dist/recall/rank/terms.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"terms.d.ts","sourceRoot":"","sources":["../../../src/recall/rank/terms.ts"],"names":[],"mappings":"AAwBA,eAAO,MAAM,cAAc,SAAU,MAAM,KAAG,MAAM,EAUnD,CAAC;AAIF,eAAO,MAAM,UAAU,UAAW,SAAS,MAAM,EAAE,KAAG,MAAuD,CAAC;AAG9G,eAAO,MAAM,UAAU,SAAU,MAAM,KAAG,MAAM,GAAG,SAGlD,CAAC"}

package/dist/recall/rank/terms.js

@@ -0,0 +1,25 @@
+import { STOPWORDS } from "@intentic/base/stopwords";
+import { stripInjectedPreamble } from "@intentic/constants";
+const PROMPT_FILLER = new Set([
+    "please thanks thank hi hey ok okay yes no sure",
+    "can could would should will let lets me my our us",
+    "just also now then again so if all some there here where",
+    "go ahead continue proceed next left off done finish finished work",
+    "fix make better help see want need get try did",
+    "don didn doesn isn ve re ll",
+]
+    .join(" ")
+    .split(" "));
+export const contentTermsOf = (text) => [
+    ...new Set((stripInjectedPreamble(text)
+        .normalize("NFD")
+        .replaceAll(/\p{M}/gu, "")
+        .toLowerCase()
+        .match(/[\p{L}\p{N}_$]+/gu) ?? []).filter((term) => term.length > 1 && !STOPWORDS.has(term) && !PROMPT_FILLER.has(term))),
+];
+export const ftsMatchOf = (terms) => terms.map((term) => `"${term}"`).join(" OR ");
+export const ftsQueryOf = (text) => {
+    const terms = contentTermsOf(text);
+    return terms.length === 0 ? undefined : ftsMatchOf(terms);
+};
+//# sourceMappingURL=terms.js.map