@coffer-org/server 1.13.0 → 2.0.1

package/dist/search-index.js

@@ -0,0 +1,100 @@
+import { textSearchKeys, titleKey } from '@coffer-org/sdk/shelf';
+import { foldText } from '@coffer-org/core/search';
+import { getLogger } from '@coffer-org/sdk/logger';
+import { getEm } from "./db.js";
+import { getDialect } from "./dialect.js";
+import { flattenEmbedded } from "./collection-io.js";
+import { encodeJson } from "./mutate.js";
+const log = getLogger('search-index');
+export const SEARCH_SEPARATOR = '\n';
+export const MIN_FTS_QUERY_LENGTH = 3;
+let ftsUnsupported = false;
+export function ftsAvailable() {
+    if (ftsUnsupported)
+        return false;
+    try {
+        return getDialect() === 'sqlite';
+    }
+    catch {
+        return false;
+    }
+}
+export function foldedTextFor(m, record) {
+    const row = encodeJson(m, flattenEmbedded(m, record));
+    const keys = [...new Set([...textSearchKeys(m), titleKey(m)])].filter(Boolean);
+    return keys
+        .map((k) => String(row[k] ?? ''))
+        .filter((s) => s.length > 0)
+        .map(foldText)
+        .join(SEARCH_SEPARATOR);
+}
+export function buildFtsMatchQuery(tokens) {
+    return tokens.map((t) => `"${t.replace(/"/g, '""')}"`).join(' AND ');
+}
+export async function ensureSearchTable() {
+    if (!ftsAvailable())
+        return;
+    try {
+        await getEm()
+            .fork()
+            .getConnection()
+            .execute(`CREATE VIRTUAL TABLE IF NOT EXISTS "_search" USING fts5(
+           shelf UNINDEXED,
+           record_id UNINDEXED,
+           folded,
+           tokenize='trigram'
+         )`);
+    }
+    catch (e) {
+        ftsUnsupported = true;
+        log.warn(`FTS5 index unavailable, global search falls back to the scan — ${e.message}`);
+    }
+}
+export async function upsertSearchRow(shelf, recordId, folded) {
+    if (!ftsAvailable())
+        return;
+    const conn = getEm().fork().getConnection();
+    await conn.execute('DELETE FROM "_search" WHERE shelf = ? AND record_id = ?', [shelf, recordId]);
+    if (!folded)
+        return;
+    await conn.execute('INSERT INTO "_search" (shelf, record_id, folded) VALUES (?, ?, ?)', [
+        shelf,
+        recordId,
+        folded,
+    ]);
+}
+export async function deleteSearchRow(shelf, recordId) {
+    if (!ftsAvailable())
+        return;
+    await getEm()
+        .fork()
+        .getConnection()
+        .execute('DELETE FROM "_search" WHERE shelf = ? AND record_id = ?', [shelf, recordId]);
+}
+export async function ftsCandidates(tokens) {
+    const out = new Map();
+    if (!ftsAvailable() || tokens.length === 0)
+        return null;
+    if (tokens.some((t) => t.length < MIN_FTS_QUERY_LENGTH))
+        return null;
+    try {
+        const rows = (await getEm()
+            .fork()
+            .getConnection()
+            .execute('SELECT shelf, record_id FROM "_search" WHERE "_search" MATCH ?', [
+            buildFtsMatchQuery(tokens),
+        ]));
+        for (const r of rows) {
+            const list = out.get(r.shelf);
+            if (list)
+                list.push(Number(r.record_id));
+            else
+                out.set(r.shelf, [Number(r.record_id)]);
+        }
+    }
+    catch (e) {
+        log.warn(`MATCH failed — ${e.message}`);
+        return null;
+    }
+    return out;
+}

package/dist/search-indexer.d.ts

@@ -0,0 +1,4 @@
+export declare const SEARCH_STATE_PLUGIN = "core";
+export declare const SEARCH_CURSOR_KEY = "fts_last_event_id";
+export declare function indexSearchOnce(batch?: number): Promise<number>;
+export declare function startSearchIndexer(batch?: number): () => void;

package/dist/search-indexer.js

@@ -0,0 +1,88 @@
+import { getLogger } from '@coffer-org/sdk/logger';
+import { listEventsSince } from "./embeddings.js";
+import { getPluginState, setPluginState } from "./plugin-state.js";
+import { getShelf } from "./registry-context.js";
+import { onRecordsChanged } from "./index-signal.js";
+import { ftsAvailable, foldedTextFor, upsertSearchRow, deleteSearchRow } from "./search-index.js";
+const log = getLogger('search-indexer');
+export const SEARCH_STATE_PLUGIN = 'core';
+export const SEARCH_CURSOR_KEY = 'fts_last_event_id';
+const DEFAULT_BATCH = 5000;
+export async function indexSearchOnce(batch = DEFAULT_BATCH) {
+    if (!ftsAvailable())
+        return 0;
+    const last = Number((await getPluginState(SEARCH_STATE_PLUGIN, SEARCH_CURSOR_KEY)) ?? '0');
+    const rows = await listEventsSince(last, batch);
+    if (rows.length === 0)
+        return 0;
+    const byRef = new Map();
+    for (const r of rows) {
+        if (r.record_id == null)
+            continue;
+        const [library, shelf] = r.type.split('/');
+        if (!library || !shelf)
+            continue;
+        const key = `${r.type}/${r.record_id}`;
+        if (r.op === 'delete') {
+            byRef.set(key, { shelf: r.type, recordId: r.record_id, op: 'delete', folded: '' });
+            continue;
+        }
+        if (!r.after)
+            continue;
+        const m = getShelf(library, shelf);
+        if (!m) {
+            byRef.delete(key);
+            continue;
+        }
+        let after;
+        try {
+            after = JSON.parse(r.after);
+        }
+        catch (e) {
+            log.warn(`event ${r.id} (${r.type}): malformed after-snapshot, skipping — ${e.message}`);
+            byRef.delete(key);
+            continue;
+        }
+        byRef.set(key, { shelf: r.type, recordId: r.record_id, op: 'upsert', folded: foldedTextFor(m, after) });
+    }
+    for (const p of byRef.values()) {
+        if (p.op === 'delete')
+            await deleteSearchRow(p.shelf, p.recordId);
+        else
+            await upsertSearchRow(p.shelf, p.recordId, p.folded);
+    }
+    const lastId = rows[rows.length - 1].id;
+    await setPluginState(SEARCH_STATE_PLUGIN, SEARCH_CURSOR_KEY, String(lastId));
+    log.debug(`indexed ${byRef.size} record(s) from ${rows.length} event(s), cursor → ${lastId}`);
+    return rows.length;
+}
+export function startSearchIndexer(batch = DEFAULT_BATCH) {
+    if (!ftsAvailable())
+        return () => { };
+    let running = false;
+    let again = false;
+    const run = async () => {
+        if (running) {
+            again = true;
+            return;
+        }
+        running = true;
+        try {
+            do {
+                again = false;
+                let processed;
+                do {
+                    processed = await indexSearchOnce(batch);
+                } while (processed === batch);
+            } while (again);
+        }
+        catch (e) {
+            log.warn(`pass failed — ${e.message}`);
+        }
+        finally {
+            running = false;
+        }
+    };
+    void run();
+    return onRecordsChanged(() => void run());
+}

package/dist/thread-store.d.ts

@@ -1,20 +1,28 @@
 export interface StoredMsg {
     msgId: string;
-    role: 'user' | 'assistant';
+    role: 'user' | 'assistant' | 'reasoning';
     sender: string | null;
     text: string;
     ts: number;
     replyToId: string | null;
 }
+export interface ThreadChat {
+    chatId: string;
+    lastTs: number;
+    count: number;
+    firstUserText: string;
+}
 export declare function getThreadMessage(connector: string, chatId: string, msgId: string): Promise<StoredMsg | null>;
 export declare function putThreadMessage(m: {
     connector: string;
     chatId: string;
     msgId: string;
-    role: 'user' | 'assistant';
+    role: 'user' | 'assistant' | 'reasoning';
     sender?: string | null;
     text: string;
     ts: number;
     replyToId: string | null;
 }): Promise<void>;
 export declare function pruneThreadMessages(connector: string, cutoffTs: number): Promise<void>;
+export declare function listThreadMessages(connector: string, chatId: string, limit?: number): Promise<StoredMsg[]>;
+export declare function listThreadChats(connector: string, chatIdPrefix: string): Promise<ThreadChat[]>;

package/dist/thread-store.js

@@ -25,3 +25,43 @@ export async function pruneThreadMessages(connector, cutoffTs) {
     const em = getEm().fork();
     await em.nativeDelete('_ThreadMessage', { connector, ts: { $lt: cutoffTs } });
 }
+export async function listThreadMessages(connector, chatId, limit = 200) {
+    const em = getEm().fork();
+    const visible = (await em.find('_ThreadMessage', { connector, chat_id: chatId, role: { $ne: 'reasoning' } }, { orderBy: { ts: 'desc', msg_id: 'desc' }, limit }));
+    const oldest = visible.at(-1)?.ts;
+    const reasoning = oldest === undefined ? [] : (await em.find('_ThreadMessage', { connector, chat_id: chatId, role: 'reasoning', ts: { $gte: oldest - 1 } }, { orderBy: { ts: 'desc', msg_id: 'desc' } }));
+    return [...visible, ...reasoning]
+        .sort((a, b) => b.ts - a.ts || (a.msg_id < b.msg_id ? 1 : a.msg_id > b.msg_id ? -1 : 0))
+        .reverse()
+        .map(toStored);
+}
+export async function listThreadChats(connector, chatIdPrefix) {
+    const em = getEm().fork();
+    if (!chatIdPrefix) {
+        return [];
+    }
+    const last = chatIdPrefix.charCodeAt(chatIdPrefix.length - 1);
+    const upper = chatIdPrefix.slice(0, -1) + String.fromCharCode(last + 1);
+    const rows = (await em.find('_ThreadMessage', { connector, chat_id: { $gte: chatIdPrefix, $lt: upper } }, { orderBy: { ts: 'asc' } }));
+    const byChat = new Map();
+    for (const r of rows) {
+        if (r.role === 'reasoning')
+            continue;
+        const cur = byChat.get(r.chat_id);
+        if (!cur) {
+            byChat.set(r.chat_id, {
+                chatId: r.chat_id,
+                lastTs: r.ts,
+                count: 1,
+                firstUserText: r.role === 'user' ? r.text : '',
+            });
+            continue;
+        }
+        cur.count += 1;
+        if (r.ts > cur.lastTs)
+            cur.lastTs = r.ts;
+        if (!cur.firstUserText && r.role === 'user')
+            cur.firstUserText = r.text;
+    }
+    return [...byChat.values()].sort((a, b) => b.lastTs - a.lastTs);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@coffer-org/server",
-  "version": "1.13.0",
+  "version": "2.0.1",
   "type": "module",
   "engines": {
     "node": ">=24"
@@ -24,8 +24,8 @@
     "postpack": "node ../../scripts/swap-exports.mjs src"
   },
   "dependencies": {
-    "@coffer-org/core": "^1.4.0",
-    "@coffer-org/sdk": "^1.7.0",
+    "@coffer-org/core": "^2.0.0",
+    "@coffer-org/sdk": "^2.0.0",
     "@extractus/oembed-extractor": "^4.1.0",
     "@fastify/cors": "^11.2.0",
     "@fastify/multipart": "^10.0.0",