@coffer-org/server 1.14.0 → 2.1.0

package/dist/counts.d.ts

@@ -0,0 +1,15 @@
+export interface CountTarget {
+    key: string;
+    table: string;
+}
+export interface ShelfRef {
+    library: string;
+    shelf: string;
+    standalone?: boolean;
+}
+export declare function countTargetsFor(shelves: ShelfRef[], filter?: {
+    library?: string;
+    shelf?: string;
+}): CountTarget[];
+export declare function buildCountsSql(shelves: CountTarget[]): string[];
+export declare function recordCounts(shelves: CountTarget[]): Promise<Record<string, number>>;

package/dist/counts.js

@@ -0,0 +1,39 @@
+import { getEm } from "./db.js";
+import { shelfTableName } from "./entity-schema.js";
+const CHUNK = 500;
+function quoteIdent(name) {
+    return `"${name.replace(/"/g, '""')}"`;
+}
+function quoteLiteral(value) {
+    return `'${value.replace(/'/g, "''")}'`;
+}
+export function countTargetsFor(shelves, filter = {}) {
+    return shelves
+        .filter((s) => s.standalone !== false)
+        .filter((s) => (filter.library ? s.library === filter.library : true))
+        .filter((s) => (filter.shelf ? s.shelf === filter.shelf : true))
+        .map((s) => ({ key: `${s.library}/${s.shelf}`, table: shelfTableName(s.library, s.shelf) }));
+}
+export function buildCountsSql(shelves) {
+    const out = [];
+    for (let i = 0; i < shelves.length; i += CHUNK) {
+        const branches = shelves
+            .slice(i, i + CHUNK)
+            .map((s) => `SELECT ${quoteLiteral(s.key)} AS k, COUNT(*) AS n FROM ${quoteIdent(s.table)}`);
+        out.push(branches.join(' UNION ALL '));
+    }
+    return out;
+}
+export async function recordCounts(shelves) {
+    const statements = buildCountsSql(shelves);
+    if (!statements.length)
+        return {};
+    const conn = getEm().fork().getConnection();
+    const out = {};
+    for (const sql of statements) {
+        const rows = (await conn.execute(sql));
+        for (const r of rows)
+            out[r.k] = Number(r.n);
+    }
+    return out;
+}

package/dist/db.js

@@ -2,6 +2,7 @@ import { mkdirSync } from 'node:fs';
 import { dirname } from 'node:path';
 import { MikroORM, EntityCaseNamingStrategy } from '@mikro-orm/core';
 import { SqliteDriver } from '@mikro-orm/sqlite';
+const SKIP_TABLES = [/^_search(_|$)/];
 let _orm;
 export function getOrm() {
     if (!_orm)
@@ -25,6 +26,7 @@ export async function initDb(entities) {
         entities,
         namingStrategy: EntityCaseNamingStrategy,
         discovery: { warnWhenNoEntities: false },
+        schemaGenerator: { skipTables: SKIP_TABLES },
     };
     if (driverName === 'postgresql') {
         const { PostgreSqlDriver } = await import('@mikro-orm/postgresql');

package/dist/embeddings.d.ts

@@ -1,11 +1,11 @@
 export type EmbeddingHit = {
-    type: string;
+    shelfKey: string;
     recordId: number;
     snippet: string;
     distance: number;
 };
 export type EmbeddingRow = {
-    type: string;
+    shelfKey: string;
     recordId: number;
     snippet: string;
     full: Float32Array;
@@ -27,12 +27,12 @@ export declare function twoPassSearch(rows: EmbeddingRow[], queryVec: ArrayLike<
     poolMinFactor?: number;
 }): EmbeddingHit[];
 export declare function upsertEmbedding(args: {
-    type: string;
+    shelfKey: string;
     recordId: number;
     snippet: string;
     vector: number[];
     model: string;
 }): Promise<void>;
-export declare function deleteEmbedding(type: string, recordId: number): Promise<void>;
+export declare function deleteEmbedding(shelfKey: string, recordId: number): Promise<void>;
 export declare function searchEmbeddings(queryVec: number[], k: number): Promise<EmbeddingHit[]>;
 export declare function listEventsSince(lastId: number, limit: number): Promise<EventRow[]>;

package/dist/embeddings.js

@@ -12,7 +12,7 @@ async function loadCache() {
     const em = getEm().fork();
     const rows = (await em.find('_Embedding', {}));
     cache = rows.map((r) => ({
-        type: r.type, recordId: r.record_id, snippet: r.snippet, full: decodeVector(r.vector),
+        shelfKey: r.shelf_key, recordId: r.record_id, snippet: r.snippet, full: decodeVector(r.vector),
     }));
     return cache;
 }
@@ -39,7 +39,7 @@ export async function migrateEmbeddingVectorsToBlob() {
         if (typeof r.vector !== 'string')
             continue;
         const f = decodeBase64Vector(r.vector);
-        await em.nativeUpdate('_Embedding', { type: r.type, record_id: r.record_id }, { vector: Buffer.from(f.buffer, f.byteOffset, f.byteLength) });
+        await em.nativeUpdate('_Embedding', { shelf_key: r.shelf_key, record_id: r.record_id }, { vector: Buffer.from(f.buffer, f.byteOffset, f.byteLength) });
         migrated++;
     }
     if (migrated > 0) {
@@ -75,7 +75,7 @@ export function twoPassSearch(rows, queryVec, k, opts = {}) {
     return pool
         .map(({ i }) => {
         const r = rows[i];
-        return { type: r.type, recordId: r.recordId, snippet: r.snippet, distance: 1 - cosine(r.full, qFull) };
+        return { shelfKey: r.shelfKey, recordId: r.recordId, snippet: r.snippet, distance: 1 - cosine(r.full, qFull) };
     })
         .sort((a, b) => a.distance - b.distance)
         .slice(0, k);
@@ -83,7 +83,7 @@ export function twoPassSearch(rows, queryVec, k, opts = {}) {
 export async function upsertEmbedding(args) {
     const em = getEm().fork();
     await em.upsert('_Embedding', {
-        type: args.type,
+        shelf_key: args.shelfKey,
         record_id: args.recordId,
         snippet: args.snippet,
         vector: encodeVector(args.vector),
@@ -94,9 +94,9 @@ export async function upsertEmbedding(args) {
     await em.flush();
     invalidateEmbeddingCache();
 }
-export async function deleteEmbedding(type, recordId) {
+export async function deleteEmbedding(shelfKey, recordId) {
     const em = getEm().fork();
-    await em.nativeDelete('_Embedding', { type, record_id: recordId });
+    await em.nativeDelete('_Embedding', { shelf_key: shelfKey, record_id: recordId });
     invalidateEmbeddingCache();
 }
 export async function searchEmbeddings(queryVec, k) {

package/dist/entity-schema.js

@@ -133,7 +133,7 @@ export const EmbeddingSchema = new EntitySchema({
     name: '_Embedding',
     tableName: '_embeddings',
     properties: {
-        type: { type: 'text', primary: true },
+        shelf_key: { type: 'text', primary: true },
         record_id: { type: 'integer', primary: true },
         snippet: { type: 'text' },
         vector: { type: 'blob' },

package/dist/file-fields.d.ts

@@ -0,0 +1,6 @@
+import type { ShelfDef } from '@coffer-org/sdk/shelf';
+import type { ValidationIssue } from './mutate.ts';
+export declare function mimeForName(name: string): string | undefined;
+export declare function touchesFileFields(m: ShelfDef, input: unknown): boolean;
+export declare function dropUnchangedFileFields(m: ShelfDef, input: Record<string, unknown>, stored: Record<string, unknown>): Record<string, unknown>;
+export declare function normalizeFileFields(m: ShelfDef, data: Record<string, unknown>): ValidationIssue[];

package/dist/file-fields.js

@@ -0,0 +1,121 @@
+import { statSync } from 'node:fs';
+import { join } from 'node:path';
+import { fieldEntries } from '@coffer-org/sdk/shelf';
+import { jsonValue } from '@coffer-org/sdk/fields';
+import { uploadsDir } from "./uploads.js";
+const MIME_BY_EXT = {
+    jpg: 'image/jpeg',
+    jpeg: 'image/jpeg',
+    png: 'image/png',
+    gif: 'image/gif',
+    webp: 'image/webp',
+    avif: 'image/avif',
+    svg: 'image/svg+xml',
+    heic: 'image/heic',
+    bmp: 'image/bmp',
+    ico: 'image/x-icon',
+    mp4: 'video/mp4',
+    webm: 'video/webm',
+    mov: 'video/quicktime',
+    mkv: 'video/x-matroska',
+    mp3: 'audio/mpeg',
+    m4a: 'audio/mp4',
+    ogg: 'audio/ogg',
+    opus: 'audio/opus',
+    wav: 'audio/wav',
+    flac: 'audio/flac',
+    pdf: 'application/pdf',
+    txt: 'text/plain',
+    csv: 'text/csv',
+    json: 'application/json',
+    zip: 'application/zip',
+    doc: 'application/msword',
+    docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+    xls: 'application/vnd.ms-excel',
+    xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+};
+export function mimeForName(name) {
+    const i = name.lastIndexOf('.');
+    if (i < 0)
+        return undefined;
+    return MIME_BY_EXT[name.slice(i + 1).toLowerCase()];
+}
+function resolveEntry(entry) {
+    let size;
+    try {
+        const st = statSync(join(uploadsDir(), entry.name));
+        if (!st.isFile())
+            return null;
+        size = st.size;
+    }
+    catch {
+        return null;
+    }
+    const mime = mimeForName(entry.name);
+    return mime ? { name: entry.name, mime, size } : { name: entry.name, size };
+}
+function sameValue(a, b) {
+    if (a === undefined || b === undefined)
+        return false;
+    try {
+        return JSON.stringify(jsonValue(a)) === JSON.stringify(jsonValue(b));
+    }
+    catch {
+        return false;
+    }
+}
+export function touchesFileFields(m, input) {
+    if (typeof input !== 'object' || input === null)
+        return false;
+    const o = input;
+    for (const [key, f] of fieldEntries(m.fields))
+        if (f.prim === 'file' && key in o)
+            return true;
+    return false;
+}
+export function dropUnchangedFileFields(m, input, stored) {
+    let out = null;
+    for (const [key, f] of fieldEntries(m.fields)) {
+        if (f.prim !== 'file')
+            continue;
+        if (!(key in input))
+            continue;
+        if (!sameValue(input[key], stored[key]))
+            continue;
+        out ??= { ...input };
+        delete out[key];
+    }
+    return out ?? input;
+}
+export function normalizeFileFields(m, data) {
+    const issues = [];
+    for (const [key, f] of fieldEntries(m.fields)) {
+        if (f.prim !== 'file')
+            continue;
+        if (!(key in data))
+            continue;
+        const raw = data[key];
+        if (raw == null || raw === '')
+            continue;
+        const parsed = jsonValue(raw);
+        const isArray = Array.isArray(parsed);
+        const items = (isArray ? parsed : [parsed]);
+        const out = [];
+        for (const it of items) {
+            if (typeof it !== 'object' || it === null || typeof it.name !== 'string') {
+                issues.push({ field: key, code: 'file_not_uploaded', path: [key] });
+                continue;
+            }
+            const resolved = resolveEntry(it);
+            if (!resolved) {
+                issues.push({ field: key, code: 'file_not_uploaded', params: { name: it.name }, path: [key] });
+                continue;
+            }
+            out.push(resolved);
+        }
+        if (issues.length)
+            continue;
+        data[key] = isArray ? out : out[0];
+    }
+    return issues;
+}

package/dist/frontend-agent.d.ts

@@ -0,0 +1 @@
+export declare function frontendInstructions(): Promise<string | null>;

package/dist/frontend-agent.js

@@ -0,0 +1,37 @@
+var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
+    if (typeof path === "string" && /^\.\.?\//.test(path)) {
+        return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
+            return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
+        });
+    }
+    return path;
+};
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { getLogger } from '@coffer-org/sdk/logger';
+import { configuredPublicUrl } from "./public-url.js";
+const log = getLogger('frontend-agent');
+export async function frontendInstructions() {
+    const dist = process.env['WEB_DIST'];
+    if (!dist)
+        return null;
+    try {
+        const root = join(dist, '..');
+        const pkg = JSON.parse(await readFile(join(root, 'package.json'), 'utf8'));
+        const rel = pkg.coffer?.agent;
+        if (!rel)
+            return null;
+        const mod = (await import(__rewriteRelativeImportExtension(pathToFileURL(join(root, rel)).href)));
+        if (typeof mod.agent !== 'function') {
+            log.warn(`frontend ${rel} exports no agent() — no site-link instructions`);
+            return null;
+        }
+        const text = await mod.agent({ siteUrl: await configuredPublicUrl() });
+        return typeof text === 'string' && text.trim() ? text : null;
+    }
+    catch (e) {
+        log.warn(`frontend agent instructions skipped — ${e.message}`);
+        return null;
+    }
+}

package/dist/global-search.d.ts

@@ -0,0 +1,16 @@
+import type { ShelfDef } from '@coffer-org/sdk/shelf';
+export declare const SEARCH_SCAN_LIMIT = 1000;
+export interface SearchShelf {
+    key: string;
+    table: string;
+    def: ShelfDef;
+}
+export interface GlobalSearchHit {
+    library: string;
+    shelf: string;
+    id: string;
+    label: string;
+    snippet: string;
+}
+export declare function globalSearchByScan(shelves: SearchShelf[], q: string, limit: number): Promise<GlobalSearchHit[]>;
+export declare function globalSearch(shelves: SearchShelf[], q: string, limit: number): Promise<GlobalSearchHit[]>;

package/dist/global-search.js

@@ -0,0 +1,87 @@
+import { textSearchKeys, titleKey, recordTitle, storageColumnsFor } from '@coffer-org/sdk/shelf';
+import { serialize } from '@mikro-orm/core';
+import { tokenize } from '@coffer-org/core/search';
+import { getLogger } from '@coffer-org/sdk/logger';
+import { getEm } from "./db.js";
+import { rowMatch } from "./records-api.js";
+import { ftsAvailable, ftsCandidates, MIN_FTS_QUERY_LENGTH } from "./search-index.js";
+const log = getLogger('search');
+export const SEARCH_SCAN_LIMIT = 1000;
+function matchCols(m) {
+    return storageColumnsFor(m, [...new Set([...textSearchKeys(m), titleKey(m)])].filter(Boolean));
+}
+function tieKey(hit) {
+    return `${hit.library}/${hit.shelf}/${hit.id}`;
+}
+function scoreRows(shelf, rows, tokens) {
+    const [library, shelfName] = shelf.key.split('/');
+    const out = [];
+    for (const row of rows) {
+        const m = rowMatch(shelf.def, row, tokens);
+        if (!m)
+            continue;
+        out.push({
+            score: m.score,
+            result: {
+                library,
+                shelf: shelfName,
+                id: String(row['id']),
+                label: recordTitle(shelf.def, row),
+                snippet: m.snippet,
+            },
+        });
+    }
+    return out;
+}
+export async function globalSearchByScan(shelves, q, limit) {
+    const tokens = tokenize(q);
+    if (tokens.length === 0)
+        return [];
+    const fork = getEm().fork();
+    const scored = [];
+    for (const shelf of shelves) {
+        if (shelf.def.standalone === false)
+            continue;
+        if (!textSearchKeys(shelf.def).length)
+            continue;
+        const rows = (await fork.find(shelf.table, {}, {
+            limit: SEARCH_SCAN_LIMIT,
+            fields: matchCols(shelf.def),
+        })).map((r) => serialize(r));
+        if (rows.length === SEARCH_SCAN_LIMIT) {
+            log.warn('scan cap reached', { shelf: shelf.key, limit: SEARCH_SCAN_LIMIT });
+        }
+        scored.push(...scoreRows(shelf, rows, tokens));
+    }
+    scored.sort((a, b) => b.score - a.score || tieKey(a.result).localeCompare(tieKey(b.result)));
+    return scored.slice(0, limit).map((s) => s.result);
+}
+export async function globalSearch(shelves, q, limit) {
+    const tokens = tokenize(q);
+    if (tokens.length === 0)
+        return [];
+    if (!ftsAvailable() || tokens.some((t) => t.length < MIN_FTS_QUERY_LENGTH)) {
+        return globalSearchByScan(shelves, q, limit);
+    }
+    const candidates = await ftsCandidates(tokens);
+    if (candidates === null)
+        return globalSearchByScan(shelves, q, limit);
+    if (candidates.size === 0)
+        return [];
+    const byKey = new Map(shelves.map((s) => [s.key, s]));
+    const fork = getEm().fork();
+    const scored = [];
+    for (const [key, ids] of candidates) {
+        const shelf = byKey.get(key);
+        if (!shelf || shelf.def.standalone === false)
+            continue;
+        if (!textSearchKeys(shelf.def).length)
+            continue;
+        const rows = (await fork.find(shelf.table, { id: { $in: ids } }, {
+            fields: matchCols(shelf.def),
+        })).map((r) => serialize(r));
+        scored.push(...scoreRows(shelf, rows, tokens));
+    }
+    scored.sort((a, b) => b.score - a.score || tieKey(a.result).localeCompare(tieKey(b.result)));
+    return scored.slice(0, limit).map((s) => s.result);
+}

package/dist/index-signal.d.ts

@@ -1,3 +1,4 @@
-export declare function onRecordsChanged(cb: () => void): void;
-export declare function offRecordsChanged(): void;
+export declare function onRecordsChanged(cb: () => void): () => void;
+export declare function offRecordsChanged(cb?: () => void): void;
+export declare function recordsChangedListenerCount(): number;
 export declare function notifyRecordsChanged(): void;

package/dist/index-signal.js

@@ -1,14 +1,28 @@
 import { getLogger } from '@coffer-org/sdk/logger';
-const log = getLogger('rag-signal');
-let listener = null;
+const log = getLogger('index-signal');
+const listeners = new Set();
 export function onRecordsChanged(cb) {
-    if (listener)
-        log.warn('onRecordsChanged: overwriting an already-registered listener without offRecordsChanged');
-    listener = cb;
+    listeners.add(cb);
+    return () => {
+        listeners.delete(cb);
+    };
 }
-export function offRecordsChanged() {
-    listener = null;
+export function offRecordsChanged(cb) {
+    if (cb)
+        listeners.delete(cb);
+    else
+        listeners.clear();
+}
+export function recordsChangedListenerCount() {
+    return listeners.size;
 }
 export function notifyRecordsChanged() {
-    listener?.();
+    for (const cb of [...listeners]) {
+        try {
+            cb();
+        }
+        catch (e) {
+            log.error(`records-changed listener threw — ${e.message}`);
+        }
+    }
 }