@bendyline/gezel-inference 0.0.0-stage → 0.1.0

package/dist/knowledge/registry.d.ts

@@ -0,0 +1,35 @@
+/**
+ * The per-user knowledge registry — `~/.gezel/knowledge/registry.json`, the
+ * authoritative record of what THIS user installed and enabled
+ * (docs/knowledge-catalogs.md). The manager never scans storage trees and
+ * infers state from directory presence; the registry is the truth, and a
+ * catalog whose bytes exist but whose entry is gone is just unreferenced
+ * disk to prune. Owned by this module — a deliberate Store carve-out like
+ * the other feature-owned subtrees (see CLAUDE.md).
+ */
+import type { KnowledgeCatalogRef, KnowledgeInstallSourceKind, KnowledgeUserRegistry } from '@bendyline/gezel';
+export type KnowledgeRegistryEntry = KnowledgeUserRegistry['catalogs'][number];
+export declare class KnowledgeRegistry {
+    private readonly home;
+    private readonly file;
+    constructor(home: string);
+    read(): KnowledgeUserRegistry;
+    private write;
+    /** Find an entry by publisher+catalog identity (one entry per catalog id). */
+    find(publisherId: string, catalogId: string): KnowledgeRegistryEntry | null;
+    /**
+     * Add or replace the entry for this catalog identity. The same-catalogId/
+     * different-publisher conflict is rejected here: the catalog id is the
+     * browse and route handle (`/api/knowledge/catalogs/:id`), so one id maps
+     * to one publisher per user even though `knowledge://` references carry both.
+     */
+    upsert(ref: KnowledgeCatalogRef, opts?: {
+        enabled?: boolean;
+        autoUpdate?: boolean;
+        source?: KnowledgeInstallSourceKind;
+    }): void;
+    setEnabled(publisherId: string, catalogId: string, enabled: boolean): boolean;
+    /** Quarantine: disable with a reason the UI/CLI can show verbatim. */
+    quarantine(publisherId: string, catalogId: string, reason: string): boolean;
+    remove(publisherId: string, catalogId: string): KnowledgeRegistryEntry | null;
+}

package/dist/knowledge/registry.js

@@ -0,0 +1,123 @@
+/**
+ * The per-user knowledge registry — `~/.gezel/knowledge/registry.json`, the
+ * authoritative record of what THIS user installed and enabled
+ * (docs/knowledge-catalogs.md). The manager never scans storage trees and
+ * infers state from directory presence; the registry is the truth, and a
+ * catalog whose bytes exist but whose entry is gone is just unreferenced
+ * disk to prune. Owned by this module — a deliberate Store carve-out like
+ * the other feature-owned subtrees (see CLAUDE.md).
+ */
+import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
+import { dirname } from 'node:path';
+import { KnowledgeUserRegistrySchema, createLogger } from '@bendyline/gezel';
+import { knowledgeRegistryFile } from '@bendyline/gezel/paths';
+const log = createLogger('knowledge');
+const EMPTY = { version: 1, catalogs: [] };
+export class KnowledgeRegistry {
+    home;
+    file;
+    constructor(home) {
+        this.home = home;
+        this.file = knowledgeRegistryFile(home);
+    }
+    read() {
+        let raw;
+        try {
+            raw = readFileSync(this.file, 'utf8');
+        }
+        catch {
+            return structuredClone(EMPTY);
+        }
+        try {
+            return KnowledgeUserRegistrySchema.parse(JSON.parse(raw));
+        }
+        catch (err) {
+            // A corrupt registry must not brick the daemon: keep the bytes for
+            // inspection and start empty. Catalog directories are untouched, so
+            // re-adding entries recovers everything.
+            log.error(`knowledge registry unreadable (${err instanceof Error ? err.message : String(err)}); starting empty`);
+            try {
+                renameSync(this.file, `${this.file}.corrupt-${Date.now()}`);
+            }
+            catch {
+                /* leave it in place */
+            }
+            return structuredClone(EMPTY);
+        }
+    }
+    write(registry) {
+        const parsed = KnowledgeUserRegistrySchema.parse(registry);
+        mkdirSync(dirname(this.file), { recursive: true });
+        const tmp = `${this.file}.tmp`;
+        writeFileSync(tmp, `${JSON.stringify(parsed, null, 2)}\n`, 'utf8');
+        renameSync(tmp, this.file);
+    }
+    /** Find an entry by publisher+catalog identity (one entry per catalog id). */
+    find(publisherId, catalogId) {
+        return (this.read().catalogs.find((c) => c.ref.publisherId === publisherId && c.ref.catalogId === catalogId) ?? null);
+    }
+    /**
+     * Add or replace the entry for this catalog identity. The same-catalogId/
+     * different-publisher conflict is rejected here: the catalog id is the
+     * browse and route handle (`/api/knowledge/catalogs/:id`), so one id maps
+     * to one publisher per user even though `knowledge://` references carry both.
+     */
+    upsert(ref, opts = {}) {
+        const registry = this.read();
+        const conflict = registry.catalogs.find((c) => c.ref.catalogId === ref.catalogId && c.ref.publisherId !== ref.publisherId);
+        if (conflict) {
+            throw new Error(`catalog id '${ref.catalogId}' is already installed from publisher '${conflict.ref.publisherId}'; remove it before installing the one from '${ref.publisherId}'`);
+        }
+        const existing = registry.catalogs.find((c) => c.ref.publisherId === ref.publisherId && c.ref.catalogId === ref.catalogId);
+        if (existing) {
+            existing.ref = ref;
+            existing.enabled = opts.enabled ?? existing.enabled;
+            if (opts.autoUpdate !== undefined)
+                existing.autoUpdate = opts.autoUpdate;
+            if (opts.source !== undefined)
+                existing.source = opts.source;
+            existing.disabledReason = undefined;
+        }
+        else {
+            registry.catalogs.push({
+                ref,
+                enabled: opts.enabled ?? true,
+                addedAt: new Date().toISOString(),
+                ...(opts.autoUpdate !== undefined ? { autoUpdate: opts.autoUpdate } : {}),
+                ...(opts.source !== undefined ? { source: opts.source } : {}),
+            });
+        }
+        this.write(registry);
+    }
+    setEnabled(publisherId, catalogId, enabled) {
+        const registry = this.read();
+        const entry = registry.catalogs.find((c) => c.ref.publisherId === publisherId && c.ref.catalogId === catalogId);
+        if (!entry)
+            return false;
+        entry.enabled = enabled;
+        if (enabled)
+            entry.disabledReason = undefined;
+        this.write(registry);
+        return true;
+    }
+    /** Quarantine: disable with a reason the UI/CLI can show verbatim. */
+    quarantine(publisherId, catalogId, reason) {
+        const registry = this.read();
+        const entry = registry.catalogs.find((c) => c.ref.publisherId === publisherId && c.ref.catalogId === catalogId);
+        if (!entry)
+            return false;
+        entry.enabled = false;
+        entry.disabledReason = reason;
+        this.write(registry);
+        return true;
+    }
+    remove(publisherId, catalogId) {
+        const registry = this.read();
+        const index = registry.catalogs.findIndex((c) => c.ref.publisherId === publisherId && c.ref.catalogId === catalogId);
+        if (index === -1)
+            return null;
+        const [removed] = registry.catalogs.splice(index, 1);
+        this.write(registry);
+        return removed ?? null;
+    }
+}

package/dist/knowledge/search-worker.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Knowledge catalog worker — owns every catalog SQLite connection.
+ * CatalogHandle is synchronous by design (node:sqlite), so a 10-20 ms shard
+ * scan (or a 100 ms cold burst) must run here, never on the daemon loop
+ * (docs/gezk-format.md). One long-lived worker; scans run sequentially
+ * inside it, which is the format's stated concurrency model.
+ *
+ * Protocol (structured-clone messages over the worker port):
+ *   host → worker:  { id, op, ...args }   — one op per KnowledgeCatalogHost method
+ *   worker → host:  { id, result }        — success
+ *                   { id, error }         — failure (message string)
+ */
+export interface KnowledgeWorkerRequest {
+    id: number;
+    op: 'mount' | 'unmount' | 'mounted' | 'prewarm' | 'validate' | 'topics' | 'documentsPage' | 'nearbyDocuments' | 'getDocument' | 'getChunk' | 'assets' | 'readAsset' | 'assetFile' | 'search' | 'dispose';
+    args: unknown[];
+}

package/dist/knowledge/search-worker.js

@@ -0,0 +1,36 @@
+/**
+ * Knowledge catalog worker — owns every catalog SQLite connection.
+ * CatalogHandle is synchronous by design (node:sqlite), so a 10-20 ms shard
+ * scan (or a 100 ms cold burst) must run here, never on the daemon loop
+ * (docs/gezk-format.md). One long-lived worker; scans run sequentially
+ * inside it, which is the format's stated concurrency model.
+ *
+ * Protocol (structured-clone messages over the worker port):
+ *   host → worker:  { id, op, ...args }   — one op per KnowledgeCatalogHost method
+ *   worker → host:  { id, result }        — success
+ *                   { id, error }         — failure (message string)
+ */
+import { parentPort } from 'node:worker_threads';
+import { createInProcessCatalogHost } from './catalog-host.js';
+if (!parentPort) {
+    throw new Error('knowledge search-worker must be run as a worker thread');
+}
+const port = parentPort;
+let hostPromise = null;
+function host() {
+    hostPromise ??= createInProcessCatalogHost();
+    return hostPromise;
+}
+port.on('message', (msg) => {
+    void (async () => {
+        try {
+            const h = await host();
+            // biome-ignore lint/suspicious/noExplicitAny: op-indexed dynamic dispatch over the host interface
+            const result = await h[msg.op](...msg.args);
+            port.postMessage({ id: msg.id, result });
+        }
+        catch (err) {
+            port.postMessage({ id: msg.id, error: err instanceof Error ? err.message : String(err) });
+        }
+    })();
+});

package/dist/knowledge/shared-install-contract.d.ts

@@ -0,0 +1,19 @@
+import type { KnowledgeInstallEvent, TrustedKnowledgeCoordinate } from '@bendyline/gezel';
+export type SharedEnsureResult = {
+    status: 'ready';
+} | {
+    status: 'unavailable';
+    reason: string;
+} | {
+    status: 'failed';
+    error: string;
+    mismatch: {
+        expected: string;
+        actual: string;
+    };
+};
+export interface SharedKnowledgeInstaller {
+    available(): boolean;
+    ensure(coordinate: TrustedKnowledgeCoordinate, onEvent: (event: KnowledgeInstallEvent) => void, signal?: AbortSignal): Promise<SharedEnsureResult>;
+    cancel(coordinate: TrustedKnowledgeCoordinate): Promise<boolean>;
+}

package/dist/knowledge/shared-install-contract.js

@@ -0,0 +1 @@
+export {};

package/dist/knowledge/spatial-query.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Radius pagination binds offsets to the authorized catalog snapshot and
+ * query. Cursors confer no authority; the manager recalculates scope on
+ * every request. Regional replicas share identity only under Qualla's
+ * declared regional catalog and source-document naming convention.
+ */
+export declare class KnowledgeSpatialCursorError extends Error {
+}
+export declare function encodeNearbyCursor(snapshot: string, offset: number): string;
+export declare function nearbyCursorOffset(cursor: string | undefined, snapshot: string): number;
+export declare function regionalDocumentIdentity(publisherId: string, catalogId: string, documentId: string): string;

package/dist/knowledge/spatial-query.js

@@ -0,0 +1,33 @@
+/**
+ * Radius pagination binds offsets to the authorized catalog snapshot and
+ * query. Cursors confer no authority; the manager recalculates scope on
+ * every request. Regional replicas share identity only under Qualla's
+ * declared regional catalog and source-document naming convention.
+ */
+export class KnowledgeSpatialCursorError extends Error {
+}
+export function encodeNearbyCursor(snapshot, offset) {
+    return Buffer.from(JSON.stringify({ snapshot, offset })).toString('base64url');
+}
+export function nearbyCursorOffset(cursor, snapshot) {
+    if (!cursor)
+        return 0;
+    try {
+        if (cursor.length > 1024 || !/^[A-Za-z0-9_-]+$/.test(cursor))
+            throw new Error();
+        const value = JSON.parse(Buffer.from(cursor, 'base64url').toString('utf8'));
+        if (value.snapshot !== snapshot || !Number.isSafeInteger(value.offset) || value.offset < 0)
+            throw new Error();
+        return value.offset;
+    }
+    catch {
+        throw new KnowledgeSpatialCursorError('Invalid or stale nearby cursor; restart the query.');
+    }
+}
+export function regionalDocumentIdentity(publisherId, catalogId, documentId) {
+    const regional = catalogId.startsWith('qualla-region-') &&
+        /^qualla-(wikipedia|wikivoyage)-[0-9]+$/.test(documentId);
+    return regional
+        ? `${publisherId}\u0000${documentId}`
+        : `${publisherId}\u0000${catalogId}\u0000${documentId}`;
+}

package/dist/knowledge/trust-anchors.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Trust-anchor resolution for signed knowledge registries: the anchors
+ * BUILT INTO the app plus an operator/test overlay from
+ * `GEZEL_KNOWLEDGE_TRUST_ANCHORS` (a JSON file of KnowledgeTrustAnchor
+ * rows). The overlay can only ADD trust — built-ins are never removed by
+ * configuration, so a hostile env cannot silently blind verification of
+ * the shipped publisher.
+ */
+import type { KnowledgeTrustAnchor } from '@bendyline/gezel-knowledge';
+export declare const KNOWLEDGE_TRUST_ANCHORS_ENV = "GEZEL_KNOWLEDGE_TRUST_ANCHORS";
+export declare function loadKnowledgeTrustAnchors(env?: NodeJS.ProcessEnv): KnowledgeTrustAnchor[];

package/dist/knowledge/trust-anchors.js

@@ -0,0 +1,51 @@
+/**
+ * Trust-anchor resolution for signed knowledge registries: the anchors
+ * BUILT INTO the app plus an operator/test overlay from
+ * `GEZEL_KNOWLEDGE_TRUST_ANCHORS` (a JSON file of KnowledgeTrustAnchor
+ * rows). The overlay can only ADD trust — built-ins are never removed by
+ * configuration, so a hostile env cannot silently blind verification of
+ * the shipped publisher.
+ */
+import { readFileSync } from 'node:fs';
+import { isAbsolute } from 'node:path';
+import { createLogger } from '@bendyline/gezel';
+import { BUILTIN_KNOWLEDGE_TRUST_ANCHORS } from '@bendyline/gezel-knowledge';
+const log = createLogger('knowledge');
+export const KNOWLEDGE_TRUST_ANCHORS_ENV = 'GEZEL_KNOWLEDGE_TRUST_ANCHORS';
+export function loadKnowledgeTrustAnchors(env = process.env) {
+    const anchors = [...BUILTIN_KNOWLEDGE_TRUST_ANCHORS];
+    const overlayPath = env[KNOWLEDGE_TRUST_ANCHORS_ENV]?.trim();
+    if (!overlayPath)
+        return anchors;
+    if (!isAbsolute(overlayPath)) {
+        log.warn(`${KNOWLEDGE_TRUST_ANCHORS_ENV} must be an absolute path; ignoring ${overlayPath}`);
+        return anchors;
+    }
+    try {
+        const parsed = JSON.parse(readFileSync(overlayPath, 'utf8'));
+        if (!Array.isArray(parsed))
+            throw new Error('expected a JSON array');
+        const overlay = [];
+        for (const row of parsed) {
+            if (typeof row === 'object' &&
+                row !== null &&
+                typeof row.keyId === 'string' &&
+                typeof row.publicKeyPem === 'string') {
+                const anchor = row;
+                overlay.push({
+                    keyId: anchor.keyId,
+                    publicKeyPem: anchor.publicKeyPem,
+                    ...(typeof anchor.publisherId === 'string' ? { publisherId: anchor.publisherId } : {}),
+                });
+            }
+            else {
+                throw new Error('every row needs string keyId + publicKeyPem');
+            }
+        }
+        anchors.push(...overlay);
+    }
+    catch (err) {
+        log.warn(`ignoring ${KNOWLEDGE_TRUST_ANCHORS_ENV} overlay at ${overlayPath}: ${err instanceof Error ? err.message : String(err)}`);
+    }
+    return anchors;
+}

package/dist/knowledge/vector-floors.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Cosine floors for knowledge-catalog vector hits.
+ *
+ * KNN has no notion of "no good answer": the nearest chunk comes back however
+ * far away it is, and the fused rank then scores it 1.0. A floor is what lets
+ * the vector arm say nothing. The project index learned this first
+ * (`VECTOR_ARM_MIN_SIMILARITY`); these are the same idea for catalogs.
+ *
+ * Keys are a catalog (`publisher/catalog`) or an embedding profile id; a
+ * catalog's own entry wins. A floor is a property of the embedder AND the
+ * corpus: multilingual-e5 scores unrelated text higher than bge-small scores
+ * a match, and bge-small's genuine matches sit at 0.57 in the project library
+ * but 0.69 in the Handboek. So the profile entries are defaults for catalogs
+ * nobody measured, and a catalog whose scale is known gets its own.
+ *
+ * A vector hit under its floor is not evidence: it neither ranks the document
+ * nor labels it `vector`. A profile with no floor at all keeps its hits for
+ * ranking but labels them `fts`, so proactive injection asks them to be
+ * grounded in a query term — an unmeasured scale is not trusted to vouch.
+ *
+ * Measured with the retrieval preview's `similarity` trace field; method and
+ * numbers in evals/src/retrieval-bench/KNOWLEDGE-CALIBRATION-2026-09-30.md.
+ * Re-measure on any profile or catalog change: these numbers do not travel.
+ */
+export declare const KNOWLEDGE_VECTOR_FLOORS: Readonly<Record<string, number>>;
+export interface KnowledgeVectorFloors {
+    /**
+     * The floor for a catalog, or null when its scale was never measured. With
+     * a `modality`, the floor for that kind of media row (`<key>#image`, …):
+     * a text query sits further from a matching image than from a matching
+     * passage, so media rows need their own scale.
+     */
+    floorFor(catalog: {
+        catalogKey: string;
+        profileId: string;
+        modality?: 'image' | 'video' | 'audio';
+    }): number | null;
+}
+/**
+ * `GEZEL_KNOWLEDGE_VECTOR_FLOORS` overrides for calibration: `off` treats
+ * every catalog as measured with floor 0, and `key=0.6,key2=0.8` replaces the
+ * named entries (catalog or profile keys).
+ */
+export declare function resolveKnowledgeVectorFloors(env?: NodeJS.ProcessEnv): KnowledgeVectorFloors;

package/dist/knowledge/vector-floors.js

@@ -0,0 +1,92 @@
+import { createLogger } from '@bendyline/gezel';
+const log = createLogger('knowledge');
+/**
+ * Cosine floors for knowledge-catalog vector hits.
+ *
+ * KNN has no notion of "no good answer": the nearest chunk comes back however
+ * far away it is, and the fused rank then scores it 1.0. A floor is what lets
+ * the vector arm say nothing. The project index learned this first
+ * (`VECTOR_ARM_MIN_SIMILARITY`); these are the same idea for catalogs.
+ *
+ * Keys are a catalog (`publisher/catalog`) or an embedding profile id; a
+ * catalog's own entry wins. A floor is a property of the embedder AND the
+ * corpus: multilingual-e5 scores unrelated text higher than bge-small scores
+ * a match, and bge-small's genuine matches sit at 0.57 in the project library
+ * but 0.69 in the Handboek. So the profile entries are defaults for catalogs
+ * nobody measured, and a catalog whose scale is known gets its own.
+ *
+ * A vector hit under its floor is not evidence: it neither ranks the document
+ * nor labels it `vector`. A profile with no floor at all keeps its hits for
+ * ranking but labels them `fts`, so proactive injection asks them to be
+ * grounded in a query term — an unmeasured scale is not trusted to vouch.
+ *
+ * Measured with the retrieval preview's `similarity` trace field; method and
+ * numbers in evals/src/retrieval-bench/KNOWLEDGE-CALIBRATION-2026-09-30.md.
+ * Re-measure on any profile or catalog change: these numbers do not travel.
+ */
+export const KNOWLEDGE_VECTOR_FLOORS = {
+    // Handboek 1.1.2: answers ≥ 0.687, off-topic ≤ 0.624.
+    'bendyline/handboek': 0.65,
+    // The project index's measured floor for the daemon's own embedder
+    // (index-store VECTOR_ARM_MIN_SIMILARITY) — what a folder-built catalog gets.
+    'bge-small-en-v1.5@1': 0.55,
+    // Wikipedia Food & Drink 2026.4.3: answers ≥ 0.869; the scale is compressed
+    // and 2 of 25 off-topic prompts still reach it, so grounding and the
+    // relevance model do the rest.
+    'multilingual-e5-small@2': 0.865,
+    // Revision 1 has the same model, instructions and int8 vectors — only its
+    // stage-1 sign bits differ — so its rerank cosine is on the same scale.
+    'multilingual-e5-small@1': 0.865,
+    // Handboek 1.2.2 + a Food & Drink sample re-embedded in this profile (the
+    // labelled calibration queries): answers ≥ 0.762 in both, off-topic ≤ 0.706,
+    // a query's best chunk in the OTHER catalog ≤ 0.715. A wide gap; the scale
+    // is far less compressed than e5's.
+    'embeddinggemma-2-512@1': 0.735,
+    // Media rows of the same profile, workspace and catalogs alike
+    // (evals/src/retrieval-bench/MEDIA-BENCH-2026-10-07.md). Each sits a step
+    // above the lowest floor no off-topic prompt reached on the bench corpus,
+    // because a real library is larger and its off-topic maximum higher.
+    // COCO photos: no off-topic hit from 0.66; 293 of 300 answers keep their photo.
+    'embeddinggemma-2-512@1#image': 0.67,
+    // MSR-VTT clips: no off-topic hit from 0.65; 54 of 60 answers clear 0.65 and 0.66.
+    'embeddinggemma-2-512@1#video': 0.66,
+    // ESC-50 sounds: scores bunch (a wrong clip clears 0.66 for most queries),
+    // so the floor does less here than ranking; 47 of 50 answers clear 0.68.
+    'embeddinggemma-2-512@1#audio': 0.68,
+};
+const warnedUnmeasured = new Set();
+/**
+ * `GEZEL_KNOWLEDGE_VECTOR_FLOORS` overrides for calibration: `off` treats
+ * every catalog as measured with floor 0, and `key=0.6,key2=0.8` replaces the
+ * named entries (catalog or profile keys).
+ */
+export function resolveKnowledgeVectorFloors(env = process.env) {
+    const raw = env.GEZEL_KNOWLEDGE_VECTOR_FLOORS?.trim();
+    if (raw === 'off')
+        return { floorFor: () => 0 };
+    const floors = new Map(Object.entries(KNOWLEDGE_VECTOR_FLOORS));
+    for (const pair of raw ? raw.split(',') : []) {
+        const at = pair.lastIndexOf('=');
+        const value = Number(pair.slice(at + 1));
+        if (at <= 0 || !Number.isFinite(value)) {
+            log.warn(`ignoring GEZEL_KNOWLEDGE_VECTOR_FLOORS entry ${JSON.stringify(pair)}`);
+            continue;
+        }
+        floors.set(pair.slice(0, at).trim(), value);
+    }
+    return {
+        floorFor({ catalogKey, profileId, modality }) {
+            if (modality) {
+                return (floors.get(`${catalogKey}#${modality}`) ?? floors.get(`${profileId}#${modality}`) ?? null);
+            }
+            const floor = floors.get(catalogKey) ?? floors.get(profileId);
+            if (floor !== undefined)
+                return floor;
+            if (!warnedUnmeasured.has(profileId)) {
+                warnedUnmeasured.add(profileId);
+                log.info(`no measured vector floor for embedding profile ${profileId}; its semantic hits must be grounded to be injected`);
+            }
+            return null;
+        },
+    };
+}

package/dist/knowledge/worker-host.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Worker-backed KnowledgeCatalogHost — the daemon's production host. Spawns
+ * the knowledge search-worker (embed-worker spawn pattern: long-lived,
+ * unref'd, id-correlated request/response, reject-in-flight on death, crash
+ * cap → fail closed). Under vitest there is no dist build, so the explicitly
+ * test-only in-process host is used directly.
+ */
+import { type KnowledgeCatalogHost } from './catalog-host.js';
+export declare function createWorkerCatalogHost(): KnowledgeCatalogHost;