@knolo/core 3.2.1 → 3.2.4

package/dist/query.js

@@ -15,6 +15,8 @@ import { diversifyAndDedupe } from "./quality/diversify.js";
 import { knsSignature, knsDistance } from "./quality/signature.js";
 import { decodeScaleF16, quantizeEmbeddingInt8L2Norm } from "./semantic.js";
 import { expandQueryWithGraph } from "./graph/query_expand.js";
+import { rerankCandidates } from "./semantic/rerank.js";
+import { parseSidecar } from "./semantic/sidecar.js";
 export function validateQueryOptions(opts) {
     if (!opts)
         return;
@@ -78,6 +80,19 @@ export function validateSemanticQueryOptions(options) {
     if (options.queryEmbedding !== undefined && !(options.queryEmbedding instanceof Float32Array)) {
         throw new Error("query(...): semantic.queryEmbedding must be a Float32Array.");
     }
+    if (options.sidecarPath !== undefined && typeof options.sidecarPath !== "string") {
+        throw new Error("query(...): semantic.sidecarPath must be a string when provided.");
+    }
+    if (options.minSemanticScore !== undefined && (!Number.isFinite(options.minSemanticScore) || options.minSemanticScore < 0 || options.minSemanticScore > 1)) {
+        throw new Error("query(...): semantic.minSemanticScore must be a finite number between 0 and 1.");
+    }
+    if (options.provider) {
+        if (options.provider.type !== "ollama")
+            throw new Error('query(...): semantic.provider.type must be "ollama".');
+        if (typeof options.provider.modelId !== "string" || !options.provider.modelId.trim()) {
+            throw new Error("query(...): semantic.provider.modelId must be a non-empty string.");
+        }
+    }
     if (options.blend) {
         if (options.blend.enabled !== undefined && typeof options.blend.enabled !== "boolean") {
             throw new Error("query(...): semantic.blend.enabled must be a boolean when provided.");
@@ -115,6 +130,9 @@ export function query(pack, q, opts = {}) {
             wSem: Math.max(0, opts.semantic?.blend?.wSem ?? 0.25),
         },
         queryEmbedding: opts.semantic?.queryEmbedding,
+        sidecar: resolveSemanticSidecar(opts.semantic?.sidecar, opts.semantic?.sidecarPath),
+        provider: opts.semantic?.provider,
+        minSemanticScore: opts.semantic?.minSemanticScore,
         force: opts.semantic?.force ?? false,
     };
     const graphQuery = opts.graph?.expand === true
@@ -163,7 +181,7 @@ export function query(pack, q, opts = {}) {
                 let pos = p[i++];
                 const positions = [];
                 while (pos !== 0) {
-                    positions.push(pos);
+                    positions.push(pos - 1);
                     pos = p[i++];
                 }
                 termDf++;
@@ -284,9 +302,16 @@ export function query(pack, q, opts = {}) {
             return [];
     }
     const confidence = lexConfidence(prelim);
+    let semanticScores;
+    let blendedScores;
+    const originalLexicalScores = new Map(prelim.map((item) => [item.blockId, item.score]));
     if (shouldRerankWithSemantic(pack, semanticOpts, confidence)) {
-        prelim = rerankLexicalHitsWithSemantic(pack, prelim, semanticOpts);
+        const semanticResult = rerankLexicalHitsWithSemantic(pack, prelim, semanticOpts);
+        prelim = semanticResult.hits;
+        semanticScores = semanticResult.semanticScores;
+        blendedScores = semanticResult.blendedScores;
     }
+    const retrievalMode = semanticScores ? "hybrid" : "lexical";
     // --- KNS tie-breaker + de-dup/MMR
     const qSig = knsSignature(normalize(q));
     const pool = prelim.slice(0, topK * 5).map((r) => {
@@ -298,6 +323,13 @@ export function query(pack, q, opts = {}) {
             text,
             source: pack.docIds?.[r.blockId] ?? undefined,
             namespace: pack.namespaces?.[r.blockId] ?? undefined,
+            evidence: {
+                retrieval: retrievalMode,
+                lexicalScore: originalLexicalScores.get(r.blockId) ?? r.score,
+                semanticScore: semanticScores?.get(r.blockId),
+                blendedScore: blendedScores?.get(r.blockId),
+                modelId: semanticOpts.provider?.modelId ?? semanticOpts.sidecar?.modelId,
+            },
         };
     });
     const finalHits = diversifyAndDedupe(pool, { k: topK });
@@ -315,19 +347,66 @@ export function lexConfidence(hits) {
 function shouldRerankWithSemantic(pack, opts, confidence) {
     if (!opts.enabled || opts.mode !== "rerank")
         return false;
-    if (!pack.semantic)
+    if (!pack.semantic && !opts.sidecar)
         return false;
     if (!opts.queryEmbedding) {
         throw new Error("query(...): semantic.queryEmbedding (Float32Array) is required when semantic.enabled=true.");
     }
     return opts.force || confidence < opts.minLexConfidence;
 }
+function resolveSemanticSidecar(sidecar, sidecarPath) {
+    if (sidecar)
+        return sidecar;
+    if (!sidecarPath)
+        return undefined;
+    const raw = sidecarPath.trim();
+    if (!raw)
+        return undefined;
+    if (raw.startsWith("{")) {
+        return parseSidecar(raw);
+    }
+    if (raw.startsWith("data:")) {
+        const comma = raw.indexOf(",");
+        if (comma <= 0)
+            return undefined;
+        const meta = raw.slice(5, comma).toLowerCase();
+        const payload = raw.slice(comma + 1);
+        const decoded = meta.includes(";base64")
+            ? decodeBase64(payload)
+            : decodeURIComponent(payload);
+        if (!decoded.trim())
+            return undefined;
+        return parseSidecar(decoded);
+    }
+    return undefined;
+}
+function decodeBase64(input) {
+    const normalized = input.replace(/\s+/g, "");
+    const atobFn = globalThis.atob;
+    if (typeof atobFn === "function")
+        return atobFn(normalized);
+    const maybeBufferCtor = globalThis.Buffer;
+    if (maybeBufferCtor?.from)
+        return maybeBufferCtor.from(normalized, "base64").toString("utf8");
+    throw new Error("query(...): Unable to decode semantic.sidecarPath base64 payload in this runtime.");
+}
 function rerankLexicalHitsWithSemantic(pack, prelim, opts) {
+    if (opts.sidecar && opts.queryEmbedding) {
+        const sidecarResult = rerankCandidates({
+            lexical: prelim,
+            sidecar: opts.sidecar,
+            queryEmbedding: opts.queryEmbedding,
+            topN: opts.topN,
+            blend: opts.blend,
+            minSemanticScore: opts.minSemanticScore,
+        });
+        return { hits: sidecarResult.reranked, semanticScores: sidecarResult.semanticScores, blendedScores: sidecarResult.blendedScores };
+    }
     const sem = pack.semantic;
     if (!sem || !opts.queryEmbedding)
-        return prelim;
+        return { hits: prelim };
     if (sem.dims <= 0 || sem.vecs.length === 0 || sem.dims !== opts.queryEmbedding.length)
-        return prelim;
+        return { hits: prelim };
     const topN = Math.min(opts.topN, prelim.length);
     const rerankSlice = prelim.slice(0, topN);
     const tail = prelim.slice(topN);
@@ -342,15 +421,19 @@ function rerankLexicalHitsWithSemantic(pack, prelim, opts) {
     const wLex = denom > 0 ? opts.blend.wLex / denom : 0.5;
     const wSem = denom > 0 ? opts.blend.wSem / denom : 0.5;
     const reranked = new Array(topN);
+    const semanticScores = new Map();
+    const blendedScores = new Map();
     for (let i = 0; i < topN; i++) {
         const hit = rerankSlice[i];
+        semanticScores.set(hit.blockId, normSem[i]);
+        blendedScores.set(hit.blockId, opts.blend.enabled ? wLex * normLex[i] + wSem * normSem[i] : semScores[i]);
         reranked[i] = {
             blockId: hit.blockId,
-            score: opts.blend.enabled ? wLex * normLex[i] + wSem * normSem[i] : semScores[i],
+            score: blendedScores.get(hit.blockId) ?? hit.score,
         };
     }
     reranked.sort((a, b) => b.score - a.score || a.blockId - b.blockId);
-    return [...reranked, ...tail];
+    return { hits: [...reranked, ...tail], semanticScores, blendedScores };
 }
 function scoreSemanticInt8(queryQ, queryScale, semantic, hits) {
     const scores = new Float64Array(hits.length);

package/dist/semantic/cosine.d.ts

@@ -0,0 +1,2 @@
+export declare function normalizeVector(vector: Float32Array): Float32Array;
+export declare function cosineSimilarity(a: Float32Array, b: Float32Array): number;

package/dist/semantic/cosine.js

@@ -0,0 +1,20 @@
+export function normalizeVector(vector) {
+    let normSq = 0;
+    for (let i = 0; i < vector.length; i++)
+        normSq += vector[i] * vector[i];
+    const norm = Math.sqrt(normSq);
+    if (!norm)
+        return new Float32Array(vector.length);
+    const out = new Float32Array(vector.length);
+    for (let i = 0; i < vector.length; i++)
+        out[i] = vector[i] / norm;
+    return out;
+}
+export function cosineSimilarity(a, b) {
+    if (a.length !== b.length || a.length === 0)
+        return 0;
+    let dot = 0;
+    for (let i = 0; i < a.length; i++)
+        dot += a[i] * b[i];
+    return dot;
+}

package/dist/semantic/provider.d.ts

@@ -0,0 +1,3 @@
+import type { EmbeddingProvider, SemanticQueryOptions } from './types.js';
+export declare function ensureProviderModelId(options?: SemanticQueryOptions): string | undefined;
+export declare function assertProviderCompatible(options?: SemanticQueryOptions, provider?: EmbeddingProvider): void;

package/dist/semantic/provider.js

@@ -0,0 +1,13 @@
+export function ensureProviderModelId(options) {
+    return options?.provider?.modelId;
+}
+export function assertProviderCompatible(options, provider) {
+    if (!options?.enabled)
+        return;
+    if (!provider && !options.queryEmbedding) {
+        throw new Error('semantic.enabled=true requires either semantic.queryEmbedding or an EmbeddingProvider.');
+    }
+    if (provider && options.provider?.modelId && options.provider.modelId !== provider.modelId) {
+        throw new Error(`Semantic provider model mismatch: options requested ${options.provider.modelId}, provider exposes ${provider.modelId}.`);
+    }
+}

package/dist/semantic/rerank.d.ts

@@ -0,0 +1,23 @@
+import type { SemanticSidecar } from './types.js';
+export declare function rerankCandidates(params: {
+    lexical: Array<{
+        blockId: number;
+        score: number;
+    }>;
+    sidecar: SemanticSidecar;
+    queryEmbedding: Float32Array;
+    topN: number;
+    blend: {
+        enabled: boolean;
+        wLex: number;
+        wSem: number;
+    };
+    minSemanticScore?: number;
+}): {
+    reranked: Array<{
+        blockId: number;
+        score: number;
+    }>;
+    semanticScores: Map<number, number>;
+    blendedScores: Map<number, number>;
+};

package/dist/semantic/rerank.js

@@ -0,0 +1,42 @@
+import { cosineSimilarity, normalizeVector } from './cosine.js';
+export function rerankCandidates(params) {
+    const topN = Math.min(params.topN, params.lexical.length);
+    const head = params.lexical.slice(0, topN);
+    const tail = params.lexical.slice(topN);
+    const q = normalizeVector(params.queryEmbedding);
+    const semanticScores = new Map();
+    const blendedScores = new Map();
+    const lexNorm = minMax(head.map((h) => h.score));
+    const semRaw = [];
+    for (const item of head) {
+        const rec = params.sidecar.blocks.find((b) => b.blockId === item.blockId);
+        const vec = rec ? Float32Array.from(rec.vector) : new Float32Array(q.length);
+        semRaw.push(cosineSimilarity(q, vec));
+    }
+    const semNorm = minMax(semRaw);
+    const denom = params.blend.wLex + params.blend.wSem;
+    const wLex = denom > 0 ? params.blend.wLex / denom : 0.7;
+    const wSem = denom > 0 ? params.blend.wSem / denom : 0.3;
+    const reranked = head.map((item, idx) => {
+        const sem = semNorm[idx];
+        semanticScores.set(item.blockId, sem);
+        if ((params.minSemanticScore ?? 0) > sem) {
+            blendedScores.set(item.blockId, lexNorm[idx]);
+            return { blockId: item.blockId, score: lexNorm[idx] };
+        }
+        const blended = params.blend.enabled ? wLex * lexNorm[idx] + wSem * sem : sem;
+        blendedScores.set(item.blockId, blended);
+        return { blockId: item.blockId, score: blended };
+    });
+    reranked.sort((a, b) => b.score - a.score || a.blockId - b.blockId);
+    return { reranked: [...reranked, ...tail], semanticScores, blendedScores };
+}
+function minMax(values) {
+    if (values.length === 0)
+        return values;
+    const min = Math.min(...values);
+    const max = Math.max(...values);
+    if (!Number.isFinite(min) || !Number.isFinite(max) || max <= min)
+        return values.map(() => 1);
+    return values.map((v) => Math.min(1, Math.max(0, (v - min) / (max - min))));
+}

package/dist/semantic/sidecar.d.ts

@@ -0,0 +1,10 @@
+import type { Pack } from '../pack.runtime.js';
+import type { SemanticSidecar } from './types.js';
+export declare function createPackFingerprint(pack: Pick<Pack, 'blocks' | 'docIds' | 'meta'>): string;
+export declare function serializeSidecar(sidecar: SemanticSidecar): string;
+export declare function parseSidecar(raw: string): SemanticSidecar;
+export declare function validateSidecarForPack(input: {
+    sidecar: SemanticSidecar;
+    pack: Pick<Pack, 'blocks' | 'docIds' | 'meta'>;
+    modelId: string;
+}): void;

package/dist/semantic/sidecar.js

@@ -0,0 +1,32 @@
+export function createPackFingerprint(pack) {
+    let hash = 2166136261;
+    const parts = [String(pack.meta?.version ?? 0), ...(pack.docIds ?? []), ...pack.blocks];
+    for (const part of parts) {
+        const text = String(part ?? '');
+        for (let i = 0; i < text.length; i++) {
+            hash ^= text.charCodeAt(i);
+            hash = Math.imul(hash, 16777619);
+        }
+    }
+    return `fnv1a-${(hash >>> 0).toString(16).padStart(8, '0')}`;
+}
+export function serializeSidecar(sidecar) {
+    return `${JSON.stringify(sidecar, null, 2)}\n`;
+}
+export function parseSidecar(raw) {
+    const parsed = JSON.parse(raw);
+    if (parsed.version !== 1)
+        throw new Error(`Unsupported semantic sidecar version: ${parsed.version}`);
+    if (parsed.metric !== 'cosine')
+        throw new Error(`Unsupported semantic metric: ${parsed.metric}`);
+    return parsed;
+}
+export function validateSidecarForPack(input) {
+    const expectedFingerprint = createPackFingerprint(input.pack);
+    if (input.sidecar.packFingerprint !== expectedFingerprint) {
+        throw new Error(`Semantic sidecar pack fingerprint mismatch: expected ${expectedFingerprint}, got ${input.sidecar.packFingerprint}. Regenerate the sidecar for this pack.`);
+    }
+    if (input.sidecar.modelId !== input.modelId) {
+        throw new Error(`Semantic model mismatch: sidecar model is ${input.sidecar.modelId}, but query provider is ${input.modelId}. Use the same embedding model or regenerate the sidecar.`);
+    }
+}

package/dist/semantic/types.d.ts

@@ -0,0 +1,44 @@
+export interface EmbeddingProvider {
+    readonly modelId: string;
+    embedQuery(text: string): Promise<Float32Array>;
+    embedTexts(texts: string[]): Promise<Float32Array[]>;
+}
+export interface SemanticSidecar {
+    version: 1;
+    packFingerprint: string;
+    modelId: string;
+    dimension: number;
+    metric: 'cosine';
+    createdAt: string;
+    blocks: Array<{
+        blockId: number;
+        vector: number[];
+    }>;
+}
+export type SemanticQueryOptions = {
+    enabled?: boolean;
+    mode?: 'rerank';
+    topN?: number;
+    minLexConfidence?: number;
+    minSemanticScore?: number;
+    blend?: {
+        enabled?: boolean;
+        wLex?: number;
+        wSem?: number;
+    };
+    provider?: {
+        type: 'ollama';
+        modelId: string;
+        endpoint?: string;
+    };
+    sidecarPath?: string;
+    queryEmbedding?: Float32Array;
+    force?: boolean;
+};
+export type RetrievalEvidence = {
+    retrieval: 'lexical' | 'hybrid';
+    lexicalScore?: number;
+    semanticScore?: number;
+    blendedScore?: number;
+    modelId?: string;
+};

package/dist/semantic/types.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@knolo/core",
-  "version": "3.2.1",
+  "version": "3.2.4",
   "type": "module",
   "description": "Local-first knowledge packs for small LLMs.",
   "keywords": [
@@ -34,7 +34,7 @@
     "build": "tsc -p tsconfig.json",
     "prepublishOnly": "npm run build",
     "smoke": "node scripts/smoke.mjs",
-    "test": "npm run build && node scripts/check-runtime-no-node.mjs && node scripts/test.mjs",
+    "test": "npm run build && node scripts/check-runtime-no-node.mjs && node --test test/*.test.mjs && node scripts/test.mjs",
     "format": "prettier --write src/agent.ts src/pack.ts src/pack.runtime.ts src/pack.node.ts src/node.ts src/builder.ts src/index.ts scripts/test.mjs scripts/check-runtime-no-node.mjs ../../README.md README.md",
     "format:check": "prettier --check src/agent.ts src/pack.ts src/pack.runtime.ts src/pack.node.ts src/node.ts src/builder.ts src/index.ts scripts/test.mjs scripts/check-runtime-no-node.mjs ../../README.md README.md",
     "check:runtime-no-node": "node scripts/check-runtime-no-node.mjs"