knolo-core 0.2.3 → 0.3.0

package/dist/query.js

@@ -13,18 +13,76 @@ import { rankBM25L } from "./rank.js";
 import { minCoverSpan, proximityMultiplier } from "./quality/proximity.js";
 import { diversifyAndDedupe } from "./quality/diversify.js";
 import { knsSignature, knsDistance } from "./quality/signature.js";
+import { decodeScaleF16, quantizeEmbeddingInt8L2Norm } from "./semantic.js";
+export function validateSemanticQueryOptions(options) {
+    if (!options)
+        return;
+    if (options.enabled !== undefined && typeof options.enabled !== "boolean") {
+        throw new Error("query(...): semantic.enabled must be a boolean when provided.");
+    }
+    if (options.mode !== undefined && options.mode !== "rerank") {
+        throw new Error('query(...): semantic.mode currently only supports "rerank".');
+    }
+    if (options.topN !== undefined && (!Number.isInteger(options.topN) || options.topN < 1)) {
+        throw new Error("query(...): semantic.topN must be a positive integer.");
+    }
+    if (options.minLexConfidence !== undefined && (!Number.isFinite(options.minLexConfidence) || options.minLexConfidence < 0 || options.minLexConfidence > 1)) {
+        throw new Error("query(...): semantic.minLexConfidence must be a finite number between 0 and 1.");
+    }
+    if (options.queryEmbedding !== undefined && !(options.queryEmbedding instanceof Float32Array)) {
+        throw new Error("query(...): semantic.queryEmbedding must be a Float32Array.");
+    }
+    if (options.blend) {
+        if (options.blend.enabled !== undefined && typeof options.blend.enabled !== "boolean") {
+            throw new Error("query(...): semantic.blend.enabled must be a boolean when provided.");
+        }
+        if (options.blend.wLex !== undefined && (!Number.isFinite(options.blend.wLex) || options.blend.wLex < 0)) {
+            throw new Error("query(...): semantic.blend.wLex must be a finite number >= 0.");
+        }
+        if (options.blend.wSem !== undefined && (!Number.isFinite(options.blend.wSem) || options.blend.wSem < 0)) {
+            throw new Error("query(...): semantic.blend.wSem must be a finite number >= 0.");
+        }
+    }
+    if (options.force !== undefined && typeof options.force !== "boolean") {
+        throw new Error("query(...): semantic.force must be a boolean when provided.");
+    }
+}
 export function query(pack, q, opts = {}) {
+    validateSemanticQueryOptions(opts.semantic);
     const topK = opts.topK ?? 10;
+    const minScore = Number.isFinite(opts.minScore) ? Math.max(0, opts.minScore) : 0;
+    const expansionOpts = {
+        enabled: opts.queryExpansion?.enabled ?? true,
+        docs: Math.max(1, opts.queryExpansion?.docs ?? 3),
+        terms: Math.max(1, opts.queryExpansion?.terms ?? 4),
+        weight: Math.max(0, opts.queryExpansion?.weight ?? 0.35),
+        minTermLength: Math.max(2, opts.queryExpansion?.minTermLength ?? 3),
+    };
+    const semanticOpts = {
+        enabled: opts.semantic?.enabled ?? false,
+        mode: opts.semantic?.mode ?? "rerank",
+        topN: Math.max(1, opts.semantic?.topN ?? 50),
+        minLexConfidence: clamp01(opts.semantic?.minLexConfidence ?? 0.35),
+        blend: {
+            enabled: opts.semantic?.blend?.enabled ?? true,
+            wLex: Math.max(0, opts.semantic?.blend?.wLex ?? 0.75),
+            wSem: Math.max(0, opts.semantic?.blend?.wSem ?? 0.25),
+        },
+        queryEmbedding: opts.semantic?.queryEmbedding,
+        force: opts.semantic?.force ?? false,
+    };
     // --- Query parsing
     const normTokens = tokenize(q).map((t) => t.term);
     // Normalize quoted phrases from q
-    const quotedRaw = parsePhrases(q); // arrays of raw terms
-    const quoted = quotedRaw.map(seq => seq.map(t => normalize(t)).flatMap(s => s.split(/\s+/)).filter(Boolean));
+    const quotedRaw = parsePhrases(q);
+    const quoted = quotedRaw.map((seq) => seq.map((t) => normalize(t)).flatMap((s) => s.split(/\s+/)).filter(Boolean));
     // Normalize requirePhrases the same way
     const extraReq = (opts.requirePhrases ?? [])
-        .map(s => tokenize(s).map(t => t.term)) // <<< normalize via tokenizer
-        .filter(arr => arr.length > 0);
+        .map((s) => tokenize(s).map((t) => t.term))
+        .filter((arr) => arr.length > 0);
     const requiredPhrases = [...quoted, ...extraReq];
+    const namespaceFilter = normalizeNamespaceFilter(opts.namespace);
+    const sourceFilter = normalizeSourceFilter(opts.source);
     // --- Term ids for the free (unquoted) tokens in q
     const termIds = normTokens
         .map((t) => pack.lexicon.get(t))
@@ -33,39 +91,53 @@ export function query(pack, q, opts = {}) {
     const termSet = new Set(termIds);
     // --- Candidate map
     const candidates = new Map();
+    // Query-time document frequency collection for BM25 IDF.
+    const dfs = new Map();
+    const usesOffsetBlockIds = (pack.meta?.version ?? 1) >= 3;
     // Helper to harvest postings for a given set of termIds into candidates
-    function scanForTermIds(idSet) {
+    function scanForTermIds(idWeights, cfg = { collectPositions: true, createCandidates: true }) {
         const p = pack.postings;
         let i = 0;
         while (i < p.length) {
             const tid = p[i++];
             if (tid === 0)
                 continue;
-            const relevant = idSet.has(tid);
-            let bid = p[i++];
-            while (bid !== 0) {
+            const weight = idWeights.get(tid) ?? 0;
+            const relevant = weight > 0;
+            let termDf = 0;
+            let encodedBid = p[i++];
+            while (encodedBid !== 0) {
+                const bid = usesOffsetBlockIds ? encodedBid - 1 : encodedBid;
                 let pos = p[i++];
                 const positions = [];
                 while (pos !== 0) {
                     positions.push(pos);
                     pos = p[i++];
                 }
-                if (relevant) {
+                termDf++;
+                if (relevant && bid >= 0) {
                     let entry = candidates.get(bid);
-                    if (!entry) {
+                    if (!entry && cfg.createCandidates !== false) {
                         entry = { tf: new Map(), pos: new Map() };
                         candidates.set(bid, entry);
                     }
-                    entry.tf.set(tid, positions.length);
-                    entry.pos.set(tid, positions);
+                    if (entry) {
+                        const prevTf = entry.tf.get(tid) ?? 0;
+                        entry.tf.set(tid, prevTf + positions.length * weight);
+                        if (cfg.collectPositions !== false) {
+                            entry.pos.set(tid, positions);
+                        }
+                    }
                 }
-                bid = p[i++];
+                encodedBid = p[i++];
             }
+            if (relevant)
+                dfs.set(tid, termDf);
         }
     }
     // 1) Scan using tokens from q (if any)
     if (termSet.size > 0) {
-        scanForTermIds(termSet);
+        scanForTermIds(new Map(Array.from(termSet.values(), (tid) => [tid, 1])));
     }
     // 2) Phrase-first rescue:
     // If nothing matched the free tokens, but we do have required phrases,
@@ -80,7 +152,27 @@ export function query(pack, q, opts = {}) {
             }
         }
         if (phraseTokenIds.size > 0) {
-            scanForTermIds(phraseTokenIds);
+            scanForTermIds(new Map(Array.from(phraseTokenIds.values(), (tid) => [tid, 1])));
+        }
+    }
+    // --- Namespace filtering
+    if (namespaceFilter.size > 0) {
+        for (const bid of [...candidates.keys()]) {
+            const ns = pack.namespaces?.[bid];
+            const normalizedNs = typeof ns === "string" ? normalize(ns) : "";
+            if (!normalizedNs || !namespaceFilter.has(normalizedNs)) {
+                candidates.delete(bid);
+            }
+        }
+    }
+    // --- Source/docId filtering
+    if (sourceFilter.size > 0) {
+        for (const bid of [...candidates.keys()]) {
+            const source = pack.docIds?.[bid];
+            const normalizedSource = typeof source === "string" ? normalize(source) : "";
+            if (!normalizedSource || !sourceFilter.has(normalizedSource)) {
+                candidates.delete(bid);
+            }
         }
     }
     // --- Phrase enforcement (now that we have some candidates)
@@ -119,11 +211,30 @@ export function query(pack, q, opts = {}) {
         (pack.blocks.length
             ? pack.blocks.reduce((s, b) => s + tokenize(b).length, 0) / pack.blocks.length
             : 1);
-    const prelim = rankBM25L(candidates, avgLen, {
+    const docCount = pack.meta?.stats?.blocks ?? pack.blocks.length;
+    let prelim = rankBM25L(candidates, avgLen, docCount, dfs, pack.blockTokenLens, {
         proximityBonus: (cand) => proximityMultiplier(minCoverSpan(cand.pos)),
     });
+    if (expansionOpts.enabled && prelim.length > 0) {
+        const expansionWeights = deriveExpansionTerms(pack, prelim, termSet, requiredPhrases, expansionOpts);
+        if (expansionWeights.size > 0) {
+            scanForTermIds(expansionWeights, { collectPositions: false, createCandidates: true });
+            prelim = rankBM25L(candidates, avgLen, docCount, dfs, pack.blockTokenLens, {
+                proximityBonus: (cand) => proximityMultiplier(minCoverSpan(cand.pos)),
+            });
+        }
+    }
     if (prelim.length === 0)
         return [];
+    if (minScore > 0) {
+        prelim = prelim.filter((item) => item.score >= minScore);
+        if (prelim.length === 0)
+            return [];
+    }
+    const confidence = lexConfidence(prelim);
+    if (shouldRerankWithSemantic(pack, semanticOpts, confidence)) {
+        prelim = rerankLexicalHitsWithSemantic(pack, prelim, semanticOpts);
+    }
     // --- KNS tie-breaker + de-dup/MMR
     const qSig = knsSignature(normalize(q));
     const pool = prelim.slice(0, topK * 5).map((r) => {
@@ -134,17 +245,158 @@ export function query(pack, q, opts = {}) {
             score: r.score * boost,
             text,
             source: pack.docIds?.[r.blockId] ?? undefined,
+            namespace: pack.namespaces?.[r.blockId] ?? undefined,
         };
     });
     const finalHits = diversifyAndDedupe(pool, { k: topK });
     return finalHits;
 }
+export function lexConfidence(hits) {
+    if (hits.length === 0)
+        return 0;
+    const top1 = Math.max(0, hits[0]?.score ?? 0);
+    const top2 = Math.max(0, hits[1]?.score ?? 0);
+    const gapRatio = top1 > 0 ? clamp01((top1 - top2) / top1) : 0;
+    const strength = top1 > 0 ? top1 / (top1 + 1) : 0;
+    return clamp01(0.65 * gapRatio + 0.35 * strength);
+}
+function shouldRerankWithSemantic(pack, opts, confidence) {
+    if (!opts.enabled || opts.mode !== "rerank")
+        return false;
+    if (!pack.semantic)
+        return false;
+    if (!opts.queryEmbedding) {
+        throw new Error("query(...): semantic.queryEmbedding (Float32Array) is required when semantic.enabled=true.");
+    }
+    return opts.force || confidence < opts.minLexConfidence;
+}
+function rerankLexicalHitsWithSemantic(pack, prelim, opts) {
+    const sem = pack.semantic;
+    if (!sem || !opts.queryEmbedding)
+        return prelim;
+    if (sem.dims <= 0 || sem.vecs.length === 0 || sem.dims !== opts.queryEmbedding.length)
+        return prelim;
+    const topN = Math.min(opts.topN, prelim.length);
+    const rerankSlice = prelim.slice(0, topN);
+    const tail = prelim.slice(topN);
+    const lexScores = new Float64Array(topN);
+    for (let i = 0; i < topN; i++)
+        lexScores[i] = rerankSlice[i].score;
+    const normLex = minMaxNormalizeTyped(lexScores);
+    const quantizedQuery = quantizeEmbeddingInt8L2Norm(opts.queryEmbedding);
+    const semScores = scoreSemanticInt8(quantizedQuery.q, quantizedQuery.scale, sem, rerankSlice);
+    const normSem = minMaxNormalizeTyped(semScores);
+    const denom = opts.blend.wLex + opts.blend.wSem;
+    const wLex = denom > 0 ? opts.blend.wLex / denom : 0.5;
+    const wSem = denom > 0 ? opts.blend.wSem / denom : 0.5;
+    const reranked = new Array(topN);
+    for (let i = 0; i < topN; i++) {
+        const hit = rerankSlice[i];
+        reranked[i] = {
+            blockId: hit.blockId,
+            score: opts.blend.enabled ? wLex * normLex[i] + wSem * normSem[i] : semScores[i],
+        };
+    }
+    reranked.sort((a, b) => b.score - a.score || a.blockId - b.blockId);
+    return [...reranked, ...tail];
+}
+function scoreSemanticInt8(queryQ, queryScale, semantic, hits) {
+    const scores = new Float64Array(hits.length);
+    const dims = semantic.dims;
+    if (queryQ.length !== dims || queryScale === 0)
+        return scores;
+    const vecs = semantic.vecs;
+    const scales = semantic.scales;
+    for (let h = 0; h < hits.length; h++) {
+        const blockId = hits[h].blockId;
+        if (blockId < 0)
+            continue;
+        const base = blockId * dims;
+        if (base + dims > vecs.length)
+            continue;
+        let dot = 0;
+        for (let i = 0; i < dims; i++) {
+            dot += queryQ[i] * vecs[base + i];
+        }
+        const blockScale = scales?.[blockId] !== undefined ? decodeScaleF16(scales[blockId]) : (1 / 127);
+        scores[h] = dot * queryScale * blockScale;
+    }
+    return scores;
+}
+function minMaxNormalizeTyped(values) {
+    if (values.length === 0)
+        return values;
+    let min = Infinity;
+    let max = -Infinity;
+    for (let i = 0; i < values.length; i++) {
+        const v = values[i];
+        if (v < min)
+            min = v;
+        if (v > max)
+            max = v;
+    }
+    if (!Number.isFinite(min) || !Number.isFinite(max) || max <= min) {
+        const out = new Float64Array(values.length);
+        out.fill(1);
+        return out;
+    }
+    const out = new Float64Array(values.length);
+    const span = max - min;
+    for (let i = 0; i < values.length; i++) {
+        out[i] = clamp01((values[i] - min) / span);
+    }
+    return out;
+}
+function clamp01(v) {
+    if (!Number.isFinite(v))
+        return 0;
+    if (v < 0)
+        return 0;
+    if (v > 1)
+        return 1;
+    return v;
+}
+function deriveExpansionTerms(pack, prelim, baseTermSet, requiredPhrases, opts) {
+    if (prelim.length === 0 || opts.weight <= 0)
+        return new Map();
+    const forbidden = new Set(baseTermSet);
+    for (const seq of requiredPhrases) {
+        for (const term of seq) {
+            const tid = pack.lexicon.get(term);
+            if (tid !== undefined)
+                forbidden.add(tid);
+        }
+    }
+    const cap = Math.min(opts.docs, prelim.length);
+    const bestScore = Math.max(prelim[0]?.score ?? 0, 1e-6);
+    const termScores = new Map();
+    for (let i = 0; i < cap; i++) {
+        const item = prelim[i];
+        const text = pack.blocks[item.blockId] ?? "";
+        const docWeight = Math.max(item.score / bestScore, 0.2);
+        const localTfs = new Map();
+        for (const tok of tokenize(text)) {
+            if (tok.term.length < opts.minTermLength)
+                continue;
+            const tid = pack.lexicon.get(tok.term);
+            if (tid === undefined || forbidden.has(tid))
+                continue;
+            localTfs.set(tid, (localTfs.get(tid) ?? 0) + 1);
+        }
+        for (const [tid, tf] of localTfs) {
+            termScores.set(tid, (termScores.get(tid) ?? 0) + tf * docWeight);
+        }
+    }
+    const selected = [...termScores.entries()]
+        .sort((a, b) => b[1] - a[1])
+        .slice(0, opts.terms);
+    return new Map(selected.map(([tid, score]) => [tid, opts.weight * Math.max(0.5, Math.min(1.5, score))]));
+}
 /** Ordered phrase check using the SAME tokenizer/normalizer path as the index. */
 function containsPhrase(text, seq) {
     if (seq.length === 0)
         return false;
-    // normalize seq via tokenizer to be extra safe (handles diacritics/case)
-    const seqNorm = tokenize(seq.join(" ")).map(t => t.term);
+    const seqNorm = tokenize(seq.join(" ")).map((t) => t.term);
     const toks = tokenize(text).map((t) => t.term);
     outer: for (let i = 0; i <= toks.length - seqNorm.length; i++) {
         for (let j = 0; j < seqNorm.length; j++) {
@@ -155,3 +407,15 @@ function containsPhrase(text, seq) {
     }
     return false;
 }
+function normalizeNamespaceFilter(input) {
+    if (input === undefined)
+        return new Set();
+    const values = Array.isArray(input) ? input : [input];
+    return new Set(values.map((v) => normalize(v)).filter(Boolean));
+}
+function normalizeSourceFilter(input) {
+    if (input === undefined)
+        return new Set();
+    const values = Array.isArray(input) ? input : [input];
+    return new Set(values.map((v) => normalize(v)).filter(Boolean));
+}

package/dist/rank.d.ts

@@ -15,7 +15,7 @@ export declare function rankBM25L(candidates: Map<number, {
     pos?: Map<number, number[]>;
     hasPhrase?: boolean;
     headingScore?: number;
-}>, avgLen: number, opts?: RankOptions): Array<{
+}>, avgLen: number, docCount: number, dfs: Map<number, number>, blockTokenLens?: number[], opts?: RankOptions): Array<{
     blockId: number;
     score: number;
 }>;

package/dist/rank.js

@@ -2,17 +2,18 @@
  * rank.ts
  * BM25L ranker with optional heading/phrase boosts and a proximity bonus hook.
  */
-export function rankBM25L(candidates, avgLen, opts = {}) {
+export function rankBM25L(candidates, avgLen, docCount, dfs, blockTokenLens, opts = {}) {
     const k1 = opts.k1 ?? 1.5;
     const b = opts.b ?? 0.75;
     const headingBoost = opts.headingBoost ?? 0.3;
     const phraseBoost = opts.phraseBoost ?? 0.6;
     const results = [];
     for (const [bid, data] of candidates) {
-        const len = Array.from(data.tf.values()).reduce((sum, tf) => sum + tf, 0) || 1;
+        const len = blockTokenLens?.[bid] ?? (Array.from(data.tf.values()).reduce((sum, tf) => sum + tf, 0) || 1);
         let score = 0;
-        for (const [, tf] of data.tf) {
-            const idf = 1; // v0: no DF; can be extended later
+        for (const [tid, tf] of data.tf) {
+            const df = dfs.get(tid) ?? 0;
+            const idf = Math.log(1 + (docCount - df + 0.5) / (df + 0.5));
             const numer = tf * (k1 + 1);
             const denom = tf + k1 * (1 - b + b * (len / avgLen));
             score += idf * (numer / denom);

package/dist/semantic.d.ts

@@ -0,0 +1,7 @@
+export type QuantizedVector = {
+    q: Int8Array;
+    scale: number;
+};
+export declare function quantizeEmbeddingInt8L2Norm(embedding: Float32Array): QuantizedVector;
+export declare function encodeScaleF16(scale: number): number;
+export declare function decodeScaleF16(encoded: number): number;

package/dist/semantic.js

@@ -0,0 +1,98 @@
+export function quantizeEmbeddingInt8L2Norm(embedding) {
+    const dims = embedding.length;
+    const normalized = new Float32Array(dims);
+    let normSq = 0;
+    for (let i = 0; i < dims; i++)
+        normSq += embedding[i] * embedding[i];
+    const norm = Math.sqrt(normSq);
+    if (norm === 0) {
+        return { q: new Int8Array(dims), scale: 0 };
+    }
+    let maxAbs = 0;
+    for (let i = 0; i < dims; i++) {
+        const value = embedding[i] / norm;
+        normalized[i] = value;
+        const abs = Math.abs(value);
+        if (abs > maxAbs)
+            maxAbs = abs;
+    }
+    const scale = maxAbs / 127;
+    if (scale === 0) {
+        return { q: new Int8Array(dims), scale: 0 };
+    }
+    const q = new Int8Array(dims);
+    for (let i = 0; i < dims; i++) {
+        const quantized = Math.round(normalized[i] / scale);
+        q[i] = clampInt8(quantized);
+    }
+    return { q, scale };
+}
+export function encodeScaleF16(scale) {
+    return float32ToFloat16(scale);
+}
+export function decodeScaleF16(encoded) {
+    return float16ToFloat32(encoded);
+}
+function clampInt8(value) {
+    if (value > 127)
+        return 127;
+    if (value < -127)
+        return -127;
+    return value;
+}
+function float32ToFloat16(value) {
+    if (Number.isNaN(value))
+        return 0x7e00;
+    if (value === Infinity)
+        return 0x7c00;
+    if (value === -Infinity)
+        return 0xfc00;
+    const f32 = new Float32Array(1);
+    const u32 = new Uint32Array(f32.buffer);
+    f32[0] = value;
+    const bits = u32[0];
+    const sign = (bits >>> 16) & 0x8000;
+    let exp = (bits >>> 23) & 0xff;
+    let mantissa = bits & 0x7fffff;
+    if (exp === 0xff) {
+        return sign | (mantissa ? 0x7e00 : 0x7c00);
+    }
+    const halfExp = exp - 127 + 15;
+    if (halfExp >= 0x1f) {
+        return sign | 0x7c00;
+    }
+    if (halfExp <= 0) {
+        if (halfExp < -10)
+            return sign;
+        mantissa = (mantissa | 0x800000) >>> (1 - halfExp);
+        if (mantissa & 0x1000)
+            mantissa += 0x2000;
+        return sign | (mantissa >>> 13);
+    }
+    if (mantissa & 0x1000) {
+        mantissa += 0x2000;
+        if (mantissa & 0x800000) {
+            mantissa = 0;
+            exp += 1;
+            if (exp > 142)
+                return sign | 0x7c00;
+        }
+    }
+    return sign | (halfExp << 10) | (mantissa >>> 13);
+}
+function float16ToFloat32(bits) {
+    const sign = (bits & 0x8000) ? -1 : 1;
+    const exp = (bits >>> 10) & 0x1f;
+    const frac = bits & 0x03ff;
+    if (exp === 0) {
+        if (frac === 0)
+            return sign * 0;
+        return sign * Math.pow(2, -14) * (frac / 1024);
+    }
+    if (exp === 0x1f) {
+        if (frac === 0)
+            return sign * Infinity;
+        return NaN;
+    }
+    return sign * Math.pow(2, exp - 15) * (1 + frac / 1024);
+}

package/dist/tokenize.js

@@ -42,7 +42,7 @@ export function tokenize(text) {
  */
 export function parsePhrases(q) {
     const parts = [];
-    const regex = /"([^\"]+)"/g;
+    const regex = /["“”]([^"“”]+)["“”]/g;
     let match;
     while ((match = regex.exec(q)) !== null) {
         const phrase = match[1].trim().split(/\s+/);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "knolo-core",
-  "version": "0.2.3",
+  "version": "0.3.0",
   "type": "module",
   "description": "Local-first knowledge packs for small LLMs.",
   "keywords": [
@@ -33,7 +33,8 @@
   "scripts": {
     "build": "tsc -p tsconfig.json",
     "prepublishOnly": "npm run build",
-    "smoke": "node scripts/smoke.mjs"
+    "smoke": "node scripts/smoke.mjs",
+    "test": "npm run build && node scripts/test.mjs"
   },
   "devDependencies": {
     "typescript": "^5.5.0",