knolo-core 0.1.4 → 0.2.1

package/dist/quality/proximity.js

@@ -0,0 +1,31 @@
+// src/quality/proximity.ts
+// Map<termId, positions[]>
+export function minCoverSpan(posMap) {
+    const lists = posMap ? [...posMap.values()].map(arr => arr.slice().sort((a, b) => a - b)) : [];
+    if (lists.length === 0)
+        return null;
+    const idx = new Array(lists.length).fill(0);
+    let best = null;
+    while (true) {
+        const cur = [];
+        for (let i = 0; i < lists.length; i++) {
+            const val = lists[i][idx[i]];
+            if (val === undefined)
+                return best;
+            cur.push(val);
+        }
+        const min = Math.min(...cur);
+        const max = Math.max(...cur);
+        const span = max - min;
+        if (best === null || span < best)
+            best = span;
+        // advance list with current min
+        const minList = cur.indexOf(min);
+        idx[minList]++;
+    }
+}
+export function proximityMultiplier(span, strength = 0.15) {
+    if (span === null)
+        return 1;
+    return 1 + strength / (1 + span); // gentle, bounded
+}

package/dist/quality/signature.d.ts

@@ -0,0 +1,3 @@
+export type KNSSignature = [number, number, number];
+export declare function knsSignature(s: string): KNSSignature;
+export declare function knsDistance(a: KNSSignature, b: KNSSignature): number;

package/dist/quality/signature.js

@@ -0,0 +1,24 @@
+// src/quality/signature.ts
+// "KNS" — simple, deterministic lexical numeric signature for tie-breaking.
+const PRIMES = [257, 263, 269];
+export function knsSignature(s) {
+    let s1 = 0, s2 = 0, s3 = 0;
+    for (let i = 0; i < s.length; i++) {
+        const code = s.charCodeAt(i);
+        s1 = (s1 + code) % PRIMES[0];
+        s2 = (s2 + code * (i + 1)) % PRIMES[1];
+        s3 = (s3 + ((code << 1) ^ (i + 7))) % PRIMES[2];
+    }
+    return [s1, s2, s3];
+}
+export function knsDistance(a, b) {
+    // circular distance on a mod prime, averaged & normalized to 0..1
+    let acc = 0;
+    for (let i = 0; i < PRIMES.length; i++) {
+        const p = PRIMES[i];
+        const diff = Math.abs(a[i] - b[i]);
+        const circ = Math.min(diff, p - diff) / p;
+        acc += circ;
+    }
+    return acc / PRIMES.length;
+}

package/dist/quality/similarity.d.ts

@@ -0,0 +1,3 @@
+export declare function ngramSet(s: string, n?: number): Set<string>;
+export declare function jaccardFromSets(a: Set<string>, b: Set<string>): number;
+export declare function jaccard5(s1: string, s2: string): number;

package/dist/quality/similarity.js

@@ -0,0 +1,27 @@
+// src/quality/similarity.ts
+import { normalize } from '../tokenize.js';
+export function ngramSet(s, n = 5) {
+    const t = normalize(s);
+    const out = new Set();
+    if (t.length < n) {
+        if (t)
+            out.add(t);
+        return out;
+    }
+    for (let i = 0; i <= t.length - n; i++)
+        out.add(t.slice(i, i + n));
+    return out;
+}
+export function jaccardFromSets(a, b) {
+    if (a.size === 0 && b.size === 0)
+        return 1;
+    let inter = 0;
+    for (const x of a)
+        if (b.has(x))
+            inter++;
+    const uni = a.size + b.size - inter;
+    return uni ? inter / uni : 0;
+}
+export function jaccard5(s1, s2) {
+    return jaccardFromSets(ngramSet(s1, 5), ngramSet(s2, 5));
+}

package/dist/query.d.ts

@@ -1,7 +1,6 @@
-import type { Pack } from './pack';
+import type { Pack } from "./pack.js";
 export type QueryOptions = {
     topK?: number;
-    /** Additional phrases (unquoted) that must be present in results. */
     requirePhrases?: string[];
 };
 export type Hit = {
@@ -10,9 +9,4 @@ export type Hit = {
     text: string;
     source?: string;
 };
-/** Execute a search against a mounted pack. The query string can contain
- * quoted phrases; unquoted terms are treated individually. The `topK`
- * parameter controls how many results are returned. If `requirePhrases`
- * contains strings, those phrases must appear verbatim in candidate blocks.
- */
 export declare function query(pack: Pack, q: string, opts?: QueryOptions): Hit[];

package/dist/query.js

@@ -1,95 +1,154 @@
 /*
  * query.ts
  *
- * Implements the main query function that ties together tokenization, lexical
- * lookup in the pack's lexicon, scanning the postings list to generate
- * candidates, and ranking them using the BM25L ranker. Phrase detection
- * operates on the block text itself and is naive but effective for short
- * queries. Future versions may incorporate more advanced mechanisms.
+ * Deterministic, embedding-free retrieval with:
+ *  - REQUIRED phrase enforcement (quoted and requirePhrases)
+ *  - Proximity bonus based on min cover span
+ *  - Optional heading overlap boost
+ *  - KNS numeric-signature tie-breaker (tiny)
+ *  - Near-duplicate suppression + MMR diversity
  */
-import { tokenize, parsePhrases } from "./tokenize.js";
+import { tokenize, parsePhrases, normalize } from "./tokenize.js";
 import { rankBM25L } from "./rank.js";
-/** Execute a search against a mounted pack. The query string can contain
- * quoted phrases; unquoted terms are treated individually. The `topK`
- * parameter controls how many results are returned. If `requirePhrases`
- * contains strings, those phrases must appear verbatim in candidate blocks.
- */
+import { minCoverSpan, proximityMultiplier } from "./quality/proximity.js";
+import { diversifyAndDedupe } from "./quality/diversify.js";
+import { knsSignature, knsDistance } from "./quality/signature.js";
 export function query(pack, q, opts = {}) {
     const topK = opts.topK ?? 10;
+    // --- Query parsing
     const normTokens = tokenize(q).map((t) => t.term);
-    const phraseTerms = parsePhrases(q);
-    // Include explicitly provided phrases
-    if (opts.requirePhrases) {
-        for (const p of opts.requirePhrases) {
-            const terms = p.split(/\s+/).filter(Boolean);
-            if (terms.length > 0)
-                phraseTerms.push(terms);
-        }
-    }
-    // Translate tokens to term IDs. Terms not in the lexicon are skipped.
+    // Normalize quoted phrases from q
+    const quotedRaw = parsePhrases(q); // arrays of raw terms
+    const quoted = quotedRaw.map(seq => seq.map(t => normalize(t)).flatMap(s => s.split(/\s+/)).filter(Boolean));
+    // Normalize requirePhrases the same way
+    const extraReq = (opts.requirePhrases ?? [])
+        .map(s => tokenize(s).map(t => t.term)) // <<< normalize via tokenizer
+        .filter(arr => arr.length > 0);
+    const requiredPhrases = [...quoted, ...extraReq];
+    // --- Term ids for the free (unquoted) tokens in q
     const termIds = normTokens
         .map((t) => pack.lexicon.get(t))
         .filter((id) => id !== undefined);
-    // Map from blockId to candidate data (term frequencies, flags)
+    // If there are no free tokens but there ARE required phrases, we'll fill candidates from phrases later.
+    const termSet = new Set(termIds);
+    // --- Candidate map
     const candidates = new Map();
-    // Scan postings list. The format is [termId, blockId, pos... 0, blockId, pos... 0, 0, termId, ...]
-    const p = pack.postings;
-    let i = 0;
-    while (i < p.length) {
-        const tid = p[i++];
-        if (tid === 0)
-            continue;
-        const relevant = termIds.includes(tid);
-        let bid = p[i++];
-        while (bid !== 0) {
-            let pos = p[i++];
-            const positions = [];
-            while (pos !== 0) {
-                positions.push(pos);
-                pos = p[i++];
-            }
-            if (relevant) {
-                let entry = candidates.get(bid);
-                if (!entry) {
-                    entry = { tf: new Map() };
-                    candidates.set(bid, entry);
+    // Helper to harvest postings for a given set of termIds into candidates
+    function scanForTermIds(idSet) {
+        const p = pack.postings;
+        let i = 0;
+        while (i < p.length) {
+            const tid = p[i++];
+            if (tid === 0)
+                continue;
+            const relevant = idSet.has(tid);
+            let bid = p[i++];
+            while (bid !== 0) {
+                let pos = p[i++];
+                const positions = [];
+                while (pos !== 0) {
+                    positions.push(pos);
+                    pos = p[i++];
                 }
-                // accumulate tf per termId; positions array length is tf
-                entry.tf.set(tid, positions.length);
+                if (relevant) {
+                    let entry = candidates.get(bid);
+                    if (!entry) {
+                        entry = { tf: new Map(), pos: new Map() };
+                        candidates.set(bid, entry);
+                    }
+                    entry.tf.set(tid, positions.length);
+                    entry.pos.set(tid, positions);
+                }
+                bid = p[i++];
+            }
+        }
+    }
+    // 1) Scan using tokens from q (if any)
+    if (termSet.size > 0) {
+        scanForTermIds(termSet);
+    }
+    // 2) Phrase-first rescue:
+    // If nothing matched the free tokens, but we do have required phrases,
+    // build a fallback term set from ALL tokens that appear in those phrases and scan again.
+    if (candidates.size === 0 && requiredPhrases.length > 0) {
+        const phraseTokenIds = new Set();
+        for (const seq of requiredPhrases) {
+            for (const t of seq) {
+                const id = pack.lexicon.get(t);
+                if (id !== undefined)
+                    phraseTokenIds.add(id);
             }
-            bid = p[i++];
         }
-        // end of term section; skip trailing zero already consumed
+        if (phraseTokenIds.size > 0) {
+            scanForTermIds(phraseTokenIds);
+        }
+    }
+    // --- Phrase enforcement (now that we have some candidates)
+    if (requiredPhrases.length > 0) {
+        for (const [bid, data] of [...candidates]) {
+            const text = pack.blocks[bid] || "";
+            const ok = requiredPhrases.every((seq) => containsPhrase(text, seq));
+            if (!ok)
+                candidates.delete(bid);
+            else
+                data.hasPhrase = true;
+        }
     }
-    // Check phrases on candidate texts. If a candidate does not contain all
-    // required phrases, it will be filtered out in ranking.
-    for (const [bid, data] of candidates) {
-        const text = pack.blocks[bid] || '';
-        data.hasPhrase = phraseTerms.some((seq) => containsPhrase(text, seq));
+    else if (quoted.length > 0) {
+        for (const [bid, data] of candidates) {
+            const text = pack.blocks[bid] || "";
+            data.hasPhrase = quoted.some((seq) => containsPhrase(text, seq));
+        }
+    }
+    // If still nothing, bail early
+    if (candidates.size === 0)
+        return [];
+    // --- Heading overlap
+    if (pack.headings?.length) {
+        const qset = new Set(normTokens);
+        const qUniqueCount = new Set(normTokens).size || 1;
+        for (const [bid, data] of candidates) {
+            const h = pack.headings[bid] ?? "";
+            const hTerms = tokenize(h || "").map((t) => t.term);
+            const overlap = new Set(hTerms.filter((t) => qset.has(t))).size;
+            data.headingScore = overlap / qUniqueCount;
+        }
     }
-    // Compute average block length for ranking normalization
-    const avgLen = pack.blocks.length
-        ? pack.blocks.reduce((sum, b) => sum + tokenize(b).length, 0) / pack.blocks.length
-        : 1;
-    const ranked = rankBM25L(candidates, avgLen);
-    return ranked.slice(0, topK).map((res) => ({
-        blockId: res.blockId,
-        score: res.score,
-        text: pack.blocks[res.blockId] || '',
-    }));
+    // --- Rank with proximity bonus
+    const avgLen = pack.meta?.stats?.avgBlockLen ??
+        (pack.blocks.length
+            ? pack.blocks.reduce((s, b) => s + tokenize(b).length, 0) / pack.blocks.length
+            : 1);
+    const prelim = rankBM25L(candidates, avgLen, {
+        proximityBonus: (cand) => proximityMultiplier(minCoverSpan(cand.pos)),
+    });
+    if (prelim.length === 0)
+        return [];
+    // --- KNS tie-breaker + de-dup/MMR
+    const qSig = knsSignature(normalize(q));
+    const pool = prelim.slice(0, topK * 5).map((r) => {
+        const text = pack.blocks[r.blockId] || "";
+        const boost = 1 + 0.02 * (1 - knsDistance(qSig, knsSignature(text)));
+        return {
+            blockId: r.blockId,
+            score: r.score * boost,
+            text,
+            source: pack.docIds?.[r.blockId] ?? undefined,
+        };
+    });
+    const finalHits = diversifyAndDedupe(pool, { k: topK });
+    return finalHits;
 }
-/** Determine whether the given sequence of terms appears in order within the
- * text. The algorithm tokenizes the text and performs a sliding window
- * comparison. This is case‑insensitive and uses the same normalization as
- * other parts of the system.
- */
+/** Ordered phrase check using the SAME tokenizer/normalizer path as the index. */
 function containsPhrase(text, seq) {
     if (seq.length === 0)
         return false;
+    // normalize seq via tokenizer to be extra safe (handles diacritics/case)
+    const seqNorm = tokenize(seq.join(" ")).map(t => t.term);
     const toks = tokenize(text).map((t) => t.term);
-    outer: for (let i = 0; i <= toks.length - seq.length; i++) {
-        for (let j = 0; j < seq.length; j++) {
-            if (toks[i + j] !== seq[j])
+    outer: for (let i = 0; i <= toks.length - seqNorm.length; i++) {
+        for (let j = 0; j < seqNorm.length; j++) {
+            if (toks[i + j] !== seqNorm[j])
                 continue outer;
         }
         return true;

package/dist/rank.d.ts

@@ -3,15 +3,16 @@ export type RankOptions = {
     b?: number;
     headingBoost?: number;
     phraseBoost?: number;
+    proximityBonus?: (cand: {
+        tf: Map<number, number>;
+        pos?: Map<number, number[]>;
+        hasPhrase?: boolean;
+        headingScore?: number;
+    }) => number;
 };
-/**
- * Rank a set of candidate blocks using BM25L. Each candidate carries a term
- * frequency (tf) map keyed by termId. Additional properties may include
- * `hasPhrase` and `headingScore` to apply multiplicative boosts. The average
- * document length (avgLen) is required for BM25L normalization.
- */
 export declare function rankBM25L(candidates: Map<number, {
     tf: Map<number, number>;
+    pos?: Map<number, number[]>;
     hasPhrase?: boolean;
     headingScore?: number;
 }>, avgLen: number, opts?: RankOptions): Array<{

package/dist/rank.js

@@ -1,18 +1,6 @@
 /*
  * rank.ts
- *
- * Implements a simple BM25L ranker with optional boosts for headings and
- * phrase matches. The inputs to the ranker are a map of block IDs to term
- * frequency maps and additional boolean flags. The outputs are sorted by
- * descending relevance score. This ranking algorithm can be replaced or
- * augmented in the future without impacting the public API of the query
- * function.
- */
-/**
- * Rank a set of candidate blocks using BM25L. Each candidate carries a term
- * frequency (tf) map keyed by termId. Additional properties may include
- * `hasPhrase` and `headingScore` to apply multiplicative boosts. The average
- * document length (avgLen) is required for BM25L normalization.
+ * BM25L ranker with optional heading/phrase boosts and a proximity bonus hook.
  */
 export function rankBM25L(candidates, avgLen, opts = {}) {
     const k1 = opts.k1 ?? 1.5;
@@ -24,18 +12,17 @@ export function rankBM25L(candidates, avgLen, opts = {}) {
         const len = Array.from(data.tf.values()).reduce((sum, tf) => sum + tf, 0) || 1;
         let score = 0;
         for (const [, tf] of data.tf) {
-            const idf = 1; // placeholder; no document frequency available in v0
+            const idf = 1; // v0: no DF; can be extended later
             const numer = tf * (k1 + 1);
             const denom = tf + k1 * (1 - b + b * (len / avgLen));
             score += idf * (numer / denom);
         }
-        // Apply boosts multiplicatively
-        if (data.hasPhrase) {
+        if (opts.proximityBonus)
+            score *= opts.proximityBonus(data) ?? 1;
+        if (data.hasPhrase)
             score *= 1 + phraseBoost;
-        }
-        if (data.headingScore) {
+        if (data.headingScore)
             score *= 1 + headingBoost * data.headingScore;
-        }
         results.push({ blockId: bid, score });
     }
     results.sort((a, b2) => b2.score - a.score);

package/dist/utils/utf8.d.ts

@@ -0,0 +1,8 @@
+export type TextDecoderLike = {
+    decode: (u8: Uint8Array) => string;
+};
+export type TextEncoderLike = {
+    encode: (s: string) => Uint8Array;
+};
+export declare function getTextDecoder(): TextDecoderLike;
+export declare function getTextEncoder(): TextEncoderLike;

package/dist/utils/utf8.js

@@ -0,0 +1,72 @@
+// src/utils/utf8.ts
+// Small, dependency-free UTF-8 encoder/decoder that works in RN/Hermes.
+export function getTextDecoder() {
+    try {
+        // eslint-disable-next-line no-new
+        const td = new TextDecoder();
+        return td;
+    }
+    catch {
+        return {
+            decode: (u8) => {
+                let out = '';
+                for (let i = 0; i < u8.length;) {
+                    const a = u8[i++];
+                    if (a < 0x80) {
+                        out += String.fromCharCode(a);
+                    }
+                    else if ((a & 0xe0) === 0xc0) {
+                        const b = u8[i++] & 0x3f;
+                        const cp = ((a & 0x1f) << 6) | b;
+                        out += String.fromCharCode(cp);
+                    }
+                    else if ((a & 0xf0) === 0xe0) {
+                        const b = u8[i++] & 0x3f;
+                        const c = u8[i++] & 0x3f;
+                        const cp = ((a & 0x0f) << 12) | (b << 6) | c;
+                        out += String.fromCharCode(cp);
+                    }
+                    else {
+                        const b = u8[i++] & 0x3f;
+                        const c = u8[i++] & 0x3f;
+                        const d = u8[i++] & 0x3f;
+                        let cp = ((a & 0x07) << 18) | (b << 12) | (c << 6) | d;
+                        cp -= 0x10000;
+                        out += String.fromCharCode(0xd800 + (cp >> 10), 0xdc00 + (cp & 0x3ff));
+                    }
+                }
+                return out;
+            },
+        };
+    }
+}
+export function getTextEncoder() {
+    try {
+        // eslint-disable-next-line no-new
+        const te = new TextEncoder();
+        return te;
+    }
+    catch {
+        return {
+            encode: (s) => {
+                const out = [];
+                for (let i = 0; i < s.length; i++) {
+                    let cp = s.charCodeAt(i);
+                    if (cp >= 0xd800 && cp <= 0xdbff && i + 1 < s.length) {
+                        const next = s.charCodeAt(++i);
+                        cp = 0x10000 + ((cp - 0xd800) << 10) + (next - 0xdc00);
+                    }
+                    if (cp < 0x80)
+                        out.push(cp);
+                    else if (cp < 0x800)
+                        out.push(0xc0 | (cp >> 6), 0x80 | (cp & 0x3f));
+                    else if (cp < 0x10000)
+                        out.push(0xe0 | (cp >> 12), 0x80 | ((cp >> 6) & 0x3f), 0x80 | (cp & 0x3f));
+                    else
+                        out.push(0xf0 | (cp >> 18), 0x80 | ((cp >> 12) & 0x3f), 0x80 | ((cp >> 6) & 0x3f), 0x80 | (cp & 0x3f));
+                }
+                return new Uint8Array(out);
+            },
+        };
+    }
+}

package/package.json

@@ -1,15 +1,29 @@
 {
   "name": "knolo-core",
-  "version": "0.1.4",
+  "version": "0.2.1",
   "type": "module",
   "description": "Local-first knowledge packs for small LLMs.",
-  "keywords": ["llm", "knowledge-base", "rag", "local", "expo"],
-  "author": "Your Name",
-  "license": "MIT",
+  "keywords": [
+    "llm",
+    "knowledge-base",
+    "rag",
+    "local",
+    "expo"
+  ],
+  "author": "Sam Paniagua",
+  "license": "Apache-2.0",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
-  "files": ["dist", "bin", "README.md", "LICENSE"],
-  "bin": { "knolo": "./bin/knolo.mjs" },
+  "files": [
+    "dist",
+    "bin",
+    "README.md",
+    "LICENSE",
+    "DOCS.md"
+  ],
+  "bin": {
+    "knolo": "./bin/knolo.mjs"
+  },
   "exports": {
     ".": {
       "import": "./dist/index.js",
@@ -18,7 +32,8 @@
   },
   "scripts": {
     "build": "tsc -p tsconfig.json",
-    "prepublishOnly": "npm run build"
+    "prepublishOnly": "npm run build",
+    "smoke": "node scripts/smoke.mjs"
   },
   "devDependencies": {
     "typescript": "^5.5.0",