@knolo/core 3.1.2

package/dist/pack.js

@@ -0,0 +1,175 @@
+/*
+ * pack.ts
+ *
+ * Mount `.knolo` packs across Node, browsers, and RN/Expo. Tolerant of:
+ *  - blocks as string[] (v1) or object[] with { text, heading?, docId?, namespace?, len? }
+ *  - meta.stats.avgBlockLen (optional)
+ * Includes RN/Expo-safe TextDecoder via ponyfill.
+ */
+import { getTextDecoder } from './utils/utf8.js';
+import { validateAgentRegistry } from './agent.js';
+export function hasSemantic(pack) {
+    return Boolean(pack.semantic && pack.semantic.dims > 0 && pack.semantic.vecs.length > 0);
+}
+export async function mountPack(opts) {
+    const buf = await resolveToBuffer(opts.src);
+    const dv = new DataView(buf);
+    const dec = getTextDecoder();
+    let offset = 0;
+    // meta
+    const metaLen = dv.getUint32(offset, true);
+    offset += 4;
+    const metaJson = dec.decode(new Uint8Array(buf, offset, metaLen));
+    offset += metaLen;
+    const meta = JSON.parse(metaJson);
+    if (meta.agents) {
+        validateAgentRegistry(meta.agents);
+    }
+    // lexicon
+    const lexLen = dv.getUint32(offset, true);
+    offset += 4;
+    const lexJson = dec.decode(new Uint8Array(buf, offset, lexLen));
+    offset += lexLen;
+    const lexEntries = JSON.parse(lexJson);
+    const lexicon = new Map(lexEntries);
+    // postings
+    const postCount = dv.getUint32(offset, true);
+    offset += 4;
+    const postings = new Uint32Array(postCount);
+    for (let i = 0; i < postCount; i++) {
+        postings[i] = dv.getUint32(offset, true);
+        offset += 4;
+    }
+    // blocks (v1: string[]; v2/v3: {text, heading?, docId?, namespace?, len?}[])
+    const blocksLen = dv.getUint32(offset, true);
+    offset += 4;
+    const blocksJson = dec.decode(new Uint8Array(buf, offset, blocksLen));
+    offset += blocksLen;
+    const parsed = JSON.parse(blocksJson);
+    let blocks = [];
+    let headings;
+    let docIds;
+    let namespaces;
+    let blockTokenLens;
+    if (Array.isArray(parsed) && parsed.length && typeof parsed[0] === 'string') {
+        // v1
+        blocks = parsed;
+    }
+    else if (Array.isArray(parsed)) {
+        blocks = [];
+        headings = [];
+        docIds = [];
+        namespaces = [];
+        blockTokenLens = [];
+        for (const it of parsed) {
+            if (it && typeof it === 'object') {
+                blocks.push(String(it.text ?? ''));
+                headings.push(it.heading ?? null);
+                docIds.push(it.docId ?? null);
+                namespaces.push(it.namespace ?? null);
+                blockTokenLens.push(typeof it.len === 'number' ? it.len : 0);
+            }
+            else {
+                blocks.push(String(it ?? ''));
+                headings.push(null);
+                docIds.push(null);
+                namespaces.push(null);
+                blockTokenLens.push(0);
+            }
+        }
+    }
+    else {
+        blocks = [];
+    }
+    let semantic;
+    if (offset < buf.byteLength) {
+        const semLen = dv.getUint32(offset, true);
+        offset += 4;
+        const semJson = dec.decode(new Uint8Array(buf, offset, semLen));
+        offset += semLen;
+        const sem = JSON.parse(semJson);
+        const semBlobLen = dv.getUint32(offset, true);
+        offset += 4;
+        const semBlob = new Uint8Array(buf, offset, semBlobLen);
+        semantic = parseSemanticSection(sem, semBlob);
+    }
+    return {
+        meta,
+        lexicon,
+        postings,
+        blocks,
+        headings,
+        docIds,
+        namespaces,
+        blockTokenLens,
+        semantic,
+    };
+}
+function parseSemanticSection(sem, blob) {
+    const vectors = sem?.blocks?.vectors;
+    const scales = sem?.blocks?.scales;
+    const vecs = new Int8Array(blob.buffer, blob.byteOffset + Number(vectors?.byteOffset ?? 0), Number(vectors?.length ?? 0));
+    let scaleView;
+    if (scales) {
+        const scaleLen = Number(scales.length ?? 0);
+        const scaleOffset = Number(scales.byteOffset ?? 0);
+        const dv = new DataView(blob.buffer, blob.byteOffset + scaleOffset, scaleLen * 2);
+        scaleView = new Uint16Array(scaleLen);
+        for (let i = 0; i < scaleLen; i++) {
+            scaleView[i] = dv.getUint16(i * 2, true);
+        }
+    }
+    return {
+        version: 1,
+        modelId: String(sem?.modelId ?? ''),
+        dims: Number(sem?.dims ?? 0),
+        encoding: 'int8_l2norm',
+        perVectorScale: Boolean(sem?.perVectorScale),
+        vecs,
+        scales: scaleView,
+    };
+}
+async function resolveToBuffer(src) {
+    if (typeof src === 'string') {
+        if (isNodeRuntime() && isLikelyLocalPath(src)) {
+            return await readLocalFileAsBuffer(src);
+        }
+        const res = await fetch(src);
+        return await res.arrayBuffer();
+    }
+    if (src instanceof Uint8Array) {
+        if (src.byteOffset === 0 && src.byteLength === src.buffer.byteLength) {
+            return src.buffer;
+        }
+        const copy = src.slice();
+        return copy.buffer;
+    }
+    return src;
+}
+function isNodeRuntime() {
+    const p = globalThis
+        .process;
+    return !!p?.versions?.node;
+}
+function isLikelyLocalPath(value) {
+    if (value.startsWith('file://'))
+        return true;
+    if (value.startsWith('./') ||
+        value.startsWith('../') ||
+        value.startsWith('/') ||
+        value.startsWith('~'))
+        return true;
+    if (/^[A-Za-z]:[\\/]/.test(value))
+        return true; // Windows absolute path
+    if (/^[A-Za-z][A-Za-z\d+.-]*:/.test(value))
+        return false; // URL scheme
+    return true; // plain relative path like "knowledge.knolo"
+}
+async function readLocalFileAsBuffer(pathOrFileUrl) {
+    const { readFile } = await import('node:fs/promises');
+    const filePath = pathOrFileUrl.startsWith('file://')
+        ? decodeURIComponent(new URL(pathOrFileUrl).pathname)
+        : pathOrFileUrl;
+    const data = await readFile(filePath);
+    return data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength);
+}

package/dist/patch.d.ts

@@ -0,0 +1,22 @@
+import type { Hit } from './query.js';
+export type ContextPatch = {
+    background: string[];
+    snippets: Array<{
+        text: string;
+        source?: string;
+    }>;
+    definitions: Array<{
+        term: string;
+        def: string;
+        evidence?: number[];
+    }>;
+    facts: Array<{
+        s: string;
+        p: string;
+        o: string;
+        evidence?: number[];
+    }>;
+};
+export declare function makeContextPatch(hits: Hit[], opts?: {
+    budget?: 'mini' | 'small' | 'full';
+}): ContextPatch;

package/dist/patch.js

@@ -0,0 +1,35 @@
+/*
+ * patch.ts
+ *
+ * Produces a compact, deterministic “context patch” from ranked hits.
+ */
+export function makeContextPatch(hits, opts = {}) {
+    const budget = opts.budget ?? 'small';
+    const limits = {
+        mini: { snippets: 3, chars: 240 },
+        small: { snippets: 6, chars: 420 },
+        full: { snippets: 10, chars: 900 },
+    };
+    const limit = limits[budget];
+    const snippets = hits.slice(0, limit.snippets).map((h) => ({
+        text: truncate(h.text, limit.chars),
+        source: h.source,
+    }));
+    // Build background summary from first two snippets by extracting first sentence
+    const background = snippets.slice(0, 2).map((s) => firstSentence(s.text));
+    return {
+        background,
+        snippets,
+        definitions: [],
+        facts: [],
+    };
+}
+function firstSentence(text) {
+    const m = text.match(/^(.{10,200}?[.!?])\s/);
+    if (m)
+        return m[1];
+    return text.slice(0, 160);
+}
+function truncate(text, maxChars) {
+    return text.length > maxChars ? text.slice(0, maxChars) + '…' : text;
+}

package/dist/quality/diversify.d.ts

@@ -0,0 +1,13 @@
+export type HitLike = {
+    blockId: number;
+    score: number;
+    text: string;
+    source?: string;
+};
+export type DiversifyOptions = {
+    k: number;
+    lambda?: number;
+    simThreshold?: number;
+    sim?: (a: HitLike, b: HitLike) => number;
+};
+export declare function diversifyAndDedupe(hits: HitLike[], opts: DiversifyOptions): HitLike[];

package/dist/quality/diversify.js

@@ -0,0 +1,41 @@
+// src/quality/diversify.ts
+import { jaccard5 } from './similarity.js';
+export function diversifyAndDedupe(hits, opts) {
+    const { k, lambda = 0.8, simThreshold = 0.92, sim = (a, b) => jaccard5(a.text, b.text) } = opts;
+    const pool = [...hits].sort((a, b) => b.score - a.score);
+    const kept = [];
+    while (pool.length && kept.length < k) {
+        // compute MMR for current pool against kept
+        let bestIdx = 0;
+        let bestMMR = -Infinity;
+        for (let i = 0; i < pool.length; i++) {
+            const h = pool[i];
+            let maxSim = 0;
+            for (const s of kept) {
+                const v = sim(h, s);
+                if (v > maxSim)
+                    maxSim = v;
+                if (v >= simThreshold) {
+                    maxSim = v;
+                    break;
+                } // early out
+            }
+            // skip near-duplicates
+            if (maxSim >= simThreshold)
+                continue;
+            const mmr = lambda * h.score - (1 - lambda) * maxSim;
+            if (mmr > bestMMR) {
+                bestMMR = mmr;
+                bestIdx = i;
+            }
+        }
+        // if everything was a near-duplicate, just take the next best by score
+        const pick = pool.splice(bestMMR === -Infinity ? 0 : bestIdx, 1)[0];
+        if (!pick)
+            break;
+        // final dedupe check before push
+        if (!kept.some((x) => sim(x, pick) >= simThreshold))
+            kept.push(pick);
+    }
+    return kept;
+}

package/dist/quality/proximity.d.ts

@@ -0,0 +1,2 @@
+export declare function minCoverSpan(posMap?: Map<number, number[]>): number | null;
+export declare function proximityMultiplier(span: number | null, strength?: number): number;

package/dist/quality/proximity.js

@@ -0,0 +1,31 @@
+// src/quality/proximity.ts
+// Map<termId, positions[]>
+export function minCoverSpan(posMap) {
+    const lists = posMap ? [...posMap.values()].map(arr => arr.slice().sort((a, b) => a - b)) : [];
+    if (lists.length === 0)
+        return null;
+    const idx = new Array(lists.length).fill(0);
+    let best = null;
+    while (true) {
+        const cur = [];
+        for (let i = 0; i < lists.length; i++) {
+            const val = lists[i][idx[i]];
+            if (val === undefined)
+                return best;
+            cur.push(val);
+        }
+        const min = Math.min(...cur);
+        const max = Math.max(...cur);
+        const span = max - min;
+        if (best === null || span < best)
+            best = span;
+        // advance list with current min
+        const minList = cur.indexOf(min);
+        idx[minList]++;
+    }
+}
+export function proximityMultiplier(span, strength = 0.15) {
+    if (span === null)
+        return 1;
+    return 1 + strength / (1 + span); // gentle, bounded
+}

package/dist/quality/signature.d.ts

@@ -0,0 +1,3 @@
+export type KNSSignature = [number, number, number];
+export declare function knsSignature(s: string): KNSSignature;
+export declare function knsDistance(a: KNSSignature, b: KNSSignature): number;

package/dist/quality/signature.js

@@ -0,0 +1,24 @@
+// src/quality/signature.ts
+// "KNS" — simple, deterministic lexical numeric signature for tie-breaking.
+const PRIMES = [257, 263, 269];
+export function knsSignature(s) {
+    let s1 = 0, s2 = 0, s3 = 0;
+    for (let i = 0; i < s.length; i++) {
+        const code = s.charCodeAt(i);
+        s1 = (s1 + code) % PRIMES[0];
+        s2 = (s2 + code * (i + 1)) % PRIMES[1];
+        s3 = (s3 + ((code << 1) ^ (i + 7))) % PRIMES[2];
+    }
+    return [s1, s2, s3];
+}
+export function knsDistance(a, b) {
+    // circular distance on a mod prime, averaged & normalized to 0..1
+    let acc = 0;
+    for (let i = 0; i < PRIMES.length; i++) {
+        const p = PRIMES[i];
+        const diff = Math.abs(a[i] - b[i]);
+        const circ = Math.min(diff, p - diff) / p;
+        acc += circ;
+    }
+    return acc / PRIMES.length;
+}

package/dist/quality/similarity.d.ts

@@ -0,0 +1,3 @@
+export declare function ngramSet(s: string, n?: number): Set<string>;
+export declare function jaccardFromSets(a: Set<string>, b: Set<string>): number;
+export declare function jaccard5(s1: string, s2: string): number;

package/dist/quality/similarity.js

@@ -0,0 +1,27 @@
+// src/quality/similarity.ts
+import { normalize } from '../tokenize.js';
+export function ngramSet(s, n = 5) {
+    const t = normalize(s);
+    const out = new Set();
+    if (t.length < n) {
+        if (t)
+            out.add(t);
+        return out;
+    }
+    for (let i = 0; i <= t.length - n; i++)
+        out.add(t.slice(i, i + n));
+    return out;
+}
+export function jaccardFromSets(a, b) {
+    if (a.size === 0 && b.size === 0)
+        return 1;
+    let inter = 0;
+    for (const x of a)
+        if (b.has(x))
+            inter++;
+    const uni = a.size + b.size - inter;
+    return uni ? inter / uni : 0;
+}
+export function jaccard5(s1, s2) {
+    return jaccardFromSets(ngramSet(s1, 5), ngramSet(s2, 5));
+}

package/dist/query.d.ts

@@ -0,0 +1,41 @@
+import type { Pack } from "./pack.js";
+export type QueryOptions = {
+    topK?: number;
+    minScore?: number;
+    requirePhrases?: string[];
+    namespace?: string | string[];
+    source?: string | string[];
+    queryExpansion?: {
+        enabled?: boolean;
+        docs?: number;
+        terms?: number;
+        weight?: number;
+        minTermLength?: number;
+    };
+    semantic?: {
+        enabled?: boolean;
+        mode?: "rerank";
+        topN?: number;
+        minLexConfidence?: number;
+        blend?: {
+            enabled?: boolean;
+            wLex?: number;
+            wSem?: number;
+        };
+        queryEmbedding?: Float32Array;
+        force?: boolean;
+    };
+};
+export declare function validateQueryOptions(opts?: QueryOptions): void;
+export declare function validateSemanticQueryOptions(options?: QueryOptions["semantic"]): void;
+export type Hit = {
+    blockId: number;
+    score: number;
+    text: string;
+    source?: string;
+    namespace?: string;
+};
+export declare function query(pack: Pack, q: string, opts?: QueryOptions): Hit[];
+export declare function lexConfidence(hits: Array<{
+    score: number;
+}>): number;