knolo-core 0.2.3 → 3.1.0

package/dist/builder.js

@@ -1,48 +1,70 @@
 /*
  * builder.ts
  *
- * Build `.knolo` packs from input docs. Now persists optional headings/docIds
- * and stores avgBlockLen in meta for faster/easier normalization at query-time.
+ * Build `.knolo` packs from input docs. Persists headings/docIds/token lengths
+ * and stores avgBlockLen in meta for stable query-time normalization.
  */
 import { buildIndex } from './indexer.js';
 import { tokenize } from './tokenize.js';
 import { getTextEncoder } from './utils/utf8.js';
-export async function buildPack(docs) {
+import { encodeScaleF16, quantizeEmbeddingInt8L2Norm } from './semantic.js';
+import { validateAgentRegistry } from './agent.js';
+export async function buildPack(docs, opts = {}) {
+    const normalizedDocs = validateDocs(docs);
     // Prepare blocks (strip MD) and carry heading/docId for optional boosts.
-    const blocks = docs.map((d, i) => ({
+    const blocks = normalizedDocs.map((d, i) => ({
         id: i,
         text: stripMd(d.text),
         heading: d.heading,
     }));
     // Build index
     const { lexicon, postings } = buildIndex(blocks);
-    // Compute avg token length once (store in meta)
-    const totalTokens = blocks.reduce((sum, b) => sum + tokenize(b.text).length, 0);
+    const blockTokenLens = blocks.map((b) => tokenize(b.text).length);
+    const totalTokens = blockTokenLens.reduce((sum, len) => sum + len, 0);
     const avgBlockLen = blocks.length ? totalTokens / blocks.length : 1;
+    const agents = normalizeAgents(opts.agents);
     const meta = {
-        version: 2,
+        version: 3,
         stats: {
-            docs: docs.length,
+            docs: normalizedDocs.length,
             blocks: blocks.length,
             terms: lexicon.length,
             avgBlockLen,
         },
+        ...(agents ? { agents } : {}),
     };
-    // Persist blocks as objects to optionally carry heading/docId
+    // Persist blocks as objects to optionally carry heading/docId/token length.
     const blocksPayload = blocks.map((b, i) => ({
         text: b.text,
         heading: b.heading ?? null,
-        docId: docs[i]?.id ?? null,
+        docId: normalizedDocs[i]?.id ?? null,
+        namespace: normalizedDocs[i]?.namespace ?? null,
+        len: blockTokenLens[i] ?? 0,
     }));
     // Encode sections
     const enc = getTextEncoder();
     const metaBytes = enc.encode(JSON.stringify(meta));
     const lexBytes = enc.encode(JSON.stringify(lexicon));
     const blocksBytes = enc.encode(JSON.stringify(blocksPayload));
-    const totalLength = 4 + metaBytes.length +
-        4 + lexBytes.length +
-        4 + postings.length * 4 +
-        4 + blocksBytes.length;
+    const semanticEnabled = Boolean(opts.semantic?.enabled);
+    const semanticSection = semanticEnabled && opts.semantic
+        ? buildSemanticSection(blocks.length, opts.semantic)
+        : undefined;
+    const semBytes = semanticSection
+        ? enc.encode(JSON.stringify(semanticSection.semJson))
+        : undefined;
+    const semBlob = semanticSection?.semBlob;
+    const totalLength = 4 +
+        metaBytes.length +
+        4 +
+        lexBytes.length +
+        4 +
+        postings.length * 4 +
+        4 +
+        blocksBytes.length +
+        (semanticEnabled && semBytes && semBlob
+            ? 4 + semBytes.length + 4 + semBlob.length
+            : 0);
     const out = new Uint8Array(totalLength);
     const dv = new DataView(out.buffer);
     let offset = 0;
@@ -67,8 +89,100 @@ export async function buildPack(docs) {
     dv.setUint32(offset, blocksBytes.length, true);
     offset += 4;
     out.set(blocksBytes, offset);
+    offset += blocksBytes.length;
+    if (semanticEnabled && semBytes && semBlob) {
+        dv.setUint32(offset, semBytes.length, true);
+        offset += 4;
+        out.set(semBytes, offset);
+        offset += semBytes.length;
+        dv.setUint32(offset, semBlob.length, true);
+        offset += 4;
+        out.set(semBlob, offset);
+    }
     return out;
 }
+function normalizeAgents(input) {
+    if (!input)
+        return undefined;
+    const registry = Array.isArray(input)
+        ? { version: 1, agents: input }
+        : input;
+    validateAgentRegistry(registry);
+    return registry;
+}
+function buildSemanticSection(blockCount, semantic) {
+    const { embeddings } = semantic;
+    if (!Array.isArray(embeddings) || embeddings.length !== blockCount) {
+        throw new Error(`semantic.embeddings must be provided with one embedding per block (expected ${blockCount}).`);
+    }
+    const quantizationType = semantic.quantization?.type ?? 'int8_l2norm';
+    if (quantizationType !== 'int8_l2norm') {
+        throw new Error(`Unsupported semantic quantization type: ${quantizationType}`);
+    }
+    const dims = embeddings[0]?.length ?? 0;
+    if (!dims)
+        throw new Error('semantic.embeddings must contain vectors with non-zero dimensions.');
+    const vecs = new Int8Array(embeddings.length * dims);
+    const scales = new Uint16Array(embeddings.length);
+    for (let i = 0; i < embeddings.length; i++) {
+        const embedding = embeddings[i];
+        if (!(embedding instanceof Float32Array)) {
+            throw new Error(`semantic.embeddings[${i}] must be a Float32Array.`);
+        }
+        if (embedding.length !== dims) {
+            throw new Error(`semantic.embeddings[${i}] dims mismatch: expected ${dims}, got ${embedding.length}.`);
+        }
+        const { q, scale } = quantizeEmbeddingInt8L2Norm(embedding);
+        vecs.set(q, i * dims);
+        scales[i] = encodeScaleF16(scale);
+    }
+    const vecByteOffset = 0;
+    const vecByteLength = vecs.byteLength;
+    const scalesByteOffset = vecByteLength;
+    const scalesByteLength = scales.byteLength;
+    const semBlob = new Uint8Array(vecByteLength + scalesByteLength);
+    semBlob.set(new Uint8Array(vecs.buffer, vecs.byteOffset, vecByteLength), vecByteOffset);
+    semBlob.set(new Uint8Array(scales.buffer, scales.byteOffset, scalesByteLength), scalesByteOffset);
+    const semJson = {
+        version: 1,
+        modelId: semantic.modelId,
+        dims,
+        encoding: 'int8_l2norm',
+        perVectorScale: true,
+        blocks: {
+            vectors: { byteOffset: vecByteOffset, length: vecs.length },
+            scales: {
+                byteOffset: scalesByteOffset,
+                length: scales.length,
+                encoding: 'float16',
+            },
+        },
+    };
+    return { semJson, semBlob };
+}
+function validateDocs(docs) {
+    if (!Array.isArray(docs)) {
+        throw new Error('buildPack expects an array of docs: [{ text, id?, heading?, namespace? }, ...]');
+    }
+    return docs.map((doc, i) => {
+        if (!doc || typeof doc !== 'object') {
+            throw new Error(`Invalid doc at index ${i}: expected an object with a string "text" field.`);
+        }
+        if (typeof doc.text !== 'string' || !doc.text.trim()) {
+            throw new Error(`Invalid doc at index ${i}: "text" must be a non-empty string.`);
+        }
+        if (doc.id !== undefined && typeof doc.id !== 'string') {
+            throw new Error(`Invalid doc at index ${i}: "id" must be a string when provided.`);
+        }
+        if (doc.heading !== undefined && typeof doc.heading !== 'string') {
+            throw new Error(`Invalid doc at index ${i}: "heading" must be a string when provided.`);
+        }
+        if (doc.namespace !== undefined && typeof doc.namespace !== 'string') {
+            throw new Error(`Invalid doc at index ${i}: "namespace" must be a string when provided.`);
+        }
+        return doc;
+    });
+}
 /** Strip Markdown syntax with lightweight regexes (no deps). */
 function stripMd(md) {
     let text = md.replace(/```[\s\S]*?```/g, ' ');

package/dist/index.d.ts

@@ -1,7 +1,11 @@
-export { mountPack } from './pack.js';
-export { query } from './query.js';
+export { mountPack, hasSemantic } from './pack.js';
+export { query, lexConfidence, validateQueryOptions, validateSemanticQueryOptions, } from './query.js';
 export { makeContextPatch } from './patch.js';
 export { buildPack } from './builder.js';
+export { quantizeEmbeddingInt8L2Norm, encodeScaleF16, decodeScaleF16, } from './semantic.js';
+export { listAgents, getAgent, resolveAgent, buildSystemPrompt, isToolAllowed, assertToolAllowed, validateAgentRegistry, validateAgentDefinition, } from './agent.js';
 export type { MountOptions, PackMeta, Pack } from './pack.js';
 export type { QueryOptions, Hit } from './query.js';
 export type { ContextPatch } from './patch.js';
+export type { BuildInputDoc, BuildPackOptions } from './builder.js';
+export type { AgentPromptTemplate, AgentToolPolicy, AgentRetrievalDefaults, AgentDefinitionV1, AgentRegistry, ResolveAgentInput, ResolvedAgent, } from './agent.js';

package/dist/index.js

@@ -1,5 +1,7 @@
 // src/index.ts
-export { mountPack } from './pack.js';
-export { query } from './query.js';
+export { mountPack, hasSemantic } from './pack.js';
+export { query, lexConfidence, validateQueryOptions, validateSemanticQueryOptions, } from './query.js';
 export { makeContextPatch } from './patch.js';
 export { buildPack } from './builder.js';
+export { quantizeEmbeddingInt8L2Norm, encodeScaleF16, decodeScaleF16, } from './semantic.js';
+export { listAgents, getAgent, resolveAgent, buildSystemPrompt, isToolAllowed, assertToolAllowed, validateAgentRegistry, validateAgentDefinition, } from './agent.js';

package/dist/indexer.d.ts

@@ -13,8 +13,9 @@ export type IndexBuildResult = {
  * sequences of blockId and positions for that term, with zeros as delimiters.
  * The structure looks like:
  *
- *     [termId, blockId, pos, pos, 0, blockId, pos, 0, 0, termId, ...]
+ *     [termId, blockId+1, pos, pos, 0, blockId+1, pos, 0, 0, termId, ...]
  *
+ * Block IDs are stored as bid+1 so that 0 can remain a sentinel delimiter.
  * Each block section ends with a 0, and each term section ends with a 0. The
  * entire array can be streamed sequentially without needing to know the sizes
  * of individual lists ahead of time.

package/dist/indexer.js

@@ -14,8 +14,9 @@ import { tokenize } from "./tokenize.js";
  * sequences of blockId and positions for that term, with zeros as delimiters.
  * The structure looks like:
  *
- *     [termId, blockId, pos, pos, 0, blockId, pos, 0, 0, termId, ...]
+ *     [termId, blockId+1, pos, pos, 0, blockId+1, pos, 0, 0, termId, ...]
  *
+ * Block IDs are stored as bid+1 so that 0 can remain a sentinel delimiter.
  * Each block section ends with a 0, and each term section ends with a 0. The
  * entire array can be streamed sequentially without needing to know the sizes
  * of individual lists ahead of time.
@@ -60,7 +61,7 @@ export function buildIndex(blocks) {
     for (const [tid, blockMap] of termBlockPositions) {
         postings.push(tid);
         for (const [bid, positions] of blockMap) {
-            postings.push(bid, ...positions, 0);
+            postings.push(bid + 1, ...positions, 0);
         }
         postings.push(0); // end of term
     }

package/dist/pack.d.ts

@@ -1,3 +1,4 @@
+import type { AgentRegistry } from './agent.js';
 export type MountOptions = {
     src: string | ArrayBufferLike | Uint8Array;
 };
@@ -9,6 +10,7 @@ export type PackMeta = {
         terms: number;
         avgBlockLen?: number;
     };
+    agents?: AgentRegistry;
 };
 export type Pack = {
     meta: PackMeta;
@@ -17,5 +19,17 @@ export type Pack = {
     blocks: string[];
     headings?: (string | null)[];
     docIds?: (string | null)[];
+    namespaces?: (string | null)[];
+    blockTokenLens?: number[];
+    semantic?: {
+        version: 1;
+        modelId: string;
+        dims: number;
+        encoding: 'int8_l2norm';
+        perVectorScale: boolean;
+        vecs: Int8Array;
+        scales?: Uint16Array;
+    };
 };
+export declare function hasSemantic(pack: Pack): boolean;
 export declare function mountPack(opts: MountOptions): Promise<Pack>;

package/dist/pack.js

@@ -1,12 +1,16 @@
 /*
  * pack.ts
  *
- * Mount `.knolo` packs across Node, browsers, and RN/Expo. Now tolerant of:
- *  - blocks as string[] (v1) or object[] with { text, heading?, docId? } (v2)
+ * Mount `.knolo` packs across Node, browsers, and RN/Expo. Tolerant of:
+ *  - blocks as string[] (v1) or object[] with { text, heading?, docId?, namespace?, len? }
  *  - meta.stats.avgBlockLen (optional)
  * Includes RN/Expo-safe TextDecoder via ponyfill.
  */
 import { getTextDecoder } from './utils/utf8.js';
+import { validateAgentRegistry } from './agent.js';
+export function hasSemantic(pack) {
+    return Boolean(pack.semantic && pack.semantic.dims > 0 && pack.semantic.vecs.length > 0);
+}
 export async function mountPack(opts) {
     const buf = await resolveToBuffer(opts.src);
     const dv = new DataView(buf);
@@ -18,6 +22,9 @@ export async function mountPack(opts) {
     const metaJson = dec.decode(new Uint8Array(buf, offset, metaLen));
     offset += metaLen;
     const meta = JSON.parse(metaJson);
+    if (meta.agents) {
+        validateAgentRegistry(meta.agents);
+    }
     // lexicon
     const lexLen = dv.getUint32(offset, true);
     offset += 4;
@@ -33,14 +40,17 @@ export async function mountPack(opts) {
         postings[i] = dv.getUint32(offset, true);
         offset += 4;
     }
-    // blocks (v1: string[]; v2: {text, heading?, docId?}[])
+    // blocks (v1: string[]; v2/v3: {text, heading?, docId?, namespace?, len?}[])
     const blocksLen = dv.getUint32(offset, true);
     offset += 4;
     const blocksJson = dec.decode(new Uint8Array(buf, offset, blocksLen));
+    offset += blocksLen;
     const parsed = JSON.parse(blocksJson);
     let blocks = [];
     let headings;
     let docIds;
+    let namespaces;
+    let blockTokenLens;
     if (Array.isArray(parsed) && parsed.length && typeof parsed[0] === 'string') {
         // v1
         blocks = parsed;
@@ -49,26 +59,81 @@ export async function mountPack(opts) {
         blocks = [];
         headings = [];
         docIds = [];
+        namespaces = [];
+        blockTokenLens = [];
         for (const it of parsed) {
             if (it && typeof it === 'object') {
                 blocks.push(String(it.text ?? ''));
                 headings.push(it.heading ?? null);
                 docIds.push(it.docId ?? null);
+                namespaces.push(it.namespace ?? null);
+                blockTokenLens.push(typeof it.len === 'number' ? it.len : 0);
             }
             else {
                 blocks.push(String(it ?? ''));
                 headings.push(null);
                 docIds.push(null);
+                namespaces.push(null);
+                blockTokenLens.push(0);
             }
         }
     }
     else {
         blocks = [];
     }
-    return { meta, lexicon, postings, blocks, headings, docIds };
+    let semantic;
+    if (offset < buf.byteLength) {
+        const semLen = dv.getUint32(offset, true);
+        offset += 4;
+        const semJson = dec.decode(new Uint8Array(buf, offset, semLen));
+        offset += semLen;
+        const sem = JSON.parse(semJson);
+        const semBlobLen = dv.getUint32(offset, true);
+        offset += 4;
+        const semBlob = new Uint8Array(buf, offset, semBlobLen);
+        semantic = parseSemanticSection(sem, semBlob);
+    }
+    return {
+        meta,
+        lexicon,
+        postings,
+        blocks,
+        headings,
+        docIds,
+        namespaces,
+        blockTokenLens,
+        semantic,
+    };
+}
+function parseSemanticSection(sem, blob) {
+    const vectors = sem?.blocks?.vectors;
+    const scales = sem?.blocks?.scales;
+    const vecs = new Int8Array(blob.buffer, blob.byteOffset + Number(vectors?.byteOffset ?? 0), Number(vectors?.length ?? 0));
+    let scaleView;
+    if (scales) {
+        const scaleLen = Number(scales.length ?? 0);
+        const scaleOffset = Number(scales.byteOffset ?? 0);
+        const dv = new DataView(blob.buffer, blob.byteOffset + scaleOffset, scaleLen * 2);
+        scaleView = new Uint16Array(scaleLen);
+        for (let i = 0; i < scaleLen; i++) {
+            scaleView[i] = dv.getUint16(i * 2, true);
+        }
+    }
+    return {
+        version: 1,
+        modelId: String(sem?.modelId ?? ''),
+        dims: Number(sem?.dims ?? 0),
+        encoding: 'int8_l2norm',
+        perVectorScale: Boolean(sem?.perVectorScale),
+        vecs,
+        scales: scaleView,
+    };
 }
 async function resolveToBuffer(src) {
     if (typeof src === 'string') {
+        if (isNodeRuntime() && isLikelyLocalPath(src)) {
+            return await readLocalFileAsBuffer(src);
+        }
         const res = await fetch(src);
         return await res.arrayBuffer();
     }
@@ -81,3 +146,30 @@ async function resolveToBuffer(src) {
     }
     return src;
 }
+function isNodeRuntime() {
+    const p = globalThis
+        .process;
+    return !!p?.versions?.node;
+}
+function isLikelyLocalPath(value) {
+    if (value.startsWith('file://'))
+        return true;
+    if (value.startsWith('./') ||
+        value.startsWith('../') ||
+        value.startsWith('/') ||
+        value.startsWith('~'))
+        return true;
+    if (/^[A-Za-z]:[\\/]/.test(value))
+        return true; // Windows absolute path
+    if (/^[A-Za-z][A-Za-z\d+.-]*:/.test(value))
+        return false; // URL scheme
+    return true; // plain relative path like "knowledge.knolo"
+}
+async function readLocalFileAsBuffer(pathOrFileUrl) {
+    const { readFile } = await import('node:fs/promises');
+    const filePath = pathOrFileUrl.startsWith('file://')
+        ? decodeURIComponent(new URL(pathOrFileUrl).pathname)
+        : pathOrFileUrl;
+    const data = await readFile(filePath);
+    return data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength);
+}

package/dist/patch.d.ts

@@ -1,4 +1,4 @@
-import type { Hit } from "./query.js";
+import type { Hit } from './query.js';
 export type ContextPatch = {
     background: string[];
     snippets: Array<{
@@ -17,13 +17,6 @@ export type ContextPatch = {
         evidence?: number[];
     }>;
 };
-/** Assemble a context patch from an array of hits. The `budget` determines
- * how many snippets and how much text to include in each snippet. Currently
- * three budgets are supported:
- *  - `mini`: ~512 token contexts
- *  - `small`: ~1k token contexts
- *  - `full`: ~2k token contexts
- */
 export declare function makeContextPatch(hits: Hit[], opts?: {
     budget?: 'mini' | 'small' | 'full';
 }): ContextPatch;

package/dist/patch.js

@@ -1,17 +1,7 @@
 /*
  * patch.ts
  *
- * Provides the makeContextPatch function that assembles search results into
- * structured context patches for consumption by language models. A context
- * patch includes a background summary and selected snippets. Future versions
- * may include definitions, facts and other structured artifacts.
- */
-/** Assemble a context patch from an array of hits. The `budget` determines
- * how many snippets and how much text to include in each snippet. Currently
- * three budgets are supported:
- *  - `mini`: ~512 token contexts
- *  - `small`: ~1k token contexts
- *  - `full`: ~2k token contexts
+ * Produces a compact, deterministic “context patch” from ranked hits.
  */
 export function makeContextPatch(hits, opts = {}) {
     const budget = opts.budget ?? 'small';
@@ -23,6 +13,7 @@ export function makeContextPatch(hits, opts = {}) {
     const limit = limits[budget];
     const snippets = hits.slice(0, limit.snippets).map((h) => ({
         text: truncate(h.text, limit.chars),
+        source: h.source,
     }));
     // Build background summary from first two snippets by extracting first sentence
     const background = snippets.slice(0, 2).map((s) => firstSentence(s.text));
@@ -33,18 +24,12 @@ export function makeContextPatch(hits, opts = {}) {
         facts: [],
     };
 }
-/** Extract the first sentence from a block of text. If no terminal punctuation
- * is found, returns the first N characters up to a reasonable length.
- */
 function firstSentence(text) {
     const m = text.match(/^(.{10,200}?[.!?])\s/);
     if (m)
         return m[1];
     return text.slice(0, 160);
 }
-/** Truncate text to a maximum length and append an ellipsis if it was
- * truncated.
- */
 function truncate(text, maxChars) {
     return text.length > maxChars ? text.slice(0, maxChars) + '…' : text;
 }

package/dist/query.d.ts

@@ -1,12 +1,41 @@
 import type { Pack } from "./pack.js";
 export type QueryOptions = {
     topK?: number;
+    minScore?: number;
     requirePhrases?: string[];
+    namespace?: string | string[];
+    source?: string | string[];
+    queryExpansion?: {
+        enabled?: boolean;
+        docs?: number;
+        terms?: number;
+        weight?: number;
+        minTermLength?: number;
+    };
+    semantic?: {
+        enabled?: boolean;
+        mode?: "rerank";
+        topN?: number;
+        minLexConfidence?: number;
+        blend?: {
+            enabled?: boolean;
+            wLex?: number;
+            wSem?: number;
+        };
+        queryEmbedding?: Float32Array;
+        force?: boolean;
+    };
 };
+export declare function validateQueryOptions(opts?: QueryOptions): void;
+export declare function validateSemanticQueryOptions(options?: QueryOptions["semantic"]): void;
 export type Hit = {
     blockId: number;
     score: number;
     text: string;
     source?: string;
+    namespace?: string;
 };
 export declare function query(pack: Pack, q: string, opts?: QueryOptions): Hit[];
+export declare function lexConfidence(hits: Array<{
+    score: number;
+}>): number;