knolo-core 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 KnoLo Contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,141 @@
+
+# 🧠 KnoLo Core
+
+KnoLo Core is a **local-first knowledge base system** for small language models (LLMs).
+It lets you package your own documents into a compact `.knolo` file and query them deterministically — **no embeddings, no vector DBs, no cloud**. Perfect for **on-device LLMs**.
+
+---
+
+## ✨ Features
+
+* 📦 **Single-file packs** (`.knolo`) you can ship or load offline.
+* 🔎 **Deterministic lexical retrieval** (BM25L + phrase + heading boosts).
+* ⚡ **Tiny & fast** — runs in Node, browsers, and Expo.
+* 📑 **Context Patches**: structured snippets for direct LLM input.
+* 🔒 **Privacy-first**: all data stays local.
+
+---
+
+## 📦 Install
+
+```bash
+# local build
+npm install
+npm run build
+
+# or if published later
+npm install @knolo/core
+```
+
+---
+
+## 🚀 Usage Examples
+
+### 1. Node.js (in-memory build + query)
+
+```js
+import { buildPack, mountPack, query, makeContextPatch } from "./dist/index.js";
+
+const docs = [
+  { heading: "React Native Bridge", text: "The bridge sends messages between JS and native. You can throttle events to reduce jank." },
+  { heading: "Throttling", text: "Throttling reduces frequency of events to avoid flooding the bridge." },
+  { heading: "Debounce vs Throttle", text: "Debounce waits for silence, throttle guarantees a max rate." }
+];
+
+// Build a pack in memory
+const bytes = await buildPack(docs);
+
+// Mount it
+const kb = await mountPack({ src: bytes });
+
+// Query
+const hits = query(kb, "react native bridge throttling", { topK: 5 });
+console.log("Top hits:", hits);
+
+// Turn into an LLM-friendly context patch
+const patch = makeContextPatch(hits, { budget: "small" });
+console.log("Context Patch:", patch);
+```
+
+---
+
+### 2. CLI (build `.knolo` file)
+
+**Prepare `docs.json`:**
+
+```json
+[
+  { "heading": "Guide", "text": "Install deps.\n\n## Throttle\nLimit frequency of events." },
+  { "heading": "FAQ", "text": "What is throttling? It reduces event frequency." }
+]
+```
+
+**Build the pack:**
+
+```bash
+# writes mypack.knolo
+node bin/knolo.mjs docs.json mypack.knolo
+```
+
+**Query it in a script:**
+
+```js
+import { mountPack, query } from "./dist/index.js";
+
+const kb = await mountPack({ src: "./mypack.knolo" });
+const hits = query(kb, "throttle events", { topK: 3 });
+console.log(hits);
+```
+
+---
+
+### 3. React / Expo (load from asset)
+
+```ts
+import { Asset } from "expo-asset";
+import * as FileSystem from "expo-file-system";
+import { mountPack, query, makeContextPatch } from "@knolo/core";
+
+async function loadKnowledge() {
+  const asset = Asset.fromModule(require("./assets/mypack.knolo"));
+  await asset.downloadAsync();
+
+  const base64 = await FileSystem.readAsStringAsync(asset.localUri!, { encoding: FileSystem.EncodingType.Base64 });
+  const bytes = Uint8Array.from(atob(base64), c => c.charCodeAt(0));
+
+  const kb = await mountPack({ src: bytes.buffer });
+  const hits = query(kb, "bridge throttling", { topK: 5 });
+  return makeContextPatch(hits, { budget: "mini" });
+}
+```
+
+---
+
+## 📑 API Quick Reference
+
+```ts
+// Build a pack from docs
+buildPack([{ heading: string, text: string }[]]) -> Promise<Uint8Array>
+
+// Load a pack
+mountPack({ src: string | Uint8Array | ArrayBuffer }) -> Promise<Pack>
+
+// Query
+query(pack, "your query", { topK?: number, requirePhrases?: string[] }) -> Hit[]
+
+// Create LLM-friendly patch
+makeContextPatch(hits, { budget?: "mini"|"small"|"full" }) -> ContextPatch
+```
+
+---
+
+## 🔮 Roadmap
+
+* Multi-resolution packs (summaries + facts).
+* Overlay store for user notes.
+* WASM core for very large packs.
+
+---
+
+👉 With KnoLo, you can **carry knowledge with your model** — no servers, no dependencies, just a tiny portable pack.
+

package/bin/knolo.mjs

@@ -0,0 +1,30 @@
+#!/usr/bin/env node
+
+// Simple CLI wrapper around the KnoLo pack builder. Reads an input JSON
+// containing an array of documents with `heading` and `text` fields and
+// writes a `.knolo` binary pack. Requires that the compiled `dist` files
+// exist (run `npm run build` before using). This script uses ESM syntax.
+
+import { readFileSync, writeFileSync } from 'node:fs';
+import { buildPack } from '../dist/builder.js';
+
+async function main() {
+  const [,, inputFile, outputFile = 'knowledge.knolo'] = process.argv;
+  if (!inputFile) {
+    console.log('Usage: knolo <input.json> [output.knolo]');
+    process.exit(1);
+  }
+  const json = JSON.parse(readFileSync(inputFile, 'utf8'));
+  if (!Array.isArray(json)) {
+    console.error('Input JSON must be an array of objects');
+    process.exit(1);
+  }
+  const bytes = await buildPack(json);
+  writeFileSync(outputFile, Buffer.from(bytes));
+  console.log(`wrote ${outputFile}`);
+}
+
+main().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

package/dist/builder.d.ts

@@ -0,0 +1,11 @@
+export type BuildInputDoc = {
+    id?: string;
+    heading?: string;
+    text: string;
+};
+/** Build a `.knolo` pack from an array of input documents. At present each
+ * document becomes a single block. Future versions may split documents into
+ * multiple blocks based on headings or token count to improve retrieval
+ * granularity.
+ */
+export declare function buildPack(docs: BuildInputDoc[]): Promise<Uint8Array>;

package/dist/builder.js

@@ -0,0 +1,81 @@
+/*
+ * builder.ts
+ *
+ * Provides a programmatic interface for building `.knolo` knowledge packs
+ * from arrays of input documents. The input format accepts objects with
+ * `id`, `heading` and `text` fields. The builder performs simple
+ * Markdown stripping and calls the indexer to generate the inverted index. The
+ * resulting pack binary can be persisted to disk or served directly to
+ * clients.
+ */
+import { buildIndex } from './indexer.js';
+/** Build a `.knolo` pack from an array of input documents. At present each
+ * document becomes a single block. Future versions may split documents into
+ * multiple blocks based on headings or token count to improve retrieval
+ * granularity.
+ */
+export async function buildPack(docs) {
+    // Convert docs into blocks. For v0 each doc is a single block; heading is
+    // ignored except possibly for future heading boosting in ranking.
+    const blocks = docs.map((d, i) => ({ id: i, text: stripMd(d.text), heading: d.heading }));
+    const { lexicon, postings } = buildIndex(blocks);
+    const meta = {
+        version: 1,
+        stats: { docs: docs.length, blocks: blocks.length, terms: lexicon.length },
+    };
+    // Encode sections to bytes
+    const enc = new TextEncoder();
+    const metaBytes = enc.encode(JSON.stringify(meta));
+    const lexBytes = enc.encode(JSON.stringify(lexicon));
+    const blocksBytes = enc.encode(JSON.stringify(blocks.map((b) => b.text)));
+    // Compute lengths and allocate output
+    const totalLength = 4 + metaBytes.length +
+        4 + lexBytes.length +
+        4 + postings.length * 4 +
+        4 + blocksBytes.length;
+    const out = new Uint8Array(totalLength);
+    const dv = new DataView(out.buffer);
+    let offset = 0;
+    // meta
+    dv.setUint32(offset, metaBytes.length, true);
+    offset += 4;
+    out.set(metaBytes, offset);
+    offset += metaBytes.length;
+    // lexicon
+    dv.setUint32(offset, lexBytes.length, true);
+    offset += 4;
+    out.set(lexBytes, offset);
+    offset += lexBytes.length;
+    // postings
+    dv.setUint32(offset, postings.length, true);
+    offset += 4;
+    new Uint32Array(out.buffer, offset, postings.length).set(postings);
+    offset += postings.length * 4;
+    // blocks
+    dv.setUint32(offset, blocksBytes.length, true);
+    offset += 4;
+    out.set(blocksBytes, offset);
+    return out;
+}
+/** Strip Markdown syntax by converting to HTML and then removing tags. The
+ * `marked` library is used for parsing and rendering. A very naive HTML tag
+ * stripper removes tags by dropping anything between `<` and `>`. This is
+ * simplistic but adequate for plain text extraction.
+ */
+function stripMd(md) {
+    // Remove code fences
+    let text = md.replace(/```[^```]*```/g, ' ');
+    // Remove inline code backticks
+    text = text.replace(/`[^`]*`/g, ' ');
+    // Remove emphasis markers (*, _, ~)
+    text = text.replace(/[\*_~]+/g, ' ');
+    // Remove headings (#)
+    text = text.replace(/^#+\s*/gm, '');
+    // Remove links [text](url) -> text
+    text = text.replace(/\[([^\]]+)\]\([^\)]+\)/g, '$1');
+    // Remove any remaining brackets
+    text = text.replace(/[\[\]()]/g, ' ');
+    // Collapse whitespace
+    text = text.replace(/\s+/g, ' ').trim();
+    return text;
+}

package/dist/index.d.ts

@@ -0,0 +1,7 @@
+export type { MountOptions, PackMeta, Pack } from './pack';
+export type { QueryOptions, Hit } from './query';
+export type { ContextPatch } from './patch';
+export { mountPack } from './pack.js';
+export { query } from './query.js';
+export { makeContextPatch } from './patch.js';
+export { buildPack } from './builder.js';

package/dist/index.js

@@ -0,0 +1,5 @@
+// Public API surface for KnoLo core.
+export { mountPack } from './pack.js';
+export { query } from './query.js';
+export { makeContextPatch } from './patch.js';
+export { buildPack } from './builder.js';

package/dist/indexer.d.ts

@@ -0,0 +1,22 @@
+export type Block = {
+    id: number;
+    text: string;
+    heading?: string;
+};
+export type IndexBuildResult = {
+    lexicon: Array<[string, number]>;
+    postings: Uint32Array;
+};
+/**
+ * Build an inverted index from an array of blocks. The postings list format
+ * encodes, for each term, a header containing the termId, followed by
+ * sequences of blockId and positions for that term, with zeros as delimiters.
+ * The structure looks like:
+ *
+ *     [termId, blockId, pos, pos, 0, blockId, pos, 0, 0, termId, ...]
+ *
+ * Each block section ends with a 0, and each term section ends with a 0. The
+ * entire array can be streamed sequentially without needing to know the sizes
+ * of individual lists ahead of time.
+ */
+export declare function buildIndex(blocks: Block[]): IndexBuildResult;

package/dist/indexer.js

@@ -0,0 +1,70 @@
+/*
+ * indexer.ts
+ *
+ * Implements a basic inverted index builder. Given an array of blocks, it
+ * produces a lexicon mapping each unique term to a term identifier and a
+ * flattened postings array. This representation is intentionally naïve to
+ * prioritise clarity and portability over maximum compression. The pack
+ * builder can later swap this implementation for a more compact format.
+ */
+import { tokenize } from "./tokenize.js";
+/**
+ * Build an inverted index from an array of blocks. The postings list format
+ * encodes, for each term, a header containing the termId, followed by
+ * sequences of blockId and positions for that term, with zeros as delimiters.
+ * The structure looks like:
+ *
+ *     [termId, blockId, pos, pos, 0, blockId, pos, 0, 0, termId, ...]
+ *
+ * Each block section ends with a 0, and each term section ends with a 0. The
+ * entire array can be streamed sequentially without needing to know the sizes
+ * of individual lists ahead of time.
+ */
+export function buildIndex(blocks) {
+    // Map term to termId and interim map of termId -> blockId -> positions
+    const term2id = new Map();
+    const termBlockPositions = new Map();
+    const getTermId = (t) => {
+        let id = term2id.get(t);
+        if (id === undefined) {
+            id = term2id.size + 1; // term IDs start at 1
+            term2id.set(t, id);
+        }
+        return id;
+    };
+    // Build a local term frequency map per block, then populate the global map
+    for (const block of blocks) {
+        const toks = tokenize(block.text);
+        const perTermPositions = new Map();
+        for (const tk of toks) {
+            const id = getTermId(tk.term);
+            let positions = perTermPositions.get(id);
+            if (!positions) {
+                positions = [];
+                perTermPositions.set(id, positions);
+            }
+            positions.push(tk.pos);
+        }
+        // Merge into global structure
+        for (const [tid, positions] of perTermPositions) {
+            let blockMap = termBlockPositions.get(tid);
+            if (!blockMap) {
+                blockMap = new Map();
+                termBlockPositions.set(tid, blockMap);
+            }
+            blockMap.set(block.id, positions);
+        }
+    }
+    // Flatten postings into a single Uint32Array
+    const postings = [];
+    for (const [tid, blockMap] of termBlockPositions) {
+        postings.push(tid);
+        for (const [bid, positions] of blockMap) {
+            postings.push(bid, ...positions, 0);
+        }
+        postings.push(0); // end of term
+    }
+    // Convert lexicon to array for serialization
+    const lexicon = Array.from(term2id.entries());
+    return { lexicon, postings: new Uint32Array(postings) };
+}

package/dist/pack.d.ts

@@ -0,0 +1,47 @@
+export type MountOptions = {
+    src: string | ArrayBufferLike | Uint8Array;
+};
+/** Metadata about the pack. Version numbers should increment with format
+ *  changes, allowing the runtime to adapt accordingly. */
+export type PackMeta = {
+    version: number;
+    stats: {
+        docs: number;
+        blocks: number;
+        terms: number;
+    };
+};
+/**
+ * A mounted pack exposing the inverted index, block text and optional field
+ * metadata. The core runtime reads from these structures directly at query
+ * time.
+ */
+export type Pack = {
+    meta: PackMeta;
+    /** Map of token to term identifier used in the postings list. */
+    lexicon: Map<string, number>;
+    /** Flattened postings list where each term section starts with the termId
+     * followed by (blockId, positions..., 0) tuples, ending with a 0.
+     */
+    postings: Uint32Array;
+    /** Array of block texts. Each block corresponds to a chunk of the original
+     * documents. The blockId used in the postings list indexes into this array.
+     */
+    blocks: string[];
+};
+/**
+ * Load a `.knolo` pack from a variety of sources. The pack binary layout is
+ * currently:
+ *
+ *  [metaLen:u32][meta JSON][lexLen:u32][lexicon JSON][postCount:u32][postings][blocksLen:u32][blocks JSON]
+ *
+ * All integers are little endian. `metaLen`, `lexLen` and `blocksLen` denote
+ * the byte lengths of the subsequent JSON sections. `postCount` is the number
+ * of 32‑bit integers in the postings array. This simple layout is sufficient
+ * for v0 and avoids any additional dependencies beyond standard typed arrays.
+ *
+ * @param opts Options specifying how to load the pack. Accepts a URL string,
+ *        ArrayBuffer, or Uint8Array.
+ * @returns A Promise resolving to a mounted pack with the index and blocks.
+ */
+export declare function mountPack(opts: MountOptions): Promise<Pack>;

package/dist/pack.js

@@ -0,0 +1,77 @@
+/*
+ * pack.ts
+ *
+ * Defines the pack structure and implements the mountPack function that loads
+ * a `.knolo` pack from a variety of sources. A pack consists of a header
+ * containing metadata, followed by JSON‑encoded lexicon and blocks and a
+ * flattened postings list encoded as a 32‑bit integer array. The pack is
+ * portable across Node.js and browser environments.
+ */
+/**
+ * Load a `.knolo` pack from a variety of sources. The pack binary layout is
+ * currently:
+ *
+ *  [metaLen:u32][meta JSON][lexLen:u32][lexicon JSON][postCount:u32][postings][blocksLen:u32][blocks JSON]
+ *
+ * All integers are little endian. `metaLen`, `lexLen` and `blocksLen` denote
+ * the byte lengths of the subsequent JSON sections. `postCount` is the number
+ * of 32‑bit integers in the postings array. This simple layout is sufficient
+ * for v0 and avoids any additional dependencies beyond standard typed arrays.
+ *
+ * @param opts Options specifying how to load the pack. Accepts a URL string,
+ *        ArrayBuffer, or Uint8Array.
+ * @returns A Promise resolving to a mounted pack with the index and blocks.
+ */
+export async function mountPack(opts) {
+    const buf = await resolveToBuffer(opts.src);
+    const dv = new DataView(buf);
+    let offset = 0;
+    // Read meta section
+    const metaLen = dv.getUint32(offset, true);
+    offset += 4;
+    const metaJson = new TextDecoder().decode(new Uint8Array(buf, offset, metaLen));
+    offset += metaLen;
+    const meta = JSON.parse(metaJson);
+    // Read lexicon
+    const lexLen = dv.getUint32(offset, true);
+    offset += 4;
+    const lexJson = new TextDecoder().decode(new Uint8Array(buf, offset, lexLen));
+    offset += lexLen;
+    const lexEntries = JSON.parse(lexJson);
+    const lexicon = new Map(lexEntries);
+    // Read postings
+    const postCount = dv.getUint32(offset, true);
+    offset += 4;
+    const postings = new Uint32Array(buf, offset, postCount);
+    offset += postCount * 4;
+    // Read blocks
+    const blocksLen = dv.getUint32(offset, true);
+    offset += 4;
+    const blocksJson = new TextDecoder().decode(new Uint8Array(buf, offset, blocksLen));
+    const blocks = JSON.parse(blocksJson);
+    return { meta, lexicon, postings, blocks };
+}
+/** Resolve the `src` field of MountOptions into an ArrayBuffer. Supports:
+ *  - strings interpreted as URLs (via fetch)
+ *  - Uint8Array and ArrayBuffer inputs
+ */
+async function resolveToBuffer(src) {
+    if (typeof src === 'string') {
+        // Use fetch for browser and Node environments. For Node this requires the
+        // global fetch API (available since Node 18). Error handling is delegated
+        // to the caller.
+        const res = await fetch(src);
+        const ab = await res.arrayBuffer();
+        return ab;
+    }
+    if (src instanceof Uint8Array) {
+        // If the view covers the whole buffer, return it directly (cast to ArrayBuffer).
+        if (src.byteOffset === 0 && src.byteLength === src.buffer.byteLength) {
+            return src.buffer;
+        }
+        // Otherwise, copy to a new buffer so we return exactly the bytes for this view.
+        const copy = src.slice(); // makes a copy of the bytes for this range
+        return copy.buffer; // typed as ArrayBuffer
+    }
+    return src;
+}

package/dist/patch.d.ts

@@ -0,0 +1,29 @@
+import type { Hit } from "./query.js";
+export type ContextPatch = {
+    background: string[];
+    snippets: Array<{
+        text: string;
+        source?: string;
+    }>;
+    definitions: Array<{
+        term: string;
+        def: string;
+        evidence?: number[];
+    }>;
+    facts: Array<{
+        s: string;
+        p: string;
+        o: string;
+        evidence?: number[];
+    }>;
+};
+/** Assemble a context patch from an array of hits. The `budget` determines
+ * how many snippets and how much text to include in each snippet. Currently
+ * three budgets are supported:
+ *  - `mini`: ~512 token contexts
+ *  - `small`: ~1k token contexts
+ *  - `full`: ~2k token contexts
+ */
+export declare function makeContextPatch(hits: Hit[], opts?: {
+    budget?: 'mini' | 'small' | 'full';
+}): ContextPatch;

package/dist/patch.js

@@ -0,0 +1,50 @@
+/*
+ * patch.ts
+ *
+ * Provides the makeContextPatch function that assembles search results into
+ * structured context patches for consumption by language models. A context
+ * patch includes a background summary and selected snippets. Future versions
+ * may include definitions, facts and other structured artifacts.
+ */
+/** Assemble a context patch from an array of hits. The `budget` determines
+ * how many snippets and how much text to include in each snippet. Currently
+ * three budgets are supported:
+ *  - `mini`: ~512 token contexts
+ *  - `small`: ~1k token contexts
+ *  - `full`: ~2k token contexts
+ */
+export function makeContextPatch(hits, opts = {}) {
+    const budget = opts.budget ?? 'small';
+    const limits = {
+        mini: { snippets: 3, chars: 240 },
+        small: { snippets: 6, chars: 420 },
+        full: { snippets: 10, chars: 900 },
+    };
+    const limit = limits[budget];
+    const snippets = hits.slice(0, limit.snippets).map((h) => ({
+        text: truncate(h.text, limit.chars),
+    }));
+    // Build background summary from first two snippets by extracting first sentence
+    const background = snippets.slice(0, 2).map((s) => firstSentence(s.text));
+    return {
+        background,
+        snippets,
+        definitions: [],
+        facts: [],
+    };
+}
+/** Extract the first sentence from a block of text. If no terminal punctuation
+ * is found, returns the first N characters up to a reasonable length.
+ */
+function firstSentence(text) {
+    const m = text.match(/^(.{10,200}?[.!?])\s/);
+    if (m)
+        return m[1];
+    return text.slice(0, 160);
+}
+/** Truncate text to a maximum length and append an ellipsis if it was
+ * truncated.
+ */
+function truncate(text, maxChars) {
+    return text.length > maxChars ? text.slice(0, maxChars) + '…' : text;
+}

package/dist/query.d.ts

@@ -0,0 +1,18 @@
+import type { Pack } from './pack';
+export type QueryOptions = {
+    topK?: number;
+    /** Additional phrases (unquoted) that must be present in results. */
+    requirePhrases?: string[];
+};
+export type Hit = {
+    blockId: number;
+    score: number;
+    text: string;
+    source?: string;
+};
+/** Execute a search against a mounted pack. The query string can contain
+ * quoted phrases; unquoted terms are treated individually. The `topK`
+ * parameter controls how many results are returned. If `requirePhrases`
+ * contains strings, those phrases must appear verbatim in candidate blocks.
+ */
+export declare function query(pack: Pack, q: string, opts?: QueryOptions): Hit[];

package/dist/query.js

@@ -0,0 +1,98 @@
+/*
+ * query.ts
+ *
+ * Implements the main query function that ties together tokenization, lexical
+ * lookup in the pack's lexicon, scanning the postings list to generate
+ * candidates, and ranking them using the BM25L ranker. Phrase detection
+ * operates on the block text itself and is naive but effective for short
+ * queries. Future versions may incorporate more advanced mechanisms.
+ */
+import { tokenize, parsePhrases } from "./tokenize.js";
+import { rankBM25L } from "./rank.js";
+/** Execute a search against a mounted pack. The query string can contain
+ * quoted phrases; unquoted terms are treated individually. The `topK`
+ * parameter controls how many results are returned. If `requirePhrases`
+ * contains strings, those phrases must appear verbatim in candidate blocks.
+ */
+export function query(pack, q, opts = {}) {
+    const topK = opts.topK ?? 10;
+    const normTokens = tokenize(q).map((t) => t.term);
+    const phraseTerms = parsePhrases(q);
+    // Include explicitly provided phrases
+    if (opts.requirePhrases) {
+        for (const p of opts.requirePhrases) {
+            const terms = p.split(/\s+/).filter(Boolean);
+            if (terms.length > 0)
+                phraseTerms.push(terms);
+        }
+    }
+    // Translate tokens to term IDs. Terms not in the lexicon are skipped.
+    const termIds = normTokens
+        .map((t) => pack.lexicon.get(t))
+        .filter((id) => id !== undefined);
+    // Map from blockId to candidate data (term frequencies, flags)
+    const candidates = new Map();
+    // Scan postings list. The format is [termId, blockId, pos... 0, blockId, pos... 0, 0, termId, ...]
+    const p = pack.postings;
+    let i = 0;
+    while (i < p.length) {
+        const tid = p[i++];
+        if (tid === 0)
+            continue;
+        const relevant = termIds.includes(tid);
+        let bid = p[i++];
+        while (bid !== 0) {
+            let pos = p[i++];
+            const positions = [];
+            while (pos !== 0) {
+                positions.push(pos);
+                pos = p[i++];
+            }
+            if (relevant) {
+                let entry = candidates.get(bid);
+                if (!entry) {
+                    entry = { tf: new Map() };
+                    candidates.set(bid, entry);
+                }
+                // accumulate tf per termId; positions array length is tf
+                entry.tf.set(tid, positions.length);
+            }
+            bid = p[i++];
+        }
+        // end of term section; skip trailing zero already consumed
+    }
+    // Check phrases on candidate texts. If a candidate does not contain all
+    // required phrases, it will be filtered out in ranking.
+    for (const [bid, data] of candidates) {
+        const text = pack.blocks[bid] || '';
+        data.hasPhrase = phraseTerms.some((seq) => containsPhrase(text, seq));
+    }
+    // Compute average block length for ranking normalization
+    const avgLen = pack.blocks.length
+        ? pack.blocks.reduce((sum, b) => sum + tokenize(b).length, 0) / pack.blocks.length
+        : 1;
+    const ranked = rankBM25L(candidates, avgLen);
+    return ranked.slice(0, topK).map((res) => ({
+        blockId: res.blockId,
+        score: res.score,
+        text: pack.blocks[res.blockId] || '',
+    }));
+}
+/** Determine whether the given sequence of terms appears in order within the
+ * text. The algorithm tokenizes the text and performs a sliding window
+ * comparison. This is case‑insensitive and uses the same normalization as
+ * other parts of the system.
+ */
+function containsPhrase(text, seq) {
+    if (seq.length === 0)
+        return false;
+    const toks = tokenize(text).map((t) => t.term);
+    outer: for (let i = 0; i <= toks.length - seq.length; i++) {
+        for (let j = 0; j < seq.length; j++) {
+            if (toks[i + j] !== seq[j])
+                continue outer;
+        }
+        return true;
+    }
+    return false;
+}

package/dist/rank.d.ts

@@ -0,0 +1,20 @@
+export type RankOptions = {
+    k1?: number;
+    b?: number;
+    headingBoost?: number;
+    phraseBoost?: number;
+};
+/**
+ * Rank a set of candidate blocks using BM25L. Each candidate carries a term
+ * frequency (tf) map keyed by termId. Additional properties may include
+ * `hasPhrase` and `headingScore` to apply multiplicative boosts. The average
+ * document length (avgLen) is required for BM25L normalization.
+ */
+export declare function rankBM25L(candidates: Map<number, {
+    tf: Map<number, number>;
+    hasPhrase?: boolean;
+    headingScore?: number;
+}>, avgLen: number, opts?: RankOptions): Array<{
+    blockId: number;
+    score: number;
+}>;

package/dist/rank.js

@@ -0,0 +1,43 @@
+/*
+ * rank.ts
+ *
+ * Implements a simple BM25L ranker with optional boosts for headings and
+ * phrase matches. The inputs to the ranker are a map of block IDs to term
+ * frequency maps and additional boolean flags. The outputs are sorted by
+ * descending relevance score. This ranking algorithm can be replaced or
+ * augmented in the future without impacting the public API of the query
+ * function.
+ */
+/**
+ * Rank a set of candidate blocks using BM25L. Each candidate carries a term
+ * frequency (tf) map keyed by termId. Additional properties may include
+ * `hasPhrase` and `headingScore` to apply multiplicative boosts. The average
+ * document length (avgLen) is required for BM25L normalization.
+ */
+export function rankBM25L(candidates, avgLen, opts = {}) {
+    const k1 = opts.k1 ?? 1.5;
+    const b = opts.b ?? 0.75;
+    const headingBoost = opts.headingBoost ?? 0.3;
+    const phraseBoost = opts.phraseBoost ?? 0.6;
+    const results = [];
+    for (const [bid, data] of candidates) {
+        const len = Array.from(data.tf.values()).reduce((sum, tf) => sum + tf, 0) || 1;
+        let score = 0;
+        for (const [, tf] of data.tf) {
+            const idf = 1; // placeholder; no document frequency available in v0
+            const numer = tf * (k1 + 1);
+            const denom = tf + k1 * (1 - b + b * (len / avgLen));
+            score += idf * (numer / denom);
+        }
+        // Apply boosts multiplicatively
+        if (data.hasPhrase) {
+            score *= 1 + phraseBoost;
+        }
+        if (data.headingScore) {
+            score *= 1 + headingBoost * data.headingScore;
+        }
+        results.push({ blockId: bid, score });
+    }
+    results.sort((a, b2) => b2.score - a.score);
+    return results;
+}

package/dist/tokenize.d.ts

@@ -0,0 +1,24 @@
+export type Token = {
+    term: string;
+    pos: number;
+};
+/** Normalize a string by:
+ *  - Applying NFKD Unicode normalization
+ *  - Stripping combining diacritics
+ *  - Converting to lowercase
+ *  - Replacing non‑alphanumeric characters (except hyphen and space) with space
+ *
+ * This normalization ensures that accents and case do not affect matching.
+ */
+export declare function normalize(s: string): string;
+/** Split a piece of text into tokens with positional information. Each token
+ * contains the normalized term and its position in the sequence. Positions
+ * increment only on actual tokens, ignoring multiple whitespace separators.
+ */
+export declare function tokenize(text: string): Token[];
+/** Parse quoted phrases in a query string. A phrase is a sequence of words
+ * enclosed in double quotes. Returns an array of term arrays representing
+ * each phrase. Single words outside quotes are ignored here and handled
+ * separately by tokenize().
+ */
+export declare function parsePhrases(q: string): string[][];

package/dist/tokenize.js

@@ -0,0 +1,53 @@
+/*
+ * tokenize.ts
+ *
+ * Provides functions for normalizing strings and splitting them into tokens
+ * suitable for indexing and querying. This module deliberately avoids any
+ * language‑specific stemming or lemmatization to keep the core simple and
+ * deterministic across platforms. Basic Unicode normalization and diacritic
+ * stripping are applied to ensure consistent matches.
+ */
+/** Normalize a string by:
+ *  - Applying NFKD Unicode normalization
+ *  - Stripping combining diacritics
+ *  - Converting to lowercase
+ *  - Replacing non‑alphanumeric characters (except hyphen and space) with space
+ *
+ * This normalization ensures that accents and case do not affect matching.
+ */
+export function normalize(s) {
+    return s
+        .normalize('NFKD')
+        .replace(/\p{M}+/gu, '')
+        .toLowerCase()
+        .replace(/[^\p{L}\p{N}\s-]/gu, ' ');
+}
+/** Split a piece of text into tokens with positional information. Each token
+ * contains the normalized term and its position in the sequence. Positions
+ * increment only on actual tokens, ignoring multiple whitespace separators.
+ */
+export function tokenize(text) {
+    const norm = normalize(text);
+    const out = [];
+    let pos = 0;
+    for (const w of norm.split(/\s+/).filter(Boolean)) {
+        out.push({ term: w, pos: pos++ });
+    }
+    return out;
+}
+/** Parse quoted phrases in a query string. A phrase is a sequence of words
+ * enclosed in double quotes. Returns an array of term arrays representing
+ * each phrase. Single words outside quotes are ignored here and handled
+ * separately by tokenize().
+ */
+export function parsePhrases(q) {
+    const parts = [];
+    const regex = /"([^\"]+)"/g;
+    let match;
+    while ((match = regex.exec(q)) !== null) {
+        const phrase = match[1].trim().split(/\s+/);
+        if (phrase.length > 0)
+            parts.push(phrase);
+    }
+    return parts;
+}

package/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "knolo-core",
+  "version": "0.1.0",
+  "description": "Local-first knowledge packs for small LLMs.",
+  "keywords": ["llm", "knowledge-base", "rag", "local", "expo"],
+  "author": "Your Name",
+  "license": "MIT",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": ["dist", "bin", "README.md", "LICENSE"],
+  "bin": { "knolo": "./bin/knolo.mjs" },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "prepublishOnly": "npm run build"
+  },
+  "devDependencies": {
+    "typescript": "^5.5.0",
+    "@types/node": "^20.11.0"
+  }
+}