@sid-ai/sid-sdk 0.1.0

package/dist/cjs/cache.d.ts

@@ -0,0 +1,47 @@
+import { type CharacterRange, type RangeMode } from "./ranges";
+import { DocumentView, DocumentViewWithSnippet, type Document } from "./rendering";
+import { type Language } from "./snippet";
+export declare const SNIPPET_SIZE_DEFAULT = 50;
+export declare const MIN_SEEN_OVERLAP_DEFAULT = 100;
+export interface DocumentCacheOptions {
+    language?: Language;
+    rangeMode?: RangeMode;
+}
+export interface ApplySnippetOptions {
+    snippetField: string;
+    query: string;
+    snippetSize?: number;
+    minSeenOverlap?: number;
+    displayFields?: readonly string[];
+    language?: Language;
+}
+export interface SingleSpanOptions {
+    snippetField?: string;
+    snippetDisplaySpan?: CharacterRange;
+    displayFields?: readonly string[];
+}
+export declare class DocumentCache<T extends Document = Document> {
+    readonly language: Language;
+    readonly rangeMode: RangeMode;
+    readonly seenLedger: Map<string, CharacterRange[]>;
+    private family;
+    constructor({ language, rangeMode }?: DocumentCacheOptions);
+    addDocument(dataId: string, document: T): string;
+    contains(dataId: string): boolean;
+    containsModelFacingId(modelId: string): boolean;
+    toDataId(modelId: string): string;
+    toModelFacingId(dataId: string): string;
+    getDocument(dataId: string): T;
+    getDocumentFromModelFacingId(modelId: string): T;
+    private content;
+    resolveCharRange(dataId: string, snippetField: string, range: CharacterRange): CharacterRange;
+    /** Override in a subclass to select a different span. Returned spans are strictly validated. */
+    snippetFn(query: string, content: string, snippetSize: number, language: Language): CharacterRange;
+    applySnippet(dataId: string, { snippetField, query, snippetSize, minSeenOverlap, displayFields, language }: ApplySnippetOptions): DocumentViewWithSnippet<T>;
+    getSingleSpanDocumentView(dataId: string, options: SingleSpanOptions & {
+        snippetField: string;
+    }): DocumentViewWithSnippet<T>;
+    getSingleSpanDocumentView(dataId: string, options?: SingleSpanOptions): DocumentView<T>;
+    updateSeen(view: DocumentView): void;
+    fork(n: number): DocumentCache<T>[];
+}

package/dist/cjs/cache.js

@@ -0,0 +1,120 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DocumentCache = exports.MIN_SEEN_OVERLAP_DEFAULT = exports.SNIPPET_SIZE_DEFAULT = void 0;
+const id_stream_1 = require("./id-stream");
+const intervals_1 = require("./intervals");
+const ranges_1 = require("./ranges");
+const rendering_1 = require("./rendering");
+const snippet_1 = require("./snippet");
+exports.SNIPPET_SIZE_DEFAULT = 50;
+exports.MIN_SEEN_OVERLAP_DEFAULT = 100;
+class DocumentCache {
+    language;
+    rangeMode;
+    seenLedger = new Map();
+    family;
+    constructor({ language = "english", rangeMode = "lenient" } = {}) {
+        this.language = (0, snippet_1.validateLanguage)(language);
+        this.rangeMode = (0, ranges_1.validateRangeMode)(rangeMode);
+        this.family = { documents: new Map(), toData: new Map(), toModel: new Map(), ids: new id_stream_1.IdStream() };
+    }
+    addDocument(dataId, document) {
+        if (typeof dataId !== "string")
+            throw new TypeError("dataId must be a string");
+        const existing = this.family.toModel.get(dataId);
+        if (existing !== undefined) {
+            if (!this.seenLedger.has(dataId))
+                this.seenLedger.set(dataId, []);
+            return existing;
+        }
+        if (document === null || typeof document !== "object" || Array.isArray(document))
+            throw new TypeError("document must be a record");
+        const stored = structuredClone(document);
+        const modelId = this.family.ids.mint();
+        this.family.documents.set(dataId, stored);
+        this.family.toModel.set(dataId, modelId);
+        this.family.toData.set(modelId, dataId);
+        this.seenLedger.set(dataId, []);
+        return modelId;
+    }
+    contains(dataId) { return this.family.documents.has(dataId); }
+    containsModelFacingId(modelId) { return this.family.toData.has(modelId); }
+    toDataId(modelId) {
+        const id = this.family.toData.get(modelId);
+        if (id === undefined)
+            throw new Error(`modelFacingId ${JSON.stringify(modelId)} not found in cache`);
+        return id;
+    }
+    toModelFacingId(dataId) {
+        const id = this.family.toModel.get(dataId);
+        if (id === undefined)
+            throw new Error(`dataId ${JSON.stringify(dataId)} not found in cache`);
+        return id;
+    }
+    getDocument(dataId) {
+        const doc = this.family.documents.get(dataId);
+        if (doc === undefined)
+            throw new Error(`dataId ${JSON.stringify(dataId)} not found in the document cache`);
+        return doc;
+    }
+    getDocumentFromModelFacingId(modelId) { return this.getDocument(this.toDataId(modelId)); }
+    content(dataId, snippetField) {
+        const value = (0, rendering_1.field)(this.getDocument(dataId), snippetField);
+        if (typeof value !== "string" || !value)
+            throw new TypeError(`snippet field ${JSON.stringify(snippetField)} must be a non-empty string`);
+        return value;
+    }
+    resolveCharRange(dataId, snippetField, range) {
+        return (0, ranges_1.resolveRange)(range, new ranges_1.CodePointText(this.content(dataId, snippetField)).length, this.rangeMode, dataId);
+    }
+    /** Override in a subclass to select a different span. Returned spans are strictly validated. */
+    snippetFn(query, content, snippetSize, language) {
+        return (0, snippet_1.bm25SnippetWithStride)(query, content, { windowSize: snippetSize, stride: Math.max(1, Math.floor(snippetSize / 5)), language });
+    }
+    applySnippet(dataId, { snippetField, query, snippetSize = exports.SNIPPET_SIZE_DEFAULT, minSeenOverlap = exports.MIN_SEEN_OVERLAP_DEFAULT, displayFields, language = this.language }) {
+        const document = this.getDocument(dataId), content = this.content(dataId, snippetField);
+        const fields = displayFields ?? Object.keys(document);
+        if (!fields.includes(snippetField))
+            throw new RangeError(`snippet field ${JSON.stringify(snippetField)} must be in displayFields`);
+        (0, ranges_1.positiveInteger)(snippetSize, "snippetSize");
+        if (!Number.isSafeInteger(minSeenOverlap) || minSeenOverlap < 0)
+            throw new RangeError("minSeenOverlap must be a nonnegative safe integer");
+        const span = (0, ranges_1.resolveRange)(this.snippetFn(query, content, snippetSize, (0, snippet_1.validateLanguage)(language)), new ranges_1.CodePointText(content).length, "strict", dataId);
+        const seen = this.seenLedger.get(dataId) ?? [];
+        const overlap = (0, intervals_1.overlaps)(span, seen);
+        const fullySeen = overlap.length === 1 && overlap[0][0] === span[0] && overlap[0][1] === span[1];
+        const [shown, masked] = fullySeen ? [[], [span]] : (0, intervals_1.planSegments)(span, seen, minSeenOverlap);
+        return new rendering_1.DocumentViewWithSnippet(dataId, this.toModelFacingId(dataId), document, snippetField, masked, shown, fields);
+    }
+    getSingleSpanDocumentView(dataId, { snippetField, snippetDisplaySpan, displayFields } = {}) {
+        const document = this.getDocument(dataId), modelId = this.toModelFacingId(dataId);
+        if (snippetField === undefined) {
+            if (displayFields === undefined)
+                throw new TypeError("displayFields is required when snippetField is omitted");
+            return new rendering_1.DocumentView(dataId, modelId, document, displayFields);
+        }
+        const content = this.content(dataId, snippetField);
+        const span = snippetDisplaySpan === undefined ? [0, new ranges_1.CodePointText(content).length]
+            : this.resolveCharRange(dataId, snippetField, snippetDisplaySpan);
+        return new rendering_1.DocumentViewWithSnippet(dataId, modelId, document, snippetField, [], [span], displayFields ?? Object.keys(document));
+    }
+    updateSeen(view) {
+        this.getDocument(view.dataId);
+        let seen = this.seenLedger.get(view.dataId) ?? [];
+        for (const span of view.snippetDisplaySpans)
+            seen = (0, intervals_1.insertInterval)(seen, span);
+        this.seenLedger.set(view.dataId, seen);
+    }
+    fork(n) {
+        if (!Number.isSafeInteger(n) || n < 0)
+            throw new RangeError("fork count must be a nonnegative safe integer");
+        return Array.from({ length: n }, () => {
+            const child = new DocumentCache({ language: this.language, rangeMode: this.rangeMode });
+            child.family = this.family;
+            for (const [id, spans] of this.seenLedger)
+                child.seenLedger.set(id, spans.map(([a, b]) => [a, b]));
+            return child;
+        });
+    }
+}
+exports.DocumentCache = DocumentCache;

package/dist/cjs/id-stream.d.ts

@@ -0,0 +1,25 @@
+export interface IdStreamOptions {
+    alphabet?: string;
+    length?: number;
+    seed?: number | bigint;
+}
+export declare class IdSpaceExhausted extends Error {
+    name: string;
+}
+export declare class IdStream implements IterableIterator<string> {
+    readonly alphabet: string;
+    readonly length: number;
+    readonly space: bigint;
+    private readonly chars;
+    private readonly key;
+    private readonly halfBits;
+    private counter;
+    constructor({ alphabet, length, seed }?: IdStreamOptions);
+    get minted(): bigint;
+    get remaining(): bigint;
+    [Symbol.iterator](): IterableIterator<string>;
+    next(): IteratorResult<string, never>;
+    mint(): string;
+    at(index: number | bigint): string;
+    private permute;
+}

package/dist/cjs/id-stream.js

@@ -0,0 +1,76 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IdStream = exports.IdSpaceExhausted = void 0;
+const node_crypto_1 = require("node:crypto");
+const ranges_1 = require("./ranges");
+class IdSpaceExhausted extends Error {
+    name = "IdSpaceExhausted";
+}
+exports.IdSpaceExhausted = IdSpaceExhausted;
+const MASK = (1n << 64n) - 1n;
+function mix(value, key, round) {
+    let x = (value + key + round * 0x9e3779b97f4a7c15n) & MASK;
+    x ^= x >> 30n;
+    x = (x * 0xbf58476d1ce4e5b9n) & MASK;
+    x ^= x >> 27n;
+    x = (x * 0x94d049bb133111ebn) & MASK;
+    return x ^ (x >> 31n);
+}
+class IdStream {
+    alphabet;
+    length;
+    space;
+    chars;
+    key;
+    halfBits;
+    counter = 0n;
+    constructor({ alphabet = "abcdefghijklmnopqrstuvwxyz", length = 5, seed } = {}) {
+        this.length = (0, ranges_1.positiveInteger)(length, "length");
+        if (typeof alphabet !== "string")
+            throw new TypeError("alphabet must be a string");
+        this.alphabet = alphabet;
+        this.chars = Array.from(alphabet);
+        if (this.chars.length === 0 || new Set(this.chars).size !== this.chars.length || /[#:]/.test(alphabet)) {
+            throw new RangeError("alphabet must be nonempty, unique, and exclude # and :");
+        }
+        if (seed !== undefined && typeof seed !== "bigint" && !Number.isSafeInteger(seed)) {
+            throw new RangeError("seed must be a safe integer or bigint");
+        }
+        this.space = BigInt(this.chars.length) ** BigInt(length);
+        this.key = seed === undefined ? (0, node_crypto_1.randomBytes)(8).readBigUInt64LE() : mix(BigInt(seed), 0n, 0n);
+        this.halfBits = BigInt(Math.max(1, Math.ceil((this.space - 1n).toString(2).length / 2)));
+    }
+    get minted() { return this.counter; }
+    get remaining() { return this.space - this.counter; }
+    [Symbol.iterator]() { return this; }
+    next() { return { value: this.mint(), done: false }; }
+    mint() {
+        if (this.counter >= this.space)
+            throw new IdSpaceExhausted(`all ${this.space} ids of this stream are in use`);
+        return this.at(this.counter++);
+    }
+    at(index) {
+        if (typeof index !== "bigint" && !Number.isSafeInteger(index))
+            throw new RangeError("index must be a safe integer or bigint");
+        let x = BigInt(index);
+        if (x < 0n || x >= this.space)
+            throw new RangeError("index outside id space");
+        do {
+            x = this.permute(x);
+        } while (x >= this.space);
+        const base = BigInt(this.chars.length), result = [];
+        for (let i = 0; i < this.length; i++) {
+            result.push(this.chars[Number(x % base)]);
+            x /= base;
+        }
+        return result.reverse().join("");
+    }
+    permute(x) {
+        const mask = (1n << this.halfBits) - 1n;
+        let lo = x >> this.halfBits, hi = x & mask;
+        for (let round = 0n; round < 4n; round++)
+            [lo, hi] = [hi, lo ^ (mix(hi, this.key, round) & mask)];
+        return (lo << this.halfBits) | hi;
+    }
+}
+exports.IdStream = IdStream;

package/dist/cjs/index.d.ts

@@ -0,0 +1,10 @@
+export { DocumentCache, SNIPPET_SIZE_DEFAULT, MIN_SEEN_OVERLAP_DEFAULT } from "./cache";
+export type { DocumentCacheOptions, ApplySnippetOptions, SingleSpanOptions } from "./cache";
+export { DocumentView, DocumentViewWithSnippet, parseRenderedModelFacingId, renderMarkdownTable } from "./rendering";
+export type { Document, RenderParts } from "./rendering";
+export { InvalidCharacterRange } from "./ranges";
+export type { CharacterRange, RangeMode } from "./ranges";
+export { IdStream, IdSpaceExhausted } from "./id-stream";
+export type { IdStreamOptions } from "./id-stream";
+export { bm25SnippetWithStride, SUPPORTED_LANGUAGES } from "./snippet";
+export type { Language, SnippetOptions } from "./snippet";

package/dist/cjs/index.js

@@ -0,0 +1,20 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SUPPORTED_LANGUAGES = exports.bm25SnippetWithStride = exports.IdSpaceExhausted = exports.IdStream = exports.InvalidCharacterRange = exports.renderMarkdownTable = exports.parseRenderedModelFacingId = exports.DocumentViewWithSnippet = exports.DocumentView = exports.MIN_SEEN_OVERLAP_DEFAULT = exports.SNIPPET_SIZE_DEFAULT = exports.DocumentCache = void 0;
+var cache_1 = require("./cache");
+Object.defineProperty(exports, "DocumentCache", { enumerable: true, get: function () { return cache_1.DocumentCache; } });
+Object.defineProperty(exports, "SNIPPET_SIZE_DEFAULT", { enumerable: true, get: function () { return cache_1.SNIPPET_SIZE_DEFAULT; } });
+Object.defineProperty(exports, "MIN_SEEN_OVERLAP_DEFAULT", { enumerable: true, get: function () { return cache_1.MIN_SEEN_OVERLAP_DEFAULT; } });
+var rendering_1 = require("./rendering");
+Object.defineProperty(exports, "DocumentView", { enumerable: true, get: function () { return rendering_1.DocumentView; } });
+Object.defineProperty(exports, "DocumentViewWithSnippet", { enumerable: true, get: function () { return rendering_1.DocumentViewWithSnippet; } });
+Object.defineProperty(exports, "parseRenderedModelFacingId", { enumerable: true, get: function () { return rendering_1.parseRenderedModelFacingId; } });
+Object.defineProperty(exports, "renderMarkdownTable", { enumerable: true, get: function () { return rendering_1.renderMarkdownTable; } });
+var ranges_1 = require("./ranges");
+Object.defineProperty(exports, "InvalidCharacterRange", { enumerable: true, get: function () { return ranges_1.InvalidCharacterRange; } });
+var id_stream_1 = require("./id-stream");
+Object.defineProperty(exports, "IdStream", { enumerable: true, get: function () { return id_stream_1.IdStream; } });
+Object.defineProperty(exports, "IdSpaceExhausted", { enumerable: true, get: function () { return id_stream_1.IdSpaceExhausted; } });
+var snippet_1 = require("./snippet");
+Object.defineProperty(exports, "bm25SnippetWithStride", { enumerable: true, get: function () { return snippet_1.bm25SnippetWithStride; } });
+Object.defineProperty(exports, "SUPPORTED_LANGUAGES", { enumerable: true, get: function () { return snippet_1.SUPPORTED_LANGUAGES; } });

package/dist/cjs/intervals.d.ts

@@ -0,0 +1,4 @@
+import type { CharacterRange } from "./ranges";
+export declare function insertInterval(intervals: readonly CharacterRange[], range: CharacterRange): CharacterRange[];
+export declare function overlaps(span: CharacterRange, intervals: readonly CharacterRange[]): CharacterRange[];
+export declare function planSegments(span: CharacterRange, seen: readonly CharacterRange[], minimum: number): [CharacterRange[], CharacterRange[]];

package/dist/cjs/intervals.js

@@ -0,0 +1,50 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.insertInterval = insertInterval;
+exports.overlaps = overlaps;
+exports.planSegments = planSegments;
+function insertInterval(intervals, range) {
+    let [a, b] = range;
+    if (a >= b)
+        throw new RangeError("cannot record an empty interval");
+    const out = [];
+    for (const [x, y] of intervals) {
+        if (y < a || x > b)
+            out.push([x, y]);
+        else {
+            a = Math.min(a, x);
+            b = Math.max(b, y);
+        }
+    }
+    out.push([a, b]);
+    return out.sort((left, right) => left[0] - right[0]);
+}
+function overlaps(span, intervals) {
+    return intervals.flatMap(([x, y]) => {
+        const a = Math.max(span[0], x), b = Math.min(span[1], y);
+        return a < b ? [[a, b]] : [];
+    });
+}
+function planSegments(span, seen, minimum) {
+    const display = [], masked = [];
+    function append(a, b) {
+        const last = display.at(-1);
+        if (last?.[1] === a)
+            display[display.length - 1] = [last[0], b];
+        else
+            display.push([a, b]);
+    }
+    let cursor = span[0];
+    for (const [x, y] of overlaps(span, seen)) {
+        if (x > cursor)
+            append(cursor, x);
+        if (y - x >= minimum)
+            masked.push([x, y]);
+        else
+            append(x, y);
+        cursor = y;
+    }
+    if (cursor < span[1])
+        append(cursor, span[1]);
+    return [display, masked];
+}

package/dist/cjs/package.json

@@ -0,0 +1 @@
+{"type":"commonjs"}

package/dist/cjs/ranges.d.ts

@@ -0,0 +1,17 @@
+/** Half-open Unicode code-point offsets, matching Python string indices. */
+export type CharacterRange = readonly [number, number];
+export type RangeMode = "lenient" | "strict";
+export declare class InvalidCharacterRange extends RangeError {
+    name: string;
+}
+export declare function validateRangeMode(mode: RangeMode): RangeMode;
+export declare function resolveRange(range: CharacterRange, length: number, mode: RangeMode, id: string): CharacterRange;
+/** Build once per render; boundaries also preserve astral Unicode characters. */
+export declare class CodePointText {
+    readonly text: string;
+    private readonly boundaries;
+    constructor(text: string);
+    get length(): number;
+    slice(start: number, end: number): string;
+}
+export declare function positiveInteger(value: number, name: string): number;

package/dist/cjs/ranges.js

@@ -0,0 +1,52 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CodePointText = exports.InvalidCharacterRange = void 0;
+exports.validateRangeMode = validateRangeMode;
+exports.resolveRange = resolveRange;
+exports.positiveInteger = positiveInteger;
+class InvalidCharacterRange extends RangeError {
+    name = "InvalidCharacterRange";
+}
+exports.InvalidCharacterRange = InvalidCharacterRange;
+function validateRangeMode(mode) {
+    if (mode !== "lenient" && mode !== "strict") {
+        throw new RangeError(`unsupported range mode ${String(mode)}; supported values: lenient, strict`);
+    }
+    return mode;
+}
+function resolveRange(range, length, mode, id) {
+    const prefix = `invalid character range for document ${JSON.stringify(id)} (${length} characters): `;
+    if (!Array.isArray(range) || range.length !== 2 || !range.every(Number.isSafeInteger)) {
+        throw new InvalidCharacterRange(prefix + "expected a (start, end) pair of safe integers");
+    }
+    const [start, end] = range;
+    if (start >= end)
+        throw new InvalidCharacterRange(prefix + `start ${start} must be less than end ${end}`);
+    if (end <= 0 || start >= length)
+        throw new InvalidCharacterRange(prefix + "the requested range does not overlap the document");
+    if (mode === "strict" && (start < 0 || end > length)) {
+        throw new InvalidCharacterRange(prefix + "range exceeds the document boundaries");
+    }
+    return [Math.max(0, start), Math.min(end, length)];
+}
+/** Build once per render; boundaries also preserve astral Unicode characters. */
+class CodePointText {
+    text;
+    boundaries = [0];
+    constructor(text) {
+        this.text = text;
+        let offset = 0;
+        for (const char of text) {
+            offset += char.length;
+            this.boundaries.push(offset);
+        }
+    }
+    get length() { return this.boundaries.length - 1; }
+    slice(start, end) { return this.text.slice(this.boundaries[start], this.boundaries[end]); }
+}
+exports.CodePointText = CodePointText;
+function positiveInteger(value, name) {
+    if (!Number.isSafeInteger(value) || value <= 0)
+        throw new RangeError(`${name} must be a positive safe integer`);
+    return value;
+}

package/dist/cjs/rendering.d.ts

@@ -0,0 +1,26 @@
+import { type CharacterRange } from "./ranges";
+export type Document = Record<string, unknown>;
+export declare function field(document: Document, key: string): unknown;
+export declare function parseRenderedModelFacingId(reference: string): [string, CharacterRange | null];
+export interface RenderParts {
+    id: string;
+    attributes: Map<string, unknown>;
+    body: string | null;
+}
+export declare class DocumentView<T extends Document = Document> {
+    readonly dataId: string;
+    readonly modelFacingId: string;
+    readonly document: T;
+    readonly displayFields: readonly string[];
+    readonly snippetDisplaySpans: CharacterRange[];
+    constructor(dataId: string, modelFacingId: string, document: T, displayFields: readonly string[]);
+    renderParts(displayFields?: readonly string[]): RenderParts;
+    renderXml(): string;
+}
+export declare class DocumentViewWithSnippet<T extends Document = Document> extends DocumentView<T> {
+    readonly snippetField: string;
+    readonly snippetSeenSpans: CharacterRange[];
+    constructor(dataId: string, modelFacingId: string, document: T, snippetField: string, snippetSeenSpans: CharacterRange[], snippetDisplaySpans: CharacterRange[], displayFields: readonly string[]);
+    renderParts(displayFields?: readonly string[]): RenderParts;
+}
+export declare function renderMarkdownTable(views: readonly DocumentView[], displayFields?: readonly string[]): string;

package/dist/cjs/rendering.js

@@ -0,0 +1,147 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DocumentViewWithSnippet = exports.DocumentView = void 0;
+exports.field = field;
+exports.parseRenderedModelFacingId = parseRenderedModelFacingId;
+exports.renderMarkdownTable = renderMarkdownTable;
+const ranges_1 = require("./ranges");
+function field(document, key) {
+    if (!Object.hasOwn(document, key))
+        throw new Error(`document field ${JSON.stringify(key)} not found`);
+    return document[key];
+}
+function scalar(value) {
+    if (value === true)
+        return "True";
+    if (value === false)
+        return "False";
+    if (value === null)
+        return "None";
+    return String(value);
+}
+function stringify(value) { return Array.isArray(value) ? value.map(scalar).join(", ") : scalar(value); }
+function escape(value) { return stringify(value).replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;"); }
+function truthy(value) {
+    if (!value)
+        return false;
+    if (Array.isArray(value))
+        return value.length > 0;
+    if (typeof value === "object" && Object.getPrototypeOf(value) === Object.prototype)
+        return Object.keys(value).length > 0;
+    return true;
+}
+function decimal(value) {
+    // Python accepts Unicode decimal digits; normalize each Nd block to ASCII.
+    let digits = "";
+    for (const char of value) {
+        if (char === "-") {
+            digits += char;
+            continue;
+        }
+        let first = char.codePointAt(0);
+        while (first > 0 && /\p{Nd}/u.test(String.fromCodePoint(first - 1)))
+            first--;
+        digits += String((char.codePointAt(0) - first) % 10);
+    }
+    const result = Number(digits);
+    if (!Number.isSafeInteger(result))
+        throw new RangeError("character offsets must be safe integers");
+    return result;
+}
+function parseRenderedModelFacingId(reference) {
+    if (typeof reference !== "string" || !reference)
+        throw new TypeError("invalid document reference: expected a non-empty string");
+    const hash = reference.indexOf("#");
+    if (hash < 0)
+        return [reference, null];
+    if (hash === 0)
+        throw new RangeError("invalid document reference: missing document id before '#'");
+    const parts = reference.slice(hash + 1).split(":").map(part => part.trim());
+    if (parts.length !== 2 || !parts.every(part => /^-?\p{Nd}+$/u.test(part)))
+        throw new RangeError("invalid document reference: expected '<doc_id>#<start>:<end>' with decimal integer character offsets");
+    const start = decimal(parts[0]), end = decimal(parts[1]);
+    if (start >= end)
+        throw new RangeError("invalid document reference: start must be < end");
+    return [reference.slice(0, hash), [start, end]];
+}
+class DocumentView {
+    dataId;
+    modelFacingId;
+    document;
+    displayFields;
+    snippetDisplaySpans = [];
+    constructor(dataId, modelFacingId, document, displayFields) {
+        this.dataId = dataId;
+        this.modelFacingId = modelFacingId;
+        this.document = document;
+        this.displayFields = displayFields;
+    }
+    renderParts(displayFields = this.displayFields) {
+        return { id: this.modelFacingId, attributes: new Map(displayFields.map(key => [key, field(this.document, key)])), body: null };
+    }
+    renderXml() {
+        const { id, attributes, body } = this.renderParts();
+        const parts = [`id="${escape(id)}"`];
+        for (const [key, value] of attributes) {
+            if (truthy(value))
+                parts.push(typeof value === "boolean" || (typeof value === "number" && Number.isInteger(value))
+                    ? `${key}=${scalar(value)}` : `${key}="${escape(value)}"`);
+        }
+        const inner = body === null ? "" : `\n${escape(body)}\n`;
+        return `<doc ${parts.join(" ")}>${inner}</doc>`;
+    }
+}
+exports.DocumentView = DocumentView;
+class DocumentViewWithSnippet extends DocumentView {
+    snippetField;
+    snippetSeenSpans;
+    constructor(dataId, modelFacingId, document, snippetField, snippetSeenSpans, snippetDisplaySpans, displayFields) {
+        super(dataId, modelFacingId, document, displayFields);
+        this.snippetField = snippetField;
+        this.snippetSeenSpans = snippetSeenSpans;
+        this.snippetDisplaySpans.push(...snippetDisplaySpans);
+    }
+    renderParts(displayFields = this.displayFields) {
+        const content = field(this.document, this.snippetField);
+        if (typeof content !== "string" || !content)
+            throw new TypeError("snippet field must be a non-empty string");
+        const text = new ranges_1.CodePointText(content);
+        const segments = [
+            ...this.snippetDisplaySpans.map(([a, b]) => ({ a, b, seen: false })),
+            ...this.snippetSeenSpans.map(([a, b]) => ({ a, b, seen: true })),
+        ].sort((left, right) => left.a - right.a);
+        if (!segments.length)
+            throw new RangeError("snippet view requires at least one span");
+        const a = segments[0].a, b = segments.at(-1).b;
+        const fullySeen = this.snippetDisplaySpans.length === 0;
+        const body = (a > 0 && !fullySeen ? "... " : "")
+            + segments.map(segment => segment.seen ? `[seen: "#${segment.a}:${segment.b}"]` : text.slice(segment.a, segment.b)).join("")
+            + (b < text.length && !fullySeen ? " ..." : "");
+        const id = this.modelFacingId + (!fullySeen && (a !== 0 || b !== text.length) ? `#${a}:${b}` : "");
+        const attributes = new Map([["doc_length", text.length]]);
+        for (const key of displayFields)
+            if (key !== this.snippetField)
+                attributes.set(key, field(this.document, key));
+        return { id, attributes, body };
+    }
+}
+exports.DocumentViewWithSnippet = DocumentViewWithSnippet;
+function renderMarkdownTable(views, displayFields) {
+    if (views.length === 0)
+        return "";
+    const fields = displayFields ?? views[0].displayFields;
+    const rows = views.map(view => {
+        const { id, attributes, body } = view.renderParts(fields);
+        const cells = new Map([["id", id], ...attributes]);
+        if (body !== null && view instanceof DocumentViewWithSnippet)
+            cells.set(view.snippetField, body);
+        return cells;
+    });
+    const columns = ["id", ...Array.from(rows[0].keys()).filter(key => key !== "id" && !fields.includes(key)), ...fields];
+    const cell = (value) => stringify(value).replaceAll("|", "\\|").replaceAll("\n", " ");
+    return [
+        `| ${columns.join(" | ")} |`,
+        `|${columns.map(key => "-".repeat(Array.from(key).length + 2)).join("|")}|`,
+        ...rows.map(row => `| ${columns.map(key => cell(row.has(key) ? row.get(key) : "")).join(" | ")} |`),
+    ].join("\n");
+}

package/dist/cjs/snippet.d.ts

@@ -0,0 +1,10 @@
+import { type CharacterRange } from "./ranges";
+export declare const SUPPORTED_LANGUAGES: readonly ["danish", "dutch", "english", "finnish", "french", "german", "generic", "hungarian", "italian", "norwegian", "portuguese", "russian", "spanish", "swedish"];
+export type Language = typeof SUPPORTED_LANGUAGES[number];
+export interface SnippetOptions {
+    windowSize?: number;
+    stride?: number;
+    language?: Language;
+}
+export declare function validateLanguage(language: Language): Language;
+export declare function bm25SnippetWithStride(query: string, content: string, { windowSize, stride, language }?: SnippetOptions): CharacterRange;

package/dist/cjs/snippet.js

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SUPPORTED_LANGUAGES = void 0;
+exports.validateLanguage = validateLanguage;
+exports.bm25SnippetWithStride = bm25SnippetWithStride;
+const ranges_1 = require("./ranges");
+exports.SUPPORTED_LANGUAGES = ["danish", "dutch", "english", "finnish", "french", "german", "generic", "hungarian", "italian", "norwegian", "portuguese", "russian", "spanish", "swedish"];
+function validateLanguage(language) {
+    if (!exports.SUPPORTED_LANGUAGES.includes(language))
+        throw new RangeError(`unsupported language ${String(language)}; supported values: ${exports.SUPPORTED_LANGUAGES.join(", ")}`);
+    return language;
+}
+let engine;
+function bm25SnippetWithStride(query, content, { windowSize = 50, stride = 10, language = "english" } = {}) {
+    if (typeof query !== "string" || typeof content !== "string")
+        throw new TypeError("query and content must be strings");
+    // wasm-bindgen encodes lone surrogates as U+FFFD; reject instead of silently changing source text.
+    if (!isWellFormed(query) || !isWellFormed(content))
+        throw new TypeError("query and content must contain well-formed Unicode");
+    (0, ranges_1.positiveInteger)(windowSize, "windowSize");
+    (0, ranges_1.positiveInteger)(stride, "stride");
+    if (windowSize > 0xffffffff || stride > 0xffffffff)
+        throw new RangeError("windowSize and stride must fit unsigned 32-bit integers");
+    validateLanguage(language);
+    engine ??= require("../wasm/sid_snippet.cjs");
+    const result = engine.bm25SnippetWithStride(query, content, windowSize, stride, language);
+    return [result[0], result[1]];
+}
+function isWellFormed(value) {
+    for (const char of value) {
+        const cp = char.codePointAt(0);
+        if (cp >= 0xd800 && cp <= 0xdfff)
+            return false;
+    }
+    return true;
+}

package/dist/esm/index.js

@@ -0,0 +1 @@
+export { DocumentCache, SNIPPET_SIZE_DEFAULT, MIN_SEEN_OVERLAP_DEFAULT, DocumentView, DocumentViewWithSnippet, parseRenderedModelFacingId, renderMarkdownTable, InvalidCharacterRange, IdStream, IdSpaceExhausted, bm25SnippetWithStride, SUPPORTED_LANGUAGES } from "../cjs/index.js";