@omgbase/core 0.1.0

package/dist/src/search/cel/lexer.d.ts

@@ -0,0 +1,9 @@
+export type TokenType = "ident" | "field" | "outer" | "string" | "int" | "double" | "bool" | "null" | "op" | "lparen" | "rparen" | "comma" | "dot" | "eof";
+export interface Token {
+    type: TokenType;
+    value: string;
+    pos: number;
+}
+export declare class LexError extends Error {
+}
+export declare function lex(src: string): Token[];

package/dist/src/search/cel/lexer.js

@@ -0,0 +1,139 @@
+// Lexer for the CEL subset (query-language §3.1). Produces a flat token
+// stream for the recursive-descent parser.
+export class LexError extends Error {
+}
+const OPS_2 = new Set(["&&", "||", "==", "!=", "<=", ">="]);
+// Arithmetic operators are lexed so the parser can reject them with a precise
+// "arithmetic is not supported" message (10 §3.1) rather than a lex error.
+const OPS_1 = new Set(["!", "<", ">", "+", "-", "*", "/"]);
+export function lex(src) {
+    const tokens = [];
+    let i = 0;
+    const n = src.length;
+    const push = (type, value, pos) => {
+        tokens.push({ type, value, pos });
+    };
+    while (i < n) {
+        const c = src[i];
+        if (c === " " || c === "\t" || c === "\n" || c === "\r") {
+            i++;
+            continue;
+        }
+        if (c === "(") {
+            push("lparen", c, i);
+            i++;
+            continue;
+        }
+        if (c === ")") {
+            push("rparen", c, i);
+            i++;
+            continue;
+        }
+        if (c === ",") {
+            push("comma", c, i);
+            i++;
+            continue;
+        }
+        if (c === ".") {
+            push("dot", c, i);
+            i++;
+            continue;
+        }
+        // two-char operators
+        const two = src.slice(i, i + 2);
+        if (OPS_2.has(two)) {
+            push("op", two, i);
+            i += 2;
+            continue;
+        }
+        if (OPS_1.has(c)) {
+            push("op", c, i);
+            i++;
+            continue;
+        }
+        // strings
+        if (c === '"' || c === "'") {
+            const quote = c;
+            const start = i;
+            i++;
+            let value = "";
+            while (i < n && src[i] !== quote) {
+                if (src[i] === "\\") {
+                    i++;
+                    const esc = src[i];
+                    if (esc === undefined)
+                        throw new LexError(`unterminated escape at ${i}`);
+                    value += esc === "n" ? "\n" : esc === "t" ? "\t" : esc;
+                    i++;
+                }
+                else {
+                    value += src[i];
+                    i++;
+                }
+            }
+            if (i >= n)
+                throw new LexError(`unterminated string starting at ${start}`);
+            i++; // closing quote
+            push("string", value, start);
+            continue;
+        }
+        // numbers
+        if (c >= "0" && c <= "9") {
+            const start = i;
+            while (i < n && src[i] >= "0" && src[i] <= "9")
+                i++;
+            let isDouble = false;
+            if (src[i] === ".") {
+                isDouble = true;
+                i++;
+                while (i < n && src[i] >= "0" && src[i] <= "9")
+                    i++;
+            }
+            push(isDouble ? "double" : "int", src.slice(start, i), start);
+            continue;
+        }
+        // ^name — a one-scope-outward binding reference (the symmetric read form of
+        // the `^name:` lift). OQX resolves the name against the enclosing query
+        // scope's bindings; a bare `^` with no name is a lex error.
+        if (c === "^") {
+            const start = i;
+            i++;
+            const nameStart = i;
+            while (i < n && /[A-Za-z0-9_]/.test(src[i]))
+                i++;
+            if (i === nameStart)
+                throw new LexError(`expected a binding name after '^' at ${start}`);
+            push("outer", src.slice(nameStart, i), start);
+            continue;
+        }
+        // $-intrinsic: $ident(.ident)*
+        if (c === "$") {
+            const start = i;
+            i++;
+            while (i < n && /[A-Za-z0-9_]/.test(src[i]))
+                i++;
+            // allow dotted continuation only when followed by ident (handled by parser via dot tokens);
+            // here we capture just the leading $ident. Dots handled as separate tokens.
+            push("field", src.slice(start, i), start);
+            continue;
+        }
+        // identifiers / keywords
+        if (/[A-Za-z_]/.test(c)) {
+            const start = i;
+            while (i < n && /[A-Za-z0-9_]/.test(src[i]))
+                i++;
+            const word = src.slice(start, i);
+            if (word === "true" || word === "false")
+                push("bool", word, start);
+            else if (word === "null")
+                push("null", word, start);
+            else
+                push("ident", word, start);
+            continue;
+        }
+        throw new LexError(`unexpected character '${c}' at ${i}`);
+    }
+    push("eof", "", n);
+    return tokens;
+}
+//# sourceMappingURL=lexer.js.map

package/dist/src/search/cel/parser.d.ts

@@ -0,0 +1,7 @@
+import type { Node } from "./ast.js";
+export declare class FilterInvalid extends Error {
+    reason: string;
+    hint: string;
+    constructor(reason: string, hint?: string);
+}
+export declare function parseFilter(src: string): Node;

package/dist/src/search/cel/parser.js

@@ -0,0 +1,219 @@
+import { lex, LexError } from "./lexer.js";
+// Recursive-descent parser for the CEL subset (10 §3.1). Throws FilterInvalid
+// with a reason + hint on anything outside the grammar.
+export class FilterInvalid extends Error {
+    reason;
+    hint;
+    constructor(reason, hint = "see query-language.md §3") {
+        super(reason);
+        this.reason = reason;
+        this.hint = hint;
+    }
+}
+const RELOPS = new Set(["==", "!=", "<", "<=", ">", ">="]);
+export function parseFilter(src) {
+    let tokens;
+    try {
+        tokens = lex(src);
+    }
+    catch (e) {
+        throw new FilterInvalid(e instanceof LexError ? e.message : String(e));
+    }
+    const p = new Parser(tokens);
+    const node = p.parseExpr();
+    p.expect("eof");
+    return node;
+}
+class Parser {
+    tokens;
+    pos = 0;
+    constructor(tokens) {
+        this.tokens = tokens;
+    }
+    peek() { return this.tokens[this.pos]; }
+    next() { return this.tokens[this.pos++]; }
+    expect(type) {
+        const t = this.peek();
+        if (t.type !== type)
+            throw new FilterInvalid(`expected ${type} but found '${t.value || t.type}' at ${t.pos}`);
+        return this.next();
+    }
+    parseExpr() { return this.parseOr(); }
+    parseOr() {
+        let left = this.parseAnd();
+        while (this.peek().type === "op" && this.peek().value === "||") {
+            this.next();
+            left = { kind: "or", left, right: this.parseAnd() };
+        }
+        return left;
+    }
+    parseAnd() {
+        let left = this.parseUnary();
+        while (this.peek().type === "op" && this.peek().value === "&&") {
+            this.next();
+            left = { kind: "and", left, right: this.parseUnary() };
+        }
+        return left;
+    }
+    parseUnary() {
+        if (this.peek().type === "op" && this.peek().value === "!") {
+            this.next();
+            return { kind: "not", operand: this.parseUnary() };
+        }
+        return this.parseComparison();
+    }
+    // comparison | membership | (primary that may stand alone as boolean)
+    parseComparison() {
+        const left = this.parsePostfix(this.parsePrimary());
+        // membership: literal in list(field) — detect the `in` keyword
+        if (this.peek().type === "ident" && this.peek().value === "in") {
+            this.next();
+            // `<value> in ^name`: membership over a one-scope-outward collection
+            // binding (a lift). The left operand may be any scalar (field/value/lit).
+            if (this.peek().type === "outer") {
+                const ref = this.next();
+                return { kind: "outerMembership", value: left, collection: { kind: "outerref", name: ref.value } };
+            }
+            const listCall = this.parsePrimary();
+            if (listCall.kind !== "call" || listCall.name !== "list") {
+                throw new FilterInvalid("`in` is only allowed as `<literal> in list(field)`", "10 §4");
+            }
+            if (left.kind !== "literal") {
+                throw new FilterInvalid("left side of `in list()` must be a literal", "10 §4");
+            }
+            const fieldArg = listCall.args[0];
+            if (!fieldArg || fieldArg.kind !== "field") {
+                throw new FilterInvalid("list() takes a single field argument", "10 §4");
+            }
+            return { kind: "membership", value: left, field: fieldArg };
+        }
+        if (this.peek().type === "op" && RELOPS.has(this.peek().value)) {
+            const op = this.next().value;
+            const right = this.parsePostfix(this.parsePrimary());
+            return { kind: "comparison", op, left, right };
+        }
+        if (this.peek().type === "op" && ["+", "-", "*", "/"].includes(this.peek().value)) {
+            throw new FilterInvalid("arithmetic is not supported", "10 §3.1");
+        }
+        return left;
+    }
+    // handle method calls: receiver.name(args) and quantifiers .exists/.all
+    parsePostfix(node) {
+        let cur = node;
+        while (this.peek().type === "dot") {
+            this.next();
+            // A `$`-segment after a dot is an intrinsic reach-through continuation
+            // (`doc.$path`, `doc.$updated_at`): the lexer emits it as a `field` token
+            // (value keeps the `$`). It is never a method receiver, so append it as a
+            // field segment directly. The compiler routes `doc.$…` to docs intrinsics.
+            if (this.peek().type === "field") {
+                const seg = this.next().value;
+                if (cur.kind !== "field") {
+                    throw new FilterInvalid(`unexpected '.${seg}' — intrinsic segment requires a field receiver`, "10 §2");
+                }
+                cur = { kind: "field", segments: [...cur.segments, seg], intrinsic: cur.intrinsic };
+                continue;
+            }
+            const name = this.expect("ident").value;
+            if (this.peek().type !== "lparen") {
+                // dotted field continuation (only valid if receiver was a field)
+                if (cur.kind === "field") {
+                    cur = { kind: "field", segments: [...cur.segments, name], intrinsic: cur.intrinsic };
+                    continue;
+                }
+                throw new FilterInvalid(`unexpected '.${name}' — method call requires '('`);
+            }
+            const args = this.parseArgs();
+            if (name === "exists" || name === "all") {
+                // first arg must be an identifier (the bound var), rest is the predicate
+                const [varArg, pred] = args;
+                if (!varArg || varArg.kind !== "field" || varArg.segments.length !== 1 || varArg.intrinsic) {
+                    throw new FilterInvalid(`${name}() first argument must be a bound variable name`, "10 §3.2");
+                }
+                if (!pred)
+                    throw new FilterInvalid(`${name}() requires a predicate`, "10 §3.2");
+                cur = { kind: "quantifier", collection: cur, op: name, varName: varArg.segments[0], predicate: pred };
+            }
+            else {
+                cur = { kind: "method", receiver: cur, name, args };
+            }
+        }
+        return cur;
+    }
+    parseArgs() {
+        this.expect("lparen");
+        const args = [];
+        if (this.peek().type !== "rparen") {
+            args.push(this.parseExpr());
+            while (this.peek().type === "comma") {
+                this.next();
+                args.push(this.parseExpr());
+            }
+        }
+        this.expect("rparen");
+        return args;
+    }
+    parsePrimary() {
+        const t = this.peek();
+        if (t.type === "lparen") {
+            this.next();
+            const e = this.parseExpr();
+            this.expect("rparen");
+            return e;
+        }
+        if (t.type === "string") {
+            this.next();
+            return lit(t.value, "string");
+        }
+        if (t.type === "int") {
+            this.next();
+            return { kind: "literal", value: parseInt(t.value, 10), type: "int" };
+        }
+        if (t.type === "double") {
+            this.next();
+            return { kind: "literal", value: parseFloat(t.value), type: "double" };
+        }
+        if (t.type === "bool") {
+            this.next();
+            return { kind: "literal", value: t.value === "true", type: "bool" };
+        }
+        if (t.type === "null") {
+            this.next();
+            return { kind: "literal", value: null, type: "null" };
+        }
+        if (t.type === "field") {
+            this.next();
+            return { kind: "field", segments: [t.value], intrinsic: true };
+        }
+        if (t.type === "outer") {
+            this.next();
+            return { kind: "outerref", name: t.value };
+        }
+        if (t.type === "ident") {
+            if (t.value === "in")
+                throw new FilterInvalid("unexpected keyword `in`");
+            this.next();
+            // function call?
+            if (this.peek().type === "lparen") {
+                const args = this.parseArgs();
+                rejectClock(t.value);
+                return { kind: "call", name: t.value, args };
+            }
+            // bare field (frontmatter key or bare block field)
+            return { kind: "field", segments: [t.value], intrinsic: false };
+        }
+        if (t.type === "op" && (t.value === "+" || t.value === "-" || t.value === "*" || t.value === "/")) {
+            throw new FilterInvalid("arithmetic is not supported", "10 §3.1");
+        }
+        throw new FilterInvalid(`unexpected token '${t.value || t.type}' at ${t.pos}`);
+    }
+}
+function lit(value, type) {
+    return { kind: "literal", value, type };
+}
+function rejectClock(name) {
+    if (name === "now" || name === "random" || name === "rand") {
+        throw new FilterInvalid(`${name}() is rejected — determinism is load-bearing`, "10 §3.1");
+    }
+}
+//# sourceMappingURL=parser.js.map

package/dist/src/search/drain.d.ts

@@ -0,0 +1,39 @@
+import type { Store } from "../core/store/store.js";
+import { EmbeddingWorker } from "./embeddings.js";
+export interface DrainerOptions {
+    /** Coalesce a burst of mutations into one drain (default 500ms). */
+    debounceMs?: number;
+    /** Fired after a drain that embedded ≥1 block (progress/log hook). */
+    onDrain?: (result: {
+        embedded: number;
+        cached: number;
+    }) => void;
+    /** A drain threw (provider down, etc.). The drainer stays alive. */
+    onError?: (err: unknown) => void;
+}
+export declare class EmbedDrainer {
+    private store;
+    private repoId;
+    private worker;
+    private opts;
+    private readonly debounceMs;
+    private timer;
+    private dirty;
+    private running;
+    private closed;
+    constructor(store: Store, repoId: string, worker: EmbeddingWorker, opts?: DrainerOptions);
+    /**
+     * Mark the repo dirty and (re)arm the debounce timer. Cheap and synchronous —
+     * safe to call from a mutation-tool handler or a watcher checkpoint. Bursts
+     * within the debounce window collapse into a single drain. Does no DB work and
+     * never throws; the actual embedding happens later, off the caller's path.
+     */
+    schedule(): void;
+    /** Cancel any pending debounce and drain now; resolves when the drain (and any
+     *  re-run triggered by mutations arriving mid-drain) completes. */
+    flush(): Promise<void>;
+    /** Cancel the timer and await any in-flight drain; no drains start after. */
+    close(): Promise<void>;
+    private kick;
+    private loop;
+}

package/dist/src/search/drain.js

@@ -0,0 +1,97 @@
+import { EmbeddingWorker } from "./embeddings.js";
+import { buildEmbedTasks, buildDocEmbedTasks } from "./tasks.js";
+export class EmbedDrainer {
+    store;
+    repoId;
+    worker;
+    opts;
+    debounceMs;
+    timer = null;
+    dirty = false;
+    running = null;
+    closed = false;
+    constructor(store, repoId, worker, opts = {}) {
+        this.store = store;
+        this.repoId = repoId;
+        this.worker = worker;
+        this.opts = opts;
+        this.debounceMs = opts.debounceMs ?? 500;
+    }
+    /**
+     * Mark the repo dirty and (re)arm the debounce timer. Cheap and synchronous —
+     * safe to call from a mutation-tool handler or a watcher checkpoint. Bursts
+     * within the debounce window collapse into a single drain. Does no DB work and
+     * never throws; the actual embedding happens later, off the caller's path.
+     */
+    schedule() {
+        if (this.closed || !this.worker.available)
+            return;
+        this.dirty = true;
+        if (this.timer)
+            clearTimeout(this.timer);
+        this.timer = setTimeout(() => {
+            this.timer = null;
+            void this.kick();
+        }, this.debounceMs);
+        // Don't keep the process alive solely for a pending drain (hosts own their
+        // own lifetime via the transport/lease).
+        this.timer.unref?.();
+    }
+    /** Cancel any pending debounce and drain now; resolves when the drain (and any
+     *  re-run triggered by mutations arriving mid-drain) completes. */
+    async flush() {
+        if (this.timer) {
+            clearTimeout(this.timer);
+            this.timer = null;
+        }
+        if (this.closed || !this.worker.available)
+            return;
+        this.dirty = true;
+        await this.kick();
+    }
+    /** Cancel the timer and await any in-flight drain; no drains start after. */
+    async close() {
+        this.closed = true;
+        if (this.timer) {
+            clearTimeout(this.timer);
+            this.timer = null;
+        }
+        await this.running;
+    }
+    // Single-flight: at most one drain runs at a time. If mutations mark the repo
+    // dirty again while a drain is in flight, the loop re-runs once more after it,
+    // so the last edit is never left unembedded.
+    kick() {
+        if (this.running)
+            return this.running;
+        this.running = this.loop().finally(() => { this.running = null; });
+        return this.running;
+    }
+    async loop() {
+        while (this.dirty && !this.closed) {
+            this.dirty = false;
+            try {
+                // Blocks first: the pooled doc-embedding fallback reuses block vectors,
+                // so embedding blocks before docs means a large doc can pool from
+                // freshly-cached vectors in the same drain.
+                const tasks = buildEmbedTasks(this.store, this.repoId);
+                const result = await this.worker.process(tasks);
+                // Then docs: recompute any document whose content changed (a block edit
+                // shifts the whole-doc input, invalidating its cached vector), so doc
+                // vectors never drift from content.
+                const docTasks = buildDocEmbedTasks(this.store, this.repoId);
+                const docResult = await this.worker.processDocs(docTasks);
+                if (result.embedded > 0 || docResult.embedded > 0 || docResult.pooled > 0) {
+                    this.opts.onDrain?.({ embedded: result.embedded + docResult.embedded, cached: result.cached });
+                }
+            }
+            catch (err) {
+                this.opts.onError?.(err);
+                // Swallow — a transient provider failure must not crash the host. The
+                // next schedule() (or flush at shutdown) retries.
+                return;
+            }
+        }
+    }
+}
+//# sourceMappingURL=drain.js.map

package/dist/src/search/embeddings.d.ts

@@ -0,0 +1,131 @@
+import type { Store } from "../core/store/store.js";
+export interface EmbeddingProvider {
+    model: string;
+    dim: number;
+    /**
+     * Max input length (in tokens) the model accepts before it truncates. Used to
+     * decide the doc-embedding strategy: inputs within budget take the whole-doc
+     * path, larger ones fall back to pooling block vectors. Absent ⇒ the caller's
+     * configured/default budget applies (05 §6; don't hardcode the threshold).
+     */
+    maxInputTokens?: number;
+    /** Embed a batch of input strings → float32 vectors. */
+    embed(texts: string[]): Promise<number[][]>;
+}
+export declare class SemanticUnavailable extends Error {
+    code: "semantic_unavailable";
+}
+export declare function contextPrefix(input: {
+    docTitle: string;
+    path: string;
+    headingChain: string[];
+    blockType: string;
+}): string;
+export declare function embedInput(ctx: string, blockText: string): string;
+export declare function ctxHashHex(ctx: string): string;
+export interface EmbedTask {
+    blockId: string;
+    contentHashHex: string;
+    ctx: string;
+    text: string;
+}
+export declare function shouldEmbed(text: string, minTokens?: number): boolean;
+export declare function estimateTokens(text: string): number;
+export declare const DEFAULT_DOC_TOKEN_BUDGET = 512;
+export interface DocEmbedBlockRef {
+    contentHashHex: string;
+    ctx: string;
+    tokens: number;
+}
+export interface DocEmbedTask {
+    docId: string;
+    input: string;
+    blocks: DocEmbedBlockRef[];
+}
+export type DocEmbedMethod = "whole" | "pooled";
+export interface DocVectorRow {
+    vec: Float32Array;
+    method: DocEmbedMethod;
+}
+export declare class EmbeddingWorker {
+    private store;
+    private provider;
+    constructor(store: Store, provider: EmbeddingProvider | null);
+    get available(): boolean;
+    private cacheKey;
+    /** Look up a cached vector; null if absent (stale/missing). */
+    getCached(contentHashHex: string, ctx: string): Float32Array | null;
+    /**
+     * Embed a batch (skipping cache hits) and persist. Throws if no provider.
+     * Cache misses are embedded in chunks of `batchSize` (default:
+     * DEFAULT_EMBED_BATCH); `onProgress` fires after each chunk is persisted so
+     * callers can show live progress on a long drain.
+     */
+    process(tasks: EmbedTask[], opts?: {
+        batchSize?: number;
+        onProgress?: (p: {
+            embedded: number;
+            total: number;
+        }) => void;
+    }): Promise<{
+        embedded: number;
+        cached: number;
+    }>;
+    /** Embed a bare query string (no context prefix, 05 §6). */
+    embedQuery(query: string): Promise<Float32Array>;
+    private docTokenBudget;
+    /** The freshness key for a doc task: sha256 of the whole-doc embed input. */
+    private docInputHash;
+    /** Look up a cached doc vector; null if absent or stale (input changed). */
+    getCachedDoc(docId: string, input: string): DocVectorRow | null;
+    /** Doc ids whose current input has no fresh cached vector — the doc queue. */
+    staleDocs(tasks: DocEmbedTask[]): string[];
+    /**
+     * Compute one vector per document and persist. Whole-doc path (primary):
+     * embed the doc's input directly. Pooled fallback: only when the input
+     * exceeds the provider's token budget, pool the doc's already-cached block
+     * vectors (token-weighted mean, L2-normalized) — zero embedding calls. Skips
+     * docs whose fresh vector is already cached. Throws if no provider.
+     */
+    processDocs(tasks: DocEmbedTask[], opts?: {
+        batchSize?: number;
+        onProgress?: (p: {
+            embedded: number;
+            total: number;
+        }) => void;
+    }): Promise<{
+        embedded: number;
+        cached: number;
+        pooled: number;
+    }>;
+    /**
+     * Token-weighted mean of a document's cached block vectors, L2-normalized.
+     * Pool = Σ(wᵢ·vᵢ)/Σ(wᵢ) with wᵢ = block token count, then normalized (pooling
+     * breaks unit norm; downstream cosine assumes unit vectors). Returns null when
+     * no referenced block has a cached vector yet.
+     */
+    private poolBlockVectors;
+    /**
+     * Staleness: a heading rename changes descendants' context prefixes, so their
+     * cached vectors (keyed by the OLD ctx_hash) no longer match the current
+     * context. This returns the block ids in a doc whose current ctx has no cached
+     * vector — i.e. stale/pending — for the worker queue.
+     */
+    staleBlocks(tasks: EmbedTask[]): string[];
+    /**
+     * Vectors left behind by a previous model. The cache and both search paths key
+     * on model name, so after switching models the old model's rows are dead
+     * weight: never read, never overwritten (the new model writes under its own
+     * key). Reports how many block + doc vectors belong to a model OTHER than the
+     * current provider's, so callers can offer to reclaim the space.
+     */
+    foreignVectorCount(): {
+        blocks: number;
+        docs: number;
+    };
+    /** Delete every cached vector not produced by the current model. */
+    pruneForeignVectors(): {
+        blocks: number;
+        docs: number;
+    };
+}