@lapis-notes/api 0.1.0

package/dist/storage/search-embedding-provider.js

@@ -0,0 +1,446 @@
+async function defaultTransformersRuntimeLoader() {
+    return (await import("@huggingface/transformers"));
+}
+let transformersRuntimeLoader = defaultTransformersRuntimeLoader;
+const DEFAULT_DIMENSIONS = 48;
+const DEFAULT_REMOTE_HOST = "https://huggingface.co/";
+const DEFAULT_REMOTE_PATH_TEMPLATE = "{model}/resolve/{revision}/";
+function normalizeText(value) {
+    return value.trim().toLowerCase();
+}
+function hashString(value) {
+    let hash = 2166136261;
+    for (let index = 0; index < value.length; index += 1) {
+        hash ^= value.charCodeAt(index);
+        hash = Math.imul(hash, 16777619);
+    }
+    return hash >>> 0;
+}
+function featureTerms(value) {
+    const normalized = normalizeText(value);
+    if (!normalized.length) {
+        return [];
+    }
+    const features = [];
+    for (const token of normalized.match(/[\p{L}\p{N}_-]+/gu) ?? []) {
+        features.push(`tok:${token}`);
+        if (token.length >= 3) {
+            for (let index = 0; index <= token.length - 3; index += 1) {
+                features.push(`tri:${token.slice(index, index + 3)}`);
+            }
+        }
+    }
+    return [...new Set(features)];
+}
+function normalizeVector(values) {
+    const magnitude = Math.sqrt(values.reduce((total, value) => total + value * value, 0));
+    if (!magnitude) {
+        return values;
+    }
+    return values.map((value) => value / magnitude);
+}
+function embedText(text, dimensions) {
+    const vector = new Array(dimensions).fill(0);
+    for (const feature of featureTerms(text)) {
+        const hash = hashString(feature);
+        const index = hash % dimensions;
+        const sign = (hash & 1) === 0 ? 1 : -1;
+        vector[index] += sign;
+    }
+    return normalizeVector(vector);
+}
+function fingerprint(text) {
+    return hashString(normalizeText(text)).toString(16);
+}
+function trimModelPath(path) {
+    const normalized = path?.trim();
+    return normalized?.length ? normalized : undefined;
+}
+function normalizeEmbeddingRows(value) {
+    if (!Array.isArray(value)) {
+        return [];
+    }
+    if (value.length > 0 && value.every((entry) => typeof entry === "number")) {
+        return [value];
+    }
+    return value
+        .filter(Array.isArray)
+        .map((entry) => entry
+        .map((item) => Number(item))
+        .filter((item) => Number.isFinite(item)))
+        .filter((entry) => entry.length > 0);
+}
+function cloneRuntimeStatus(value) {
+    return { ...value };
+}
+function normalizeProgress(value) {
+    if (typeof value !== "number" || !Number.isFinite(value)) {
+        return undefined;
+    }
+    const normalized = value > 1 ? value / 100 : value;
+    return Math.max(0, Math.min(1, normalized));
+}
+function runtimeMessage(event) {
+    const parts = [event.status, event.name ?? event.file].filter(Boolean);
+    return parts.length ? parts.join(": ") : undefined;
+}
+class TokenHashSearchEmbeddingProvider {
+    config;
+    runtimeStatus;
+    constructor(config) {
+        this.config = {
+            ...config,
+            modelId: config.modelId ?? "lapis/token-hash-v0",
+            modelVersion: config.modelVersion ?? "0",
+            dimensions: config.dimensions ?? DEFAULT_DIMENSIONS,
+        };
+        this.runtimeStatus = {
+            providerKind: "token-hash",
+            phase: "ready",
+            modelId: this.config.modelId,
+            message: "Token-hash fallback is ready",
+            progress: 1,
+            updatedAt: Date.now(),
+        };
+    }
+    async ready() {
+        return true;
+    }
+    async embedDocument(document) {
+        const dimensions = this.config.dimensions ?? DEFAULT_DIMENSIONS;
+        return (document.chunks ?? [])
+            .filter((chunk) => chunk.text.trim().length)
+            .map((chunk) => ({
+            chunkId: chunk.id,
+            vector: embedText(chunk.text, dimensions),
+            fingerprint: fingerprint(chunk.text),
+        }));
+    }
+    async embedQuery(query) {
+        if (!featureTerms(query).length) {
+            return null;
+        }
+        return embedText(query, this.config.dimensions ?? DEFAULT_DIMENSIONS);
+    }
+    getRuntimeStatus() {
+        return cloneRuntimeStatus(this.runtimeStatus);
+    }
+}
+const EMBEDDING_WORKER_TIMEOUT_MS = 5 * 60_000;
+let browserSearchEmbeddingWorkerFactory = null;
+function canUseBrowserEmbeddingWorker() {
+    return (typeof window !== "undefined" &&
+        browserSearchEmbeddingWorkerFactory !== null);
+}
+export function setBrowserSearchEmbeddingWorkerFactory(factory) {
+    browserSearchEmbeddingWorkerFactory = factory;
+}
+function createEmbeddingWorkerRequestId() {
+    if (typeof crypto !== "undefined" &&
+        typeof crypto.randomUUID === "function") {
+        return crypto.randomUUID();
+    }
+    return `embedding-${Date.now()}-${Math.random().toString(16).slice(2)}`;
+}
+class BrowserWorkerSearchEmbeddingProvider {
+    config;
+    worker = null;
+    runtimeStatus;
+    pending = new Map();
+    constructor(config) {
+        this.config = { ...config };
+        this.runtimeStatus = {
+            providerKind: "transformers-js",
+            phase: "downloading",
+            modelId: config.modelId,
+            message: "Waiting to initialize embedding worker",
+            progress: 0,
+            updatedAt: Date.now(),
+        };
+    }
+    ready() {
+        return this.invoke("ready");
+    }
+    embedDocument(document) {
+        return this.invoke("embedDocument", document);
+    }
+    embedQuery(query) {
+        return this.invoke("embedQuery", query);
+    }
+    getRuntimeStatus() {
+        return cloneRuntimeStatus(this.runtimeStatus);
+    }
+    dispose() {
+        this.worker?.terminate();
+        this.worker = null;
+        for (const request of this.pending.values()) {
+            clearTimeout(request.timeoutId);
+            request.reject(new Error("Embedding worker disposed"));
+        }
+        this.pending.clear();
+    }
+    invoke(method, argument) {
+        const worker = this.ensureWorker();
+        const requestId = createEmbeddingWorkerRequestId();
+        return new Promise((resolve, reject) => {
+            const timeoutId = setTimeout(() => {
+                this.pending.delete(requestId);
+                reject(new Error(`Embedding worker request timed out: ${method}`));
+            }, EMBEDDING_WORKER_TIMEOUT_MS);
+            this.pending.set(requestId, {
+                resolve: (value) => resolve(value),
+                reject,
+                timeoutId,
+            });
+            worker.postMessage({
+                type: "search-embedding-request",
+                requestId,
+                method,
+                config: this.config,
+                argument,
+            });
+        });
+    }
+    ensureWorker() {
+        if (this.worker)
+            return this.worker;
+        if (!browserSearchEmbeddingWorkerFactory) {
+            throw new Error("Embedding worker factory is unavailable");
+        }
+        const worker = browserSearchEmbeddingWorkerFactory();
+        worker.addEventListener("message", (event) => {
+            const message = event.data;
+            if (message?.type === "search-embedding-status") {
+                this.runtimeStatus = cloneRuntimeStatus(message.status);
+                return;
+            }
+            if (message?.type !== "search-embedding-response")
+                return;
+            const request = this.pending.get(message.requestId);
+            if (!request)
+                return;
+            if (message.status) {
+                this.runtimeStatus = cloneRuntimeStatus(message.status);
+            }
+            clearTimeout(request.timeoutId);
+            this.pending.delete(message.requestId);
+            if (message.success)
+                request.resolve(message.result);
+            else
+                request.reject(new Error(message.error ?? "Embedding worker failed"));
+        });
+        worker.addEventListener("error", (event) => {
+            const error = new Error(event.message || "Embedding worker crashed");
+            this.runtimeStatus = {
+                ...this.runtimeStatus,
+                phase: "error",
+                message: error.message,
+                updatedAt: Date.now(),
+            };
+            for (const request of this.pending.values()) {
+                clearTimeout(request.timeoutId);
+                request.reject(error);
+            }
+            this.pending.clear();
+            worker.terminate();
+            if (this.worker === worker)
+                this.worker = null;
+        });
+        this.worker = worker;
+        return worker;
+    }
+}
+class TransformersJsSearchEmbeddingProvider {
+    config;
+    extractorPromise = null;
+    runtimeStatus;
+    sawDownloadProgress = false;
+    constructor(config) {
+        this.config = {
+            ...config,
+            modelVersion: config.modelVersion ?? config.revision,
+            device: config.device ?? "wasm",
+            pooling: config.pooling ?? "mean",
+            normalize: config.normalize ?? true,
+            allowRemoteModels: config.allowRemoteModels ?? true,
+            localModelPath: trimModelPath(config.localModelPath),
+            wasmPaths: trimModelPath(config.wasmPaths),
+        };
+        this.runtimeStatus = {
+            providerKind: "transformers-js",
+            phase: "downloading",
+            modelId: this.config.modelId,
+            message: "Waiting to initialize embedding runtime",
+            progress: 0,
+            updatedAt: Date.now(),
+        };
+    }
+    async ready() {
+        await this.getExtractor();
+        return true;
+    }
+    async embedDocument(document) {
+        const chunks = (document.chunks ?? []).filter((chunk) => chunk.text.trim().length);
+        if (!chunks.length) {
+            return [];
+        }
+        const vectors = await this.embedTexts(chunks.map((chunk) => chunk.text));
+        return chunks
+            .map((chunk, index) => {
+            const vector = vectors[index];
+            if (!vector?.length) {
+                return null;
+            }
+            return {
+                chunkId: chunk.id,
+                vector,
+                fingerprint: fingerprint(chunk.text),
+            };
+        })
+            .filter((entry) => Boolean(entry));
+    }
+    async embedQuery(query) {
+        if (!query.trim().length) {
+            return null;
+        }
+        const [vector] = await this.embedTexts([query]);
+        return vector?.length ? vector : null;
+    }
+    getRuntimeStatus() {
+        return cloneRuntimeStatus(this.runtimeStatus);
+    }
+    async embedTexts(texts) {
+        const extractor = await this.getExtractor();
+        this.updateRuntimeStatus({
+            phase: "embedding",
+            message: `Embedding ${texts.length} item${texts.length === 1 ? "" : "s"}`,
+            progress: 1,
+        });
+        try {
+            const output = await extractor(texts, {
+                pooling: this.config.pooling,
+                normalize: this.config.normalize,
+            });
+            return normalizeEmbeddingRows(output.tolist());
+        }
+        finally {
+            this.updateRuntimeStatus({
+                phase: "ready",
+                message: this.readyMessage(),
+                progress: 1,
+                file: undefined,
+                loadedBytes: undefined,
+                totalBytes: undefined,
+            });
+        }
+    }
+    async getExtractor() {
+        if (!this.extractorPromise) {
+            this.extractorPromise = this.loadExtractor().catch((error) => {
+                this.extractorPromise = null;
+                this.updateRuntimeStatus({
+                    phase: "error",
+                    message: error instanceof Error ? error.message : String(error),
+                });
+                throw error;
+            });
+        }
+        return this.extractorPromise;
+    }
+    async loadExtractor() {
+        const runtime = await transformersRuntimeLoader();
+        const hasExplicitLocalModelPath = Boolean(this.config.localModelPath);
+        runtime.env.allowLocalModels = hasExplicitLocalModelPath;
+        runtime.env.allowRemoteModels = this.config.allowRemoteModels ?? true;
+        runtime.env.remoteHost ||= DEFAULT_REMOTE_HOST;
+        runtime.env.remotePathTemplate ||= DEFAULT_REMOTE_PATH_TEMPLATE;
+        if (hasExplicitLocalModelPath) {
+            runtime.env.localModelPath = this.config.localModelPath;
+        }
+        if (this.config.wasmPaths) {
+            runtime.env.backends ??= {};
+            runtime.env.backends.onnx ??= {};
+            runtime.env.backends.onnx.wasm ??= {};
+            runtime.env.backends.onnx.wasm.wasmPaths = this.config.wasmPaths;
+        }
+        this.updateRuntimeStatus({
+            phase: "downloading",
+            message: "Initializing embedding runtime",
+            progress: 0,
+        });
+        const extractor = await runtime.pipeline("feature-extraction", this.config.modelId, {
+            revision: this.config.revision,
+            dtype: this.config.dtype,
+            device: this.config.device,
+            progress_callback: (event) => {
+                this.sawDownloadProgress = true;
+                this.updateRuntimeStatus({
+                    phase: "downloading",
+                    message: runtimeMessage(event) ?? "Downloading embedding model",
+                    progress: normalizeProgress(event.progress) ?? this.runtimeStatus.progress,
+                    file: event.file ?? event.name,
+                    loadedBytes: typeof event.loaded === "number" ? event.loaded : undefined,
+                    totalBytes: typeof event.total === "number" ? event.total : undefined,
+                });
+            },
+        });
+        this.updateRuntimeStatus({
+            phase: "ready",
+            message: this.readyMessage(),
+            progress: 1,
+            file: undefined,
+            loadedBytes: undefined,
+            totalBytes: undefined,
+        });
+        return extractor;
+    }
+    updateRuntimeStatus(patch) {
+        this.runtimeStatus = {
+            ...this.runtimeStatus,
+            ...patch,
+            providerKind: "transformers-js",
+            modelId: patch.modelId ?? this.runtimeStatus.modelId ?? this.config.modelId,
+            updatedAt: Date.now(),
+        };
+    }
+    readyMessage() {
+        if (this.sawDownloadProgress) {
+            return "Embedding model downloaded and cached in browser";
+        }
+        if (this.config.allowRemoteModels === false && this.config.localModelPath) {
+            return `Embedding model loaded from ${this.config.localModelPath}`;
+        }
+        if (this.config.allowRemoteModels === false) {
+            return "Embedding model loaded without remote downloads";
+        }
+        return "Embedding model ready from browser cache or existing local files";
+    }
+}
+export function createSearchEmbeddingProvider(config) {
+    if (!config) {
+        return null;
+    }
+    switch (config.kind) {
+        case "token-hash":
+            return new TokenHashSearchEmbeddingProvider(config);
+        case "transformers-js":
+            return canUseBrowserEmbeddingWorker()
+                ? new BrowserWorkerSearchEmbeddingProvider(config)
+                : new TransformersJsSearchEmbeddingProvider(config);
+        default:
+            return null;
+    }
+}
+export function setSearchEmbeddingProviderRuntimeLoaderForTests(loader) {
+    transformersRuntimeLoader = loader ?? defaultTransformersRuntimeLoader;
+}
+export function cosineSimilarity(left, right) {
+    if (!left.length || left.length !== right.length) {
+        return 0;
+    }
+    let score = 0;
+    for (let index = 0; index < left.length; index += 1) {
+        score += left[index] * right[index];
+    }
+    return score;
+}

package/dist/storage/search-embedding-worker.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/storage/search-embedding-worker.js

@@ -0,0 +1,63 @@
+import { createSearchEmbeddingProvider, } from "./search-embedding-provider";
+const workerScope = globalThis;
+let activeConfig = "";
+let provider = null;
+async function ensureProvider(config) {
+    const serialized = JSON.stringify(config);
+    if (provider && serialized === activeConfig)
+        return provider;
+    await provider?.dispose?.();
+    provider = createSearchEmbeddingProvider(config);
+    activeConfig = serialized;
+    if (!provider)
+        throw new Error("Embedding provider is unavailable");
+    return provider;
+}
+function publishStatus(current) {
+    workerScope.postMessage({
+        type: "search-embedding-status",
+        status: current.getRuntimeStatus(),
+    });
+}
+workerScope.addEventListener("message", (event) => {
+    const request = event.data;
+    if (!request || request.type !== "search-embedding-request")
+        return;
+    void (async () => {
+        let current = null;
+        let statusTimer = null;
+        try {
+            current = await ensureProvider(request.config);
+            publishStatus(current);
+            statusTimer = setInterval(() => publishStatus(current), 100);
+            const result = request.method === "ready"
+                ? await current.ready()
+                : request.method === "embedDocument"
+                    ? await current.embedDocument(request.argument)
+                    : await current.embedQuery(String(request.argument ?? ""));
+            publishStatus(current);
+            workerScope.postMessage({
+                type: "search-embedding-response",
+                requestId: request.requestId,
+                success: true,
+                result,
+                status: current.getRuntimeStatus(),
+            });
+        }
+        catch (error) {
+            if (current)
+                publishStatus(current);
+            workerScope.postMessage({
+                type: "search-embedding-response",
+                requestId: request.requestId,
+                success: false,
+                error: error instanceof Error ? error.message : String(error),
+                status: current?.getRuntimeStatus(),
+            });
+        }
+        finally {
+            if (statusTimer)
+                clearInterval(statusTimer);
+        }
+    })();
+});

package/dist/storage/search-query-evaluator.d.ts

@@ -0,0 +1,14 @@
+import type { AppDatabasePropertyRecord, AppDatabaseSearchField, AppDatabaseSearchRange, SearchDocumentRecord } from "./app-database";
+export interface SearchQueryEvaluationOptions {
+    caseSensitive?: boolean;
+}
+export interface SearchQueryTextRangeOptions extends SearchQueryEvaluationOptions {
+    field?: AppDatabaseSearchField;
+}
+export interface SearchQueryDocumentEvaluation {
+    matched: boolean;
+    score: number;
+}
+export declare function isStructuredSearchQuery(query: string): boolean;
+export declare function evaluateSearchQueryForDocument(document: SearchDocumentRecord, query: string, properties?: AppDatabasePropertyRecord[], options?: SearchQueryEvaluationOptions): SearchQueryDocumentEvaluation;
+export declare function findSearchQueryRangesInText(text: string, query: string, options?: SearchQueryTextRangeOptions): AppDatabaseSearchRange[];