@stoneforge/quarry 0.1.0

package/dist/services/embeddings/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/services/embeddings/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACnD,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC7D,YAAY,EACV,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,sBAAsB,GACvB,MAAM,YAAY,CAAC"}

package/dist/services/embeddings/index.js

@@ -0,0 +1,10 @@
+/**
+ * Embeddings Module
+ *
+ * Public API for document embedding services: semantic search,
+ * hybrid search (RRF fusion), and local embedding providers.
+ */
+export { EmbeddingService } from './service.js';
+export { LocalEmbeddingProvider } from './local-provider.js';
+export { reciprocalRankFusion } from './fusion.js';
+//# sourceMappingURL=index.js.map

package/dist/services/embeddings/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/services/embeddings/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,sBAAsB,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC"}

package/dist/services/embeddings/local-provider.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Local Embedding Provider
+ *
+ * Uses a local ONNX-based model for embedding generation.
+ * Model is downloaded on first use via `sf embeddings install`.
+ *
+ * Default model: bge-base-en-v1.5 (768 dimensions)
+ *
+ * This is a placeholder implementation. The actual ONNX runtime integration
+ * will be added when the model download infrastructure is in place.
+ */
+import type { EmbeddingProvider } from './types.js';
+export declare class LocalEmbeddingProvider implements EmbeddingProvider {
+    readonly name = "local-bge-base";
+    readonly dimensions = 768;
+    readonly isLocal = true;
+    private modelDir;
+    private initialized;
+    constructor(modelDir?: string);
+    embed(text: string): Promise<Float32Array>;
+    embedBatch(texts: string[]): Promise<Float32Array[]>;
+    isAvailable(): Promise<boolean>;
+    private ensureInitialized;
+    /**
+     * Generate a deterministic pseudo-embedding from text content.
+     * This is a placeholder until ONNX runtime integration is complete.
+     * Uses a simple hash-based approach to produce consistent vectors.
+     */
+    private pseudoEmbed;
+}
+//# sourceMappingURL=local-provider.d.ts.map

package/dist/services/embeddings/local-provider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"local-provider.d.ts","sourceRoot":"","sources":["../../../src/services/embeddings/local-provider.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAIH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAOpD,qBAAa,sBAAuB,YAAW,iBAAiB;IAC9D,QAAQ,CAAC,IAAI,oBAAoB;IACjC,QAAQ,CAAC,UAAU,OAAsB;IACzC,QAAQ,CAAC,OAAO,QAAQ;IAExB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,WAAW,CAAS;gBAEhB,QAAQ,CAAC,EAAE,MAAM;IAIvB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAO1C,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAKpD,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;YAIvB,iBAAiB;IAc/B;;;;OAIG;IACH,OAAO,CAAC,WAAW;CAwBpB"}

package/dist/services/embeddings/local-provider.js

@@ -0,0 +1,80 @@
+/**
+ * Local Embedding Provider
+ *
+ * Uses a local ONNX-based model for embedding generation.
+ * Model is downloaded on first use via `sf embeddings install`.
+ *
+ * Default model: bge-base-en-v1.5 (768 dimensions)
+ *
+ * This is a placeholder implementation. The actual ONNX runtime integration
+ * will be added when the model download infrastructure is in place.
+ */
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+/** Default model directory relative to workspace root */
+const DEFAULT_MODEL_DIR = '.stoneforge/models';
+const DEFAULT_MODEL_NAME = 'bge-base-en-v1.5';
+const DEFAULT_DIMENSIONS = 768;
+export class LocalEmbeddingProvider {
+    name = 'local-bge-base';
+    dimensions = DEFAULT_DIMENSIONS;
+    isLocal = true;
+    modelDir;
+    initialized = false;
+    constructor(modelDir) {
+        this.modelDir = modelDir ?? join(process.cwd(), DEFAULT_MODEL_DIR, DEFAULT_MODEL_NAME);
+    }
+    async embed(text) {
+        await this.ensureInitialized();
+        // Placeholder: generate a deterministic pseudo-embedding from text hash
+        // Real implementation will use ONNX runtime
+        return this.pseudoEmbed(text);
+    }
+    async embedBatch(texts) {
+        await this.ensureInitialized();
+        return texts.map((text) => this.pseudoEmbed(text));
+    }
+    async isAvailable() {
+        return existsSync(this.modelDir);
+    }
+    async ensureInitialized() {
+        if (this.initialized)
+            return;
+        const available = await this.isAvailable();
+        if (!available) {
+            throw new Error(`Local embedding model not installed. Run 'sf embeddings install' to download the model.`);
+        }
+        // TODO: Initialize ONNX runtime session
+        this.initialized = true;
+    }
+    /**
+     * Generate a deterministic pseudo-embedding from text content.
+     * This is a placeholder until ONNX runtime integration is complete.
+     * Uses a simple hash-based approach to produce consistent vectors.
+     */
+    pseudoEmbed(text) {
+        const embedding = new Float32Array(this.dimensions);
+        let hash = 0;
+        for (let i = 0; i < text.length; i++) {
+            hash = ((hash << 5) - hash + text.charCodeAt(i)) | 0;
+        }
+        for (let i = 0; i < this.dimensions; i++) {
+            // Use simple PRNG seeded from hash
+            hash = (hash * 1103515245 + 12345) & 0x7fffffff;
+            embedding[i] = (hash / 0x7fffffff) * 2 - 1;
+        }
+        // Normalize to unit vector
+        let norm = 0;
+        for (let i = 0; i < this.dimensions; i++) {
+            norm += embedding[i] * embedding[i];
+        }
+        norm = Math.sqrt(norm);
+        if (norm > 0) {
+            for (let i = 0; i < this.dimensions; i++) {
+                embedding[i] /= norm;
+            }
+        }
+        return embedding;
+    }
+}
+//# sourceMappingURL=local-provider.js.map

package/dist/services/embeddings/local-provider.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"local-provider.js","sourceRoot":"","sources":["../../../src/services/embeddings/local-provider.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,yDAAyD;AACzD,MAAM,iBAAiB,GAAG,oBAAoB,CAAC;AAC/C,MAAM,kBAAkB,GAAG,kBAAkB,CAAC;AAC9C,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAE/B,MAAM,OAAO,sBAAsB;IACxB,IAAI,GAAG,gBAAgB,CAAC;IACxB,UAAU,GAAG,kBAAkB,CAAC;IAChC,OAAO,GAAG,IAAI,CAAC;IAEhB,QAAQ,CAAS;IACjB,WAAW,GAAG,KAAK,CAAC;IAE5B,YAAY,QAAiB;QAC3B,IAAI,CAAC,QAAQ,GAAG,QAAQ,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,iBAAiB,EAAE,kBAAkB,CAAC,CAAC;IACzF,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,wEAAwE;QACxE,4CAA4C;QAC5C,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC/B,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,WAAW;QACf,OAAO,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAEO,KAAK,CAAC,iBAAiB;QAC7B,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO;QAE7B,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAC3C,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;QACJ,CAAC;QAED,wCAAwC;QACxC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED;;;;OAIG;IACK,WAAW,CAAC,IAAY;QAC9B,MAAM,SAAS,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACpD,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACvD,CAAC;QACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACzC,mCAAmC;YACnC,IAAI,GAAG,CAAC,IAAI,GAAG,UAAU,GAAG,KAAK,CAAC,GAAG,UAAU,CAAC;YAChD,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7C,CAAC;QACD,2BAA2B;QAC3B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;YACzC,IAAI,IAAI,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;YACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;gBACzC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;YACvB,CAAC;QACH,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;CACF"}

package/dist/services/embeddings/service.d.ts

@@ -0,0 +1,76 @@
+/**
+ * Embedding Service
+ *
+ * Manages document embeddings for semantic search.
+ * Handles indexing, removal, and similarity search using a configurable provider.
+ */
+import type { StorageBackend } from '@stoneforge/storage';
+import type { EmbeddingServiceConfig, SemanticSearchResult } from './types.js';
+/**
+ * Service for managing document embeddings and performing semantic search.
+ */
+export declare class EmbeddingService {
+    private provider;
+    private backend;
+    private batchSize;
+    constructor(backend: StorageBackend, config: EmbeddingServiceConfig);
+    /**
+     * Check if the embedding service is available.
+     */
+    isAvailable(): Promise<boolean>;
+    /**
+     * Get provider info.
+     */
+    getProviderInfo(): {
+        name: string;
+        dimensions: number;
+        isLocal: boolean;
+    };
+    /**
+     * Generate and store an embedding for a document.
+     *
+     * @param docId - Document ID
+     * @param content - Document content to embed (typically title + content)
+     */
+    indexDocument(docId: string, content: string): Promise<void>;
+    /**
+     * Remove the embedding for a document.
+     */
+    removeDocument(docId: string): void;
+    /**
+     * Perform semantic search using cosine similarity.
+     * Uses brute-force comparison (no ANN index yet).
+     *
+     * @param query - Search query text
+     * @param limit - Maximum results to return
+     * @returns Ranked results by similarity (descending)
+     */
+    searchSemantic(query: string, limit?: number): Promise<SemanticSearchResult[]>;
+    /**
+     * Hybrid search combining FTS5 results with semantic results via RRF.
+     *
+     * @param query - Search query text
+     * @param ftsDocIds - Document IDs from FTS5 search (in rank order)
+     * @param limit - Maximum results to return
+     * @param k - RRF smoothing constant (default: 60)
+     * @returns Fused document IDs ranked by combined score
+     */
+    searchHybrid(query: string, ftsDocIds: string[], limit?: number, k?: number): Promise<{
+        documentId: string;
+        score: number;
+    }[]>;
+    /**
+     * Re-embed all documents. Used for bulk reindexing.
+     *
+     * @param documents - Array of { id, content } to embed
+     * @param onProgress - Optional callback for progress reporting
+     */
+    reindexAll(documents: Array<{
+        id: string;
+        content: string;
+    }>, onProgress?: (indexed: number, total: number) => void): Promise<{
+        indexed: number;
+        errors: number;
+    }>;
+}
+//# sourceMappingURL=service.d.ts.map

package/dist/services/embeddings/service.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"service.d.ts","sourceRoot":"","sources":["../../../src/services/embeddings/service.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,KAAK,EAAqB,sBAAsB,EAAE,oBAAoB,EAAmB,MAAM,YAAY,CAAC;AAenH;;GAEG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,QAAQ,CAAoB;IACpC,OAAO,CAAC,OAAO,CAAiB;IAChC,OAAO,CAAC,SAAS,CAAS;gBAEd,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,sBAAsB;IAMnE;;OAEG;IACG,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC;IAIrC;;OAEG;IACH,eAAe,IAAI;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,OAAO,CAAA;KAAE;IAQzE;;;;;OAKG;IACG,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBlE;;OAEG;IACH,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAOnC;;;;;;;OAOG;IACG,cAAc,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,GAAE,MAAW,GAAG,OAAO,CAAC,oBAAoB,EAAE,CAAC;IA2BxF;;;;;;;;OAQG;IACG,YAAY,CAChB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EAAE,EACnB,KAAK,GAAE,MAAW,EAClB,CAAC,GAAE,MAAW,GACb,OAAO,CAAC;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAkBnD;;;;;OAKG;IACG,UAAU,CACd,SAAS,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,EACjD,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GACpD,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;CAyChD"}

package/dist/services/embeddings/service.js

@@ -0,0 +1,153 @@
+/**
+ * Embedding Service
+ *
+ * Manages document embeddings for semantic search.
+ * Handles indexing, removal, and similarity search using a configurable provider.
+ */
+import { reciprocalRankFusion } from './fusion.js';
+/**
+ * Cosine similarity between two vectors.
+ * Assumes both are unit-normalized for efficiency.
+ */
+function cosineSimilarity(a, b) {
+    let dot = 0;
+    for (let i = 0; i < a.length; i++) {
+        dot += a[i] * b[i];
+    }
+    return dot;
+}
+/**
+ * Service for managing document embeddings and performing semantic search.
+ */
+export class EmbeddingService {
+    provider;
+    backend;
+    batchSize;
+    constructor(backend, config) {
+        this.backend = backend;
+        this.provider = config.provider;
+        this.batchSize = config.batchSize ?? 32;
+    }
+    /**
+     * Check if the embedding service is available.
+     */
+    async isAvailable() {
+        return this.provider.isAvailable();
+    }
+    /**
+     * Get provider info.
+     */
+    getProviderInfo() {
+        return {
+            name: this.provider.name,
+            dimensions: this.provider.dimensions,
+            isLocal: this.provider.isLocal,
+        };
+    }
+    /**
+     * Generate and store an embedding for a document.
+     *
+     * @param docId - Document ID
+     * @param content - Document content to embed (typically title + content)
+     */
+    async indexDocument(docId, content) {
+        const embedding = await this.provider.embed(content);
+        const blob = Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
+        this.backend.run(`INSERT OR REPLACE INTO document_embeddings (document_id, embedding, dimensions, provider, model, created_at)
+       VALUES (?, ?, ?, ?, ?, ?)`, [
+            docId,
+            blob,
+            this.provider.dimensions,
+            this.provider.name,
+            this.provider.name,
+            new Date().toISOString(),
+        ]);
+    }
+    /**
+     * Remove the embedding for a document.
+     */
+    removeDocument(docId) {
+        this.backend.run(`DELETE FROM document_embeddings WHERE document_id = ?`, [docId]);
+    }
+    /**
+     * Perform semantic search using cosine similarity.
+     * Uses brute-force comparison (no ANN index yet).
+     *
+     * @param query - Search query text
+     * @param limit - Maximum results to return
+     * @returns Ranked results by similarity (descending)
+     */
+    async searchSemantic(query, limit = 20) {
+        const queryEmbedding = await this.provider.embed(query);
+        // Load all embeddings (brute-force for now)
+        const rows = this.backend.query('SELECT document_id, embedding, dimensions FROM document_embeddings');
+        const results = [];
+        for (const row of rows) {
+            const stored = new Float32Array(row.embedding.buffer, row.embedding.byteOffset, row.dimensions);
+            const similarity = cosineSimilarity(queryEmbedding, stored);
+            results.push({ documentId: row.document_id, similarity });
+        }
+        // Sort by similarity descending and return top results
+        results.sort((a, b) => b.similarity - a.similarity);
+        return results.slice(0, limit);
+    }
+    /**
+     * Hybrid search combining FTS5 results with semantic results via RRF.
+     *
+     * @param query - Search query text
+     * @param ftsDocIds - Document IDs from FTS5 search (in rank order)
+     * @param limit - Maximum results to return
+     * @param k - RRF smoothing constant (default: 60)
+     * @returns Fused document IDs ranked by combined score
+     */
+    async searchHybrid(query, ftsDocIds, limit = 20, k = 60) {
+        // Get semantic results
+        const semanticResults = await this.searchSemantic(query, limit * 2);
+        // Build ranked result sets
+        const ftsRanking = ftsDocIds.map((documentId, index) => ({
+            documentId,
+            rank: index + 1,
+        }));
+        const semanticRanking = semanticResults.map((result, index) => ({
+            documentId: result.documentId,
+            rank: index + 1,
+        }));
+        return reciprocalRankFusion([ftsRanking, semanticRanking], k, limit);
+    }
+    /**
+     * Re-embed all documents. Used for bulk reindexing.
+     *
+     * @param documents - Array of { id, content } to embed
+     * @param onProgress - Optional callback for progress reporting
+     */
+    async reindexAll(documents, onProgress) {
+        let indexed = 0;
+        let errors = 0;
+        for (let i = 0; i < documents.length; i += this.batchSize) {
+            const batch = documents.slice(i, i + this.batchSize);
+            const texts = batch.map((d) => d.content);
+            try {
+                const embeddings = await this.provider.embedBatch(texts);
+                for (let j = 0; j < batch.length; j++) {
+                    const blob = Buffer.from(embeddings[j].buffer, embeddings[j].byteOffset, embeddings[j].byteLength);
+                    this.backend.run(`INSERT OR REPLACE INTO document_embeddings (document_id, embedding, dimensions, provider, model, created_at)
+             VALUES (?, ?, ?, ?, ?, ?)`, [
+                        batch[j].id,
+                        blob,
+                        this.provider.dimensions,
+                        this.provider.name,
+                        this.provider.name,
+                        new Date().toISOString(),
+                    ]);
+                    indexed++;
+                }
+            }
+            catch {
+                errors += batch.length;
+            }
+            onProgress?.(indexed + errors, documents.length);
+        }
+        return { indexed, errors };
+    }
+}
+//# sourceMappingURL=service.js.map

package/dist/services/embeddings/service.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"service.js","sourceRoot":"","sources":["../../../src/services/embeddings/service.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,EAAE,oBAAoB,EAAqB,MAAM,aAAa,CAAC;AAEtE;;;GAGG;AACH,SAAS,gBAAgB,CAAC,CAAe,EAAE,CAAe;IACxD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;GAEG;AACH,MAAM,OAAO,gBAAgB;IACnB,QAAQ,CAAoB;IAC5B,OAAO,CAAiB;IACxB,SAAS,CAAS;IAE1B,YAAY,OAAuB,EAAE,MAA8B;QACjE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;QAChC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW;QACf,OAAO,IAAI,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IACrC,CAAC;IAED;;OAEG;IACH,eAAe;QACb,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;YACxB,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU;YACpC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,OAAO;SAC/B,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,aAAa,CAAC,KAAa,EAAE,OAAe;QAChD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QACrD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC;QAEvF,IAAI,CAAC,OAAO,CAAC,GAAG,CACd;iCAC2B,EAC3B;YACE,KAAK;YACL,IAAI;YACJ,IAAI,CAAC,QAAQ,CAAC,UAAU;YACxB,IAAI,CAAC,QAAQ,CAAC,IAAI;YAClB,IAAI,CAAC,QAAQ,CAAC,IAAI;YAClB,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACzB,CACF,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,cAAc,CAAC,KAAa;QAC1B,IAAI,CAAC,OAAO,CAAC,GAAG,CACd,uDAAuD,EACvD,CAAC,KAAK,CAAC,CACR,CAAC;IACJ,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,cAAc,CAAC,KAAa,EAAE,QAAgB,EAAE;QACpD,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAExD,4CAA4C;QAC5C,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAI5B,oEAAoE,CAAC,CAAC;QAEzE,MAAM,OAAO,GAA2B,EAAE,CAAC;QAE3C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,MAAM,GAAG,IAAI,YAAY,CAC7B,GAAG,CAAC,SAAS,CAAC,MAAM,EACpB,GAAG,CAAC,SAAS,CAAC,UAAU,EACxB,GAAG,CAAC,UAAU,CACf,CAAC;YACF,MAAM,UAAU,GAAG,gBAAgB,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC;YAC5D,OAAO,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,WAAW,EAAE,UAAU,EAAE,CAAC,CAAC;QAC5D,CAAC;QAED,uDAAuD;QACvD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;QACpD,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACjC,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,YAAY,CAChB,KAAa,EACb,SAAmB,EACnB,QAAgB,EAAE,EAClB,IAAY,EAAE;QAEd,uBAAuB;QACvB,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;QAEpE,2BAA2B;QAC3B,MAAM,UAAU,GAAmB,SAAS,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;YACvE,UAAU;YACV,IAAI,EAAE,KAAK,GAAG,CAAC;SAChB,CAAC,CAAC,CAAC;QAEJ,MAAM,eAAe,GAAmB,eAAe,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;YAC9E,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,IAAI,EAAE,KAAK,GAAG,CAAC;SAChB,CAAC,CAAC,CAAC;QAEJ,OAAO,oBAAoB,CAAC,CAAC,UAAU,EAAE,eAAe,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC;IACvE,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CACd,SAAiD,EACjD,UAAqD;QAErD,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,MAAM,GAAG,CAAC,CAAC;QAEf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAC1D,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;YACrD,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAE1C,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;gBAEzD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;oBACtC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CACtB,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,EACpB,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,EACxB,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CACzB,CAAC;oBAEF,IAAI,CAAC,OAAO,CAAC,GAAG,CACd;uCAC2B,EAC3B;wBACE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE;wBACX,IAAI;wBACJ,IAAI,CAAC,QAAQ,CAAC,UAAU;wBACxB,IAAI,CAAC,QAAQ,CAAC,IAAI;wBAClB,IAAI,CAAC,QAAQ,CAAC,IAAI;wBAClB,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;qBACzB,CACF,CAAC;oBACF,OAAO,EAAE,CAAC;gBACZ,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;YACzB,CAAC;YAED,UAAU,EAAE,CAAC,OAAO,GAAG,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;QACnD,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IAC7B,CAAC;CACF"}

package/dist/services/embeddings/types.d.ts

@@ -0,0 +1,70 @@
+/**
+ * Embedding Provider Types
+ *
+ * Defines the interface for embedding providers used in semantic search.
+ * Providers generate vector representations of text for similarity matching.
+ */
+/**
+ * An embedding provider generates vector representations of text.
+ *
+ * Implementations can use local models (ONNX) or remote APIs (OpenAI, Voyage).
+ */
+export interface EmbeddingProvider {
+    /** Human-readable provider name */
+    readonly name: string;
+    /** Dimensionality of output vectors */
+    readonly dimensions: number;
+    /** Whether the provider runs locally (no network required) */
+    readonly isLocal: boolean;
+    /**
+     * Generate an embedding for a single text.
+     *
+     * @param text - Input text to embed
+     * @returns Float32Array of embedding values
+     */
+    embed(text: string): Promise<Float32Array>;
+    /**
+     * Generate embeddings for multiple texts in a single batch.
+     *
+     * @param texts - Array of input texts to embed
+     * @returns Array of Float32Array embeddings (same order as input)
+     */
+    embedBatch(texts: string[]): Promise<Float32Array[]>;
+    /**
+     * Check if the provider is available and ready to use.
+     *
+     * For local providers, this checks if the model is installed.
+     * For remote providers, this checks if API keys are configured.
+     *
+     * @returns True if the provider can generate embeddings
+     */
+    isAvailable(): Promise<boolean>;
+}
+/**
+ * Stored embedding record from the database.
+ */
+export interface StoredEmbedding {
+    documentId: string;
+    embedding: Float32Array;
+    dimensions: number;
+    provider: string;
+    model: string;
+    createdAt: string;
+}
+/**
+ * A semantic search result with similarity score.
+ */
+export interface SemanticSearchResult {
+    documentId: string;
+    similarity: number;
+}
+/**
+ * Configuration for the embedding service.
+ */
+export interface EmbeddingServiceConfig {
+    /** The embedding provider to use */
+    provider: EmbeddingProvider;
+    /** Batch size for bulk operations. Default: 32 */
+    batchSize?: number;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/services/embeddings/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/services/embeddings/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,mCAAmC;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,uCAAuC;IACvC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,8DAA8D;IAC9D,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAE1B;;;;;OAKG;IACH,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAE3C;;;;;OAKG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;IAErD;;;;;;;OAOG;IACH,WAAW,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,YAAY,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACrC,oCAAoC;IACpC,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,kDAAkD;IAClD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB"}

package/dist/services/embeddings/types.js

@@ -0,0 +1,8 @@
+/**
+ * Embedding Provider Types
+ *
+ * Defines the interface for embedding providers used in semantic search.
+ * Providers generate vector representations of text for similarity matching.
+ */
+export {};
+//# sourceMappingURL=types.js.map

package/dist/services/embeddings/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/services/embeddings/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}

package/dist/services/id-length-cache.d.ts

@@ -0,0 +1,156 @@
+/**
+ * ID Length Cache Service
+ *
+ * Provides efficient caching for ID hash length calculation.
+ * This service avoids querying element count on every ID generation
+ * by caching the count and calculated length with periodic invalidation.
+ *
+ * The cache is automatically invalidated:
+ * - After a configurable TTL (default: 60 seconds)
+ * - When explicitly refreshed
+ * - After a significant number of creates (growth threshold)
+ */
+import type { StorageBackend } from '@stoneforge/storage';
+/**
+ * Configuration for the ID length cache
+ */
+export interface IdLengthCacheConfig {
+    /**
+     * Time-to-live for cached values in milliseconds.
+     * After this time, the cache will be refreshed on next access.
+     * Default: 60000 (60 seconds)
+     */
+    ttlMs?: number;
+    /**
+     * Minimum number of element creations before triggering a refresh.
+     * This helps catch rapid growth before TTL expires.
+     * Default: 100
+     */
+    growthThreshold?: number;
+    /**
+     * Minimum hash length to use, regardless of element count.
+     * Default: 4 (provides reasonable collision resistance)
+     */
+    minLength?: number;
+}
+/**
+ * Statistics for cache monitoring
+ */
+export interface IdLengthCacheStats {
+    /** Current element count */
+    elementCount: number;
+    /** Current hash length */
+    hashLength: number;
+    /** Number of cache hits since creation */
+    hits: number;
+    /** Number of cache misses (refreshes) since creation */
+    misses: number;
+    /** Cache hit rate (0-1) */
+    hitRate: number;
+    /** Time since last refresh in milliseconds */
+    ageMs: number;
+    /** Whether cache is currently stale */
+    isStale: boolean;
+}
+/**
+ * Service for caching ID hash length calculations.
+ *
+ * This service provides efficient access to the optimal ID hash length
+ * without querying the database on every ID generation.
+ *
+ * @example
+ * ```typescript
+ * const cache = createIdLengthCache(storage);
+ *
+ * // Get optimal hash length for ID generation
+ * const hashLength = cache.getHashLength();
+ *
+ * // Notify cache of new element creation (for growth tracking)
+ * cache.notifyCreate();
+ *
+ * // Force refresh if needed
+ * cache.refresh();
+ * ```
+ */
+export declare class IdLengthCache {
+    private readonly db;
+    private cache;
+    private hits;
+    private misses;
+    private readonly ttlMs;
+    private readonly growthThreshold;
+    private readonly minLength;
+    constructor(db: StorageBackend, config?: IdLengthCacheConfig);
+    /**
+     * Get the optimal hash length for ID generation.
+     *
+     * Returns the cached hash length if valid, otherwise refreshes
+     * the cache first. This is the main method to call when generating IDs.
+     *
+     * @returns The optimal hash length (3-8)
+     */
+    getHashLength(): number;
+    /**
+     * Get the cached element count.
+     *
+     * This is the element count at the last refresh, not necessarily
+     * the current count in the database.
+     *
+     * @returns The cached element count
+     */
+    getElementCount(): number;
+    /**
+     * Notify the cache that a new element was created.
+     *
+     * This increments the internal counter and may trigger a refresh
+     * if the growth threshold is exceeded.
+     */
+    notifyCreate(): void;
+    /**
+     * Force a cache refresh.
+     *
+     * Queries the database for the current element count and
+     * recalculates the optimal hash length.
+     */
+    refresh(): void;
+    /**
+     * Check if the cache is stale and needs refresh.
+     */
+    isStale(): boolean;
+    /**
+     * Clear the cache, forcing a refresh on next access.
+     */
+    clear(): void;
+    /**
+     * Get cache statistics for monitoring.
+     */
+    getStats(): IdLengthCacheStats;
+    /**
+     * Calculate the optimal hash length for an element count.
+     *
+     * Applies the minimum length constraint.
+     */
+    private calculateLength;
+}
+/**
+ * Create a new IdLengthCache instance.
+ *
+ * @param db - Storage backend for querying element count
+ * @param config - Optional cache configuration
+ * @returns A new IdLengthCache instance
+ *
+ * @example
+ * ```typescript
+ * // Basic usage
+ * const cache = createIdLengthCache(storage);
+ *
+ * // With custom configuration
+ * const cache = createIdLengthCache(storage, {
+ *   ttlMs: 30000, // 30 second TTL
+ *   growthThreshold: 50, // Refresh after 50 creates
+ *   minLength: 5, // Minimum 5-character hashes
+ * });
+ * ```
+ */
+export declare function createIdLengthCache(db: StorageBackend, config?: IdLengthCacheConfig): IdLengthCache;
+//# sourceMappingURL=id-length-cache.d.ts.map

package/dist/services/id-length-cache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"id-length-cache.d.ts","sourceRoot":"","sources":["../../src/services/id-length-cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAO1D;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;;;OAIG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAgBD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,4BAA4B;IAC5B,YAAY,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,IAAI,EAAE,MAAM,CAAC;IACb,wDAAwD;IACxD,MAAM,EAAE,MAAM,CAAC;IACf,2BAA2B;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,8CAA8C;IAC9C,KAAK,EAAE,MAAM,CAAC;IACd,uCAAuC;IACvC,OAAO,EAAE,OAAO,CAAC;CAClB;AAcD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,qBAAa,aAAa;IAStB,OAAO,CAAC,QAAQ,CAAC,EAAE;IARrB,OAAO,CAAC,KAAK,CAA2B;IACxC,OAAO,CAAC,IAAI,CAAa;IACzB,OAAO,CAAC,MAAM,CAAa;IAC3B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAGhB,EAAE,EAAE,cAAc,EACnC,MAAM,GAAE,mBAAwB;IAUlC;;;;;;;OAOG;IACH,aAAa,IAAI,MAAM;IAUvB;;;;;;;OAOG;IACH,eAAe,IAAI,MAAM;IAOzB;;;;;OAKG;IACH,YAAY,IAAI,IAAI;IAgBpB;;;;;OAKG;IACH,OAAO,IAAI,IAAI;IAYf;;OAEG;IACH,OAAO,IAAI,OAAO;IASlB;;OAEG;IACH,KAAK,IAAI,IAAI;IAIb;;OAEG;IACH,QAAQ,IAAI,kBAAkB;IAoB9B;;;;OAIG;IACH,OAAO,CAAC,eAAe;CAIxB;AAMD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,mBAAmB,CACjC,EAAE,EAAE,cAAc,EAClB,MAAM,CAAC,EAAE,mBAAmB,GAC3B,aAAa,CAEf"}