@deepagents/retrieval 7.6.0 → 7.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js
CHANGED
|
@@ -265,6 +265,8 @@ function openWithVectors(path) {
|
|
|
265
265
|
// packages/retrieval/src/lib/stores/sqlite/sqlite.ts
|
|
266
266
|
import { Declaration } from "@zukhruf/sqlite-drift";
|
|
267
267
|
import { transaction } from "@zukhruf/sqlite-transaction";
|
|
268
|
+
import { Vector } from "@zukhruf/vectors";
|
|
269
|
+
import { toSql } from "@zukhruf/vectors/sqlite";
|
|
268
270
|
import dedent from "dedent";
|
|
269
271
|
|
|
270
272
|
// packages/retrieval/src/lib/stores/sqlite/ddl.sqlite.ts
|
|
@@ -371,7 +373,7 @@ var SQLiteStore = class {
|
|
|
371
373
|
if (!embeddings.length) {
|
|
372
374
|
return [];
|
|
373
375
|
}
|
|
374
|
-
const vectorBlob =
|
|
376
|
+
const vectorBlob = toSql(Vector.from(embeddings[0]));
|
|
375
377
|
const topN = options.topN;
|
|
376
378
|
if (options.documentId) {
|
|
377
379
|
const rows2 = this.#searchByDocument({
|
|
@@ -432,7 +434,7 @@ var SQLiteStore = class {
|
|
|
432
434
|
inputs.sourceId,
|
|
433
435
|
inputs.documentId,
|
|
434
436
|
chunk.content,
|
|
435
|
-
|
|
437
|
+
toSql(Vector.from(chunk.embedding))
|
|
436
438
|
);
|
|
437
439
|
};
|
|
438
440
|
}
|
|
@@ -517,15 +519,6 @@ var SQLiteStore = class {
|
|
|
517
519
|
flush();
|
|
518
520
|
}
|
|
519
521
|
};
|
|
520
|
-
function vectorToBlob(vector) {
|
|
521
|
-
if (vector instanceof Float32Array) {
|
|
522
|
-
const copied = new Float32Array(vector.length);
|
|
523
|
-
copied.set(vector);
|
|
524
|
-
return Buffer.from(copied.buffer);
|
|
525
|
-
}
|
|
526
|
-
const floatArray = new Float32Array(vector);
|
|
527
|
-
return Buffer.from(floatArray.buffer);
|
|
528
|
-
}
|
|
529
522
|
function toSearchResult(row) {
|
|
530
523
|
if (typeof row !== "object" || row === null || !("content" in row) || typeof row.content !== "string" || !("distance" in row) || typeof row.distance !== "number" || !("document_id" in row) || typeof row.document_id !== "string" || !("metadata" in row) || row.metadata !== null && typeof row.metadata !== "string") {
|
|
531
524
|
throw new Error("Unexpected vector search row");
|
|
@@ -563,7 +556,6 @@ export {
|
|
|
563
556
|
nodeSQLite,
|
|
564
557
|
similaritySearch,
|
|
565
558
|
splitTypeScript,
|
|
566
|
-
splitTypeScriptWithPositions
|
|
567
|
-
vectorToBlob
|
|
559
|
+
splitTypeScriptWithPositions
|
|
568
560
|
};
|
|
569
561
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../src/lib/embedders/huggingface.ts", "../src/lib/embedders/fastembed.ts", "../src/lib/ingest.ts", "../src/lib/stores/cid.ts", "../src/lib/similiarty-search.ts", "../src/lib/stores/sqlite/open.ts", "../src/lib/stores/sqlite/sqlite.ts", "../src/lib/stores/sqlite/ddl.sqlite.ts", "../src/lib/stores/sqlite/node-sqlite.ts"],
|
|
4
|
-
"sourcesContent": ["import type {\n FeatureExtractionPipeline,\n Tensor,\n} from '@huggingface/transformers';\n\nimport type { Embedder } from '../stores/store.ts';\n\nexport type FeatureExtractionFn = FeatureExtractionPipeline;\n\nexport interface EmbeddingOptions {\n extractorFn: () =>\n Promise<FeatureExtractionPipeline> | FeatureExtractionPipeline;\n pooling?: 'mean' | 'cls';\n normalize?: boolean;\n}\n\nasync function textEmbeddings(\n inputs: string | string[],\n { extractorFn, pooling, normalize }: EmbeddingOptions,\n) {\n inputs = (Array.isArray(inputs) ? inputs : [inputs]).map((it) => it.trim());\n const extractor = await extractorFn();\n const tensor = await extractor(inputs, {\n pooling: pooling ?? 'mean',\n normalize: normalize ?? true,\n });\n return tensorToEmbeddings(tensor);\n}\n\nexport function tensorToEmbeddings(tensor: Tensor) {\n const dims = tensor.dims;\n const [batchSize, ...rest] = Array.isArray(dims) ? dims : [];\n const hiddenSize = rest.at(-1);\n if (hiddenSize === undefined) {\n throw new Error(`Unexpected tensor dims: ${JSON.stringify(dims)}`);\n }\n const expectedLen = batchSize * hiddenSize;\n if (tensor.data.length !== expectedLen) {\n throw new Error(\n `Data length mismatch: got ${tensor.data.length}, expected ${expectedLen} (batch=${batchSize}, hidden=${hiddenSize})`,\n );\n }\n const data = tensor.data;\n if (!(data instanceof Float32Array)) {\n throw new Error(`Expected float32 embeddings, got ${tensor.type}`);\n }\n // Reuse the underlying typed array without copying; we'll copy when creating Buffer for SQLite\n const embeddings: Float32Array[] = [];\n const dimensions = hiddenSize;\n for (let i = 0; i < batchSize; i++) {\n const start = i * hiddenSize;\n embeddings.push(data.subarray(start, start + hiddenSize));\n }\n return { embeddings, dimensions };\n}\n\nexport function huggingface(\n extractorFn: () =>\n Promise<FeatureExtractionPipeline> | FeatureExtractionPipeline,\n): Embedder {\n return (documents) => textEmbeddings(documents, { extractorFn });\n}\n", "import type { FeatureExtractionPipeline } from '@huggingface/transformers';\n\nimport type { Embedder } from '../stores/store.ts';\nimport { tensorToEmbeddings } from './huggingface.ts';\n\n/**\n * Stable model identifiers retained from the former `fastembed` adapter.\n */\nexport const EmbeddingModels = {\n AllMiniLML6V2: 'fast-all-MiniLM-L6-v2',\n BGEBaseEN: 'fast-bge-base-en',\n BGEBaseENV15: 'fast-bge-base-en-v1.5',\n BGESmallEN: 'fast-bge-small-en',\n BGESmallENV15: 'fast-bge-small-en-v1.5',\n BGESmallZH: 'fast-bge-small-zh-v1.5',\n MLE5Large: 'fast-multilingual-e5-large',\n} as const;\nexport type EmbeddingModel =\n (typeof EmbeddingModels)[keyof typeof EmbeddingModels];\n\nconst transformerModels: Record<EmbeddingModel, string> = {\n [EmbeddingModels.AllMiniLML6V2]: 'Xenova/all-MiniLM-L6-v2',\n [EmbeddingModels.BGEBaseEN]: 'Xenova/bge-base-en',\n [EmbeddingModels.BGEBaseENV15]: 'Xenova/bge-base-en-v1.5',\n [EmbeddingModels.BGESmallEN]: 'Xenova/bge-small-en',\n [EmbeddingModels.BGESmallENV15]: 'Xenova/bge-small-en-v1.5',\n [EmbeddingModels.BGESmallZH]: 'Xenova/bge-small-zh-v1.5',\n [EmbeddingModels.MLE5Large]: 'Xenova/multilingual-e5-large',\n};\n\nexport interface FastEmbedOptions {\n model?: EmbeddingModel;\n batchSize?: number;\n cacheDir?: string;\n}\n\n/**\n * Local feature extraction backed by Transformers.js. The historical function\n * and model identifiers remain stable while avoiding FastEmbed's archived tar\n * extraction dependency.\n */\nexport function fastembed(options: FastEmbedOptions = {}): Embedder {\n const {\n model: modelId = EmbeddingModels.BGESmallENV15,\n batchSize = 256,\n cacheDir,\n } = options;\n\n if (!Number.isInteger(batchSize) || batchSize <= 0) {\n throw new RangeError('batchSize must be a positive integer');\n }\n\n let modelPromise: Promise<FeatureExtractionPipeline> | null = null;\n const getModel = () => {\n if (!modelPromise) {\n modelPromise = import('@huggingface/transformers').then(({ pipeline }) =>\n pipeline('feature-extraction', transformerModels[modelId], {\n cache_dir: cacheDir,\n }),\n );\n }\n return modelPromise;\n };\n\n return async (documents: string[]) => {\n if (documents.length === 0) {\n return { embeddings: [], dimensions: 0 };\n }\n\n const model = await getModel();\n const embeddings: Float32Array[] = [];\n let dimensions = 0;\n\n for (let start = 0; start < documents.length; start += batchSize) {\n const tensor = await model(documents.slice(start, start + batchSize), {\n pooling: 'mean',\n normalize: true,\n });\n const batch = tensorToEmbeddings(tensor);\n dimensions ||= batch.dimensions;\n embeddings.push(...batch.embeddings);\n }\n\n return { embeddings, dimensions };\n };\n}\n", "import {\n MarkdownTextSplitter,\n RecursiveCharacterTextSplitter,\n} from '@langchain/textsplitters';\n\nimport type { Connector } from './connectors/connector.ts';\nimport type { Splitter } from './pipeline.ts';\nimport { cid } from './stores/cid.ts';\nimport type { Embedder, Store } from './stores/store.ts';\n\nexport interface IngestionConfig {\n connector: Connector;\n store: Store;\n splitter?: Splitter;\n embedder: Embedder;\n}\n\nexport async function ingest(\n config: IngestionConfig,\n callback?: (it: string) => void,\n) {\n const splitter = config.splitter ?? split;\n const embedder = config.embedder;\n const corpuses = config.connector.sources();\n const expiryDate = expiry(config.connector);\n\n for await (const it of corpuses) {\n callback?.(it.id);\n const content = await it.content();\n if (!content.trim()) {\n // skip empty files\n continue;\n }\n await config.store.index(\n config.connector.sourceId,\n {\n id: it.id,\n cid: cid(content),\n metadata: it.metadata,\n chunker: async function* () {\n // Embed in small batches to control memory usage\n const values = await splitter(it.id, content);\n const batchSize = 40;\n for (let i = 0; i < values.length; i += batchSize) {\n const batch = values.slice(i, i + batchSize);\n const { embeddings } = await embedder(batch);\n for (let j = 0; j < embeddings.length; j++) {\n yield {\n content: batch[j],\n embedding: embeddings[j],\n };\n }\n }\n },\n },\n expiryDate,\n );\n }\n}\n\nfunction expiry(connector: Connector) {\n if (connector.ingestWhen !== 'expired') return undefined;\n if (connector.expiresAfter === undefined) return undefined;\n return new Date(connector.clock.Date.now() + connector.expiresAfter);\n}\n\nfunction split(id: string, content: string) {\n const splitter = new MarkdownTextSplitter();\n return splitter.splitText(content);\n}\n\nexport type ChunkPosition = {\n startLine: number;\n startColumn: number;\n endLine: number;\n endColumn: number;\n};\n\nexport type SplitChunkWithPosition = {\n content: string;\n position: ChunkPosition | null;\n index: number;\n};\n\nfunction normalizeNewlines(value: string) {\n return value.replace(/\\r\\n/g, '\\n');\n}\n\nfunction computePositions(\n originalContent: string,\n chunks: string[],\n): Array<ChunkPosition | null> {\n if (!chunks.length) {\n return [];\n }\n\n const normalizedContent = normalizeNewlines(originalContent);\n const positions: Array<ChunkPosition | null> = [];\n let searchOffset = 0;\n\n for (const chunk of chunks) {\n const normalizedChunk = normalizeNewlines(chunk);\n const trimmedChunk = normalizedChunk.trim();\n\n const seek = (needle: string, fromIndex: number) =>\n needle ? normalizedContent.indexOf(needle, fromIndex) : -1;\n\n let matchIndex = seek(normalizedChunk, searchOffset);\n let matchValue = normalizedChunk;\n\n if (matchIndex === -1 && trimmedChunk) {\n matchIndex = seek(trimmedChunk, searchOffset);\n matchValue = trimmedChunk;\n }\n\n if (matchIndex === -1) {\n matchIndex = seek(normalizedChunk, 0);\n matchValue = normalizedChunk;\n }\n\n if (matchIndex === -1 && trimmedChunk) {\n matchIndex = seek(trimmedChunk, 0);\n matchValue = trimmedChunk;\n }\n\n if (matchIndex === -1) {\n positions.push(null);\n continue;\n }\n\n const before = normalizedContent.slice(0, matchIndex);\n const beforeLines = before.split('\\n');\n const startLine = beforeLines.length;\n const startColumn = before.length - before.lastIndexOf('\\n');\n\n const lines = matchValue.split('\\n');\n const endLine = startLine + lines.length - 1;\n const endColumn =\n lines.length === 1\n ? startColumn + lines[0].length\n : matchValue.length - matchValue.lastIndexOf('\\n');\n\n positions.push({ startLine, startColumn, endLine, endColumn });\n searchOffset = matchIndex + matchValue.length;\n }\n\n return positions;\n}\n\nfunction buildChunksWithPositions(\n originalContent: string,\n chunks: string[],\n): SplitChunkWithPosition[] {\n const positions = computePositions(originalContent, chunks);\n return chunks.map((content, index) => ({\n content,\n index,\n position: positions[index] ?? null,\n }));\n}\n\nexport async function splitTypeScriptWithPositions(\n id: string,\n content: string,\n): Promise<SplitChunkWithPosition[]> {\n const splitter = RecursiveCharacterTextSplitter.fromLanguage('js', {\n chunkSize: 512,\n chunkOverlap: 100,\n });\n const docs = await splitter.createDocuments([content]);\n const chunks = docs.map((d) => d.pageContent);\n return buildChunksWithPositions(content, chunks);\n}\n\nexport async function splitTypeScript(id: string, content: string) {\n const chunks = await splitTypeScriptWithPositions(id, content);\n return chunks.map((chunk) => chunk.content);\n}\n", "import { hash } from 'node:crypto';\n\nexport function cid(content: string) {\n return hash('sha256', content);\n}\n", "import { embedMany } from 'ai';\n\nimport { type IngestionConfig, ingest } from './ingest.ts';\n\nexport async function similaritySearch(\n query: string,\n config: Omit<IngestionConfig, 'splitter'>,\n) {\n // Ingest if needed, then perform vector similarity search via the configured store\n const { connector } = config;\n let shouldIngest = true;\n\n if (connector.ingestWhen === 'never') {\n if (await config.store.sourceExists(config.connector.sourceId)) {\n console.log(\n `Skipping ingestion for source ${config.connector.sourceId} (ingestWhen=never and source exists)`,\n );\n shouldIngest = false;\n }\n } else if (connector.ingestWhen === 'expired') {\n const sourceExists = await config.store.sourceExists(\n config.connector.sourceId,\n );\n if (\n sourceExists &&\n !(await config.store.sourceExpired(\n config.connector.sourceId,\n new Date(connector.clock.Date.now()),\n ))\n ) {\n console.log(\n `Skipping ingestion for source ${config.connector.sourceId} (ingestWhen=expired and source not expired)`,\n );\n shouldIngest = false;\n }\n }\n\n if (shouldIngest) {\n await ingest(config);\n }\n return config.store\n .search(\n query,\n { sourceId: config.connector.sourceId, topN: 50 },\n config.embedder,\n )\n .then((results) =>\n results.map((it) => ({\n ...it,\n similarity: 1 - it.distance,\n distance: it.distance,\n })),\n );\n}\n", "import { WriteAheadLog, open } from '@zukhruf/sqlite-open';\nimport * as sqliteVec from 'sqlite-vec';\n\n// The store's statements use the vec0 module, so every connection that runs\n// them, the dev database of the declaration included, loads sqlite-vec. The\n// rows of vectors are large, so a new file gets 32 KiB pages; the write-ahead\n// log fixes the page size, so the journal sets it before it.\nexport function openWithVectors(path: string) {\n const db = open(path, {\n allowExtension: true,\n journal: new WriteAheadLog({ pageSize: 32768 }),\n });\n db.loadExtension(sqliteVec.getLoadablePath());\n return db;\n}\n", "import { Declaration } from '@zukhruf/sqlite-drift';\nimport { transaction } from '@zukhruf/sqlite-transaction';\nimport dedent from 'dedent';\nimport type { DatabaseSync } from 'node:sqlite';\n\nimport type {\n Chunk,\n Corpus,\n Embedder,\n SearchOptions,\n SearchResult,\n Store,\n} from '../store.ts';\nimport { SETUP, statements, vectors } from './ddl.sqlite.ts';\nimport { openWithVectors } from './open.ts';\n\nconst DEFAULT_TOP_N = 10;\n\nconst tables = new Declaration(statements, { rows: 'keep' });\n\nexport class SQLiteStore implements Store {\n readonly #db: DatabaseSync;\n constructor(db: DatabaseSync, dimension: number) {\n this.#db = db;\n this.#db.exec(SETUP);\n const chunks = new Declaration([vectors(dimension)], {\n rows: 'replace',\n devDatabase: () => openWithVectors(':memory:'),\n });\n transaction(\n this.#db,\n () => {\n const stale = chunks.drift(this.#db) === 'stale';\n chunks.apply(this.#db);\n // A replaced vec_chunks is empty, yet upsertDoc skips a document whose\n // cid is unchanged and similaritySearch skips a source that exists.\n // Deleting the sources cascades to their documents, so the next\n // ingest embeds every document again.\n if (stale) this.#db.exec('DELETE FROM sources');\n },\n { mode: 'immediate' },\n );\n tables.apply(this.#db);\n }\n\n #searchByDocument(inputs: {\n sourceId: string;\n documentId: string;\n embedding: Buffer<ArrayBufferLike>;\n k: number;\n }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT v.content, v.distance, v.document_id, d.metadata\n\t\t\t\tFROM vec_chunks v\n\t\t\t\tJOIN documents d ON d.id = v.document_id\n\t\t\t\tWHERE v.source_id = ?\n\t\t\t\t\tAND v.document_id = ?\n\t\t\t\t\tAND v.embedding MATCH vec_normalize(vec_f32(?))\n\t\t\t\t\tAND v.k = ?\n\t\t\t\tORDER BY v.distance ASC\n\t\t\t`);\n return stmt.all(\n inputs.sourceId,\n inputs.documentId,\n inputs.embedding,\n inputs.k,\n );\n }\n #searchBySource(inputs: {\n sourceId: string;\n embedding: Buffer<ArrayBufferLike>;\n k: number;\n }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT v.content, v.distance, v.document_id, d.metadata\n\t\t\t\tFROM vec_chunks v\n\t\t\t\tJOIN documents d ON d.id = v.document_id\n\t\t\t\tWHERE v.source_id = ?\n\t\t\t\t\tAND v.embedding MATCH vec_normalize(vec_f32(?))\n\t\t\t\t\tAND v.k = ?\n\t\t\t\tORDER BY v.distance ASC\n\t\t\t`);\n return stmt.all(inputs.sourceId, inputs.embedding, inputs.k);\n }\n async search(query: string, options: SearchOptions, embedder: Embedder) {\n const { embeddings } = await embedder([query]);\n if (!embeddings.length) {\n return [];\n }\n const vectorBlob = vectorToBlob(embeddings[0]);\n const topN = options.topN;\n\n if (options.documentId) {\n const rows = this.#searchByDocument({\n sourceId: options.sourceId,\n documentId: options.documentId,\n embedding: vectorBlob,\n k: topN ?? DEFAULT_TOP_N,\n });\n return rows.map(toSearchResult);\n }\n\n const rows = this.#searchBySource({\n sourceId: options.sourceId,\n embedding: vectorBlob,\n k: topN ?? DEFAULT_TOP_N,\n });\n return rows.map(toSearchResult);\n }\n #upsertSource(inputs: { sourceId: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tINSERT INTO sources (source_id) VALUES (?)\n\t\t\t\tON CONFLICT(source_id) DO UPDATE SET updated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now')\n\t\t\t`);\n return stmt.run(inputs.sourceId);\n }\n #upsertSourceWithExpiry(inputs: { sourceId: string; expiresAt: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tINSERT INTO sources (source_id, expires_at) VALUES (?, ?)\n\t\t\t\tON CONFLICT(source_id) DO UPDATE SET\n\t\t\t\t\tupdated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now'),\n\t\t\t\t\texpires_at=excluded.expires_at\n\t\t\t`);\n return stmt.run(inputs.sourceId, inputs.expiresAt);\n }\n upsertDoc(inputs: {\n documentId: string;\n sourceId: string;\n cid: string;\n metadata?: Record<string, any>;\n }) {\n const stmt = this.#db.prepare(dedent`\n INSERT INTO documents (id, source_id, cid, metadata)\n VALUES (?, ?, ?, json(?))\n ON CONFLICT(id) DO UPDATE SET\n cid=excluded.cid,\n metadata=COALESCE(excluded.metadata, documents.metadata),\n updated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now')\n WHERE documents.cid != excluded.cid;\n `);\n return stmt.run(\n inputs.documentId,\n inputs.sourceId,\n inputs.cid,\n inputs.metadata ? JSON.stringify(inputs.metadata) : null,\n );\n }\n insertDoc(inputs: { sourceId: string; documentId: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tINSERT INTO vec_chunks (source_id, document_id, content, embedding)\n\t\t\t\tVALUES (?, ?, ?, vec_normalize(vec_f32(?)))\n\t\t\t`);\n return (chunk: Chunk) => {\n stmt.run(\n inputs.sourceId,\n inputs.documentId,\n chunk.content,\n vectorToBlob(chunk.embedding),\n );\n };\n }\n delete(inputs: { sourceId: string; documentId: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tDELETE FROM vec_chunks WHERE source_id = ? AND document_id = ?\n\t\t\t`);\n return stmt.run(inputs.sourceId, inputs.documentId);\n }\n sourceExists(sourceId: string) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT 1 FROM sources WHERE source_id = ? LIMIT 1\n\t\t\t`);\n const row = stmt.get(sourceId);\n return Boolean(row);\n }\n sourceExpired(sourceId: string, now: Date) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT 1 FROM sources\n\t\t\t\tWHERE source_id = ?\n\t\t\t\t\tAND expires_at IS NOT NULL\n\t\t\t\t\tAND expires_at <= ?\n\t\t\t\tLIMIT 1\n\t\t\t`);\n const row = stmt.get(sourceId, now.toISOString());\n return Boolean(row);\n }\n setSourceExpiry(sourceId: string, expiryDate: Date) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tUPDATE sources\n\t\t\t\tSET expires_at = ?, updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')\n\t\t\t\tWHERE source_id = ?\n\t\t\t`);\n stmt.run(expiryDate.toISOString(), sourceId);\n }\n async index(sourceId: string, corpus: Corpus, expiryDate?: Date) {\n if (expiryDate) {\n this.#upsertSourceWithExpiry({\n expiresAt: expiryDate.toISOString(),\n sourceId,\n });\n } else {\n this.#upsertSource({ sourceId });\n }\n\n const info = this.upsertDoc({\n documentId: corpus.id,\n sourceId,\n cid: corpus.cid,\n metadata: corpus.metadata,\n });\n const changed = info.changes > 0;\n\n if (!changed) {\n return;\n }\n\n const insert = this.insertDoc({ sourceId, documentId: corpus.id });\n // Delete previous rows once before inserting\n transaction(\n this.#db,\n () => {\n this.delete({ sourceId, documentId: corpus.id });\n },\n { mode: 'immediate' },\n );\n\n const batchSize = 32;\n let batch: Chunk[] = [];\n const flush = () => {\n if (!batch.length) return;\n transaction(\n this.#db,\n () => {\n for (let i = 0; i < batch.length; i++) {\n insert(batch[i]);\n }\n },\n { mode: 'immediate' },\n );\n batch = [];\n };\n\n for await (const chunk of corpus.chunker()) {\n batch.push(chunk);\n if (batch.length >= batchSize) flush();\n }\n // flush any remaining\n flush();\n }\n}\n\nexport function vectorToBlob(vector: number[] | Float32Array): Buffer {\n if (vector instanceof Float32Array) {\n // Copy into a fresh Buffer to avoid retaining references to a larger batch tensor buffer\n const copied = new Float32Array(vector.length);\n copied.set(vector);\n return Buffer.from(copied.buffer);\n }\n const floatArray = new Float32Array(vector);\n return Buffer.from(floatArray.buffer);\n}\n\nfunction toSearchResult(row: unknown): SearchResult {\n if (\n typeof row !== 'object' ||\n row === null ||\n !('content' in row) ||\n typeof row.content !== 'string' ||\n !('distance' in row) ||\n typeof row.distance !== 'number' ||\n !('document_id' in row) ||\n typeof row.document_id !== 'string' ||\n !('metadata' in row) ||\n (row.metadata !== null && typeof row.metadata !== 'string')\n ) {\n throw new Error('Unexpected vector search row');\n }\n return {\n content: row.content,\n distance: row.distance,\n document_id: row.document_id,\n metadata: parseMetadata(row.metadata),\n };\n}\n\nfunction parseMetadata(value: string | null): Record<string, unknown> | null {\n if (value === null) return null;\n try {\n const parsed: unknown = JSON.parse(value);\n return isRecord(parsed) ? parsed : null;\n } catch {\n return null;\n }\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n return typeof value === 'object' && value !== null && !Array.isArray(value);\n}\n", "// Embedding store schema for SQLite. SQLite stores the text of each CREATE and\n// @zukhruf/sqlite-drift compares it, so the text after each object name must\n// stay byte-identical to what earlier releases wrote, inline comments included.\n\n// Session-level PRAGMAs, run on each connection before the schema is applied.\nexport const SETUP = `\nPRAGMA synchronous = NORMAL;\nPRAGMA temp_store = MEMORY;\nPRAGMA cache_size = -131072; -- ~128 MiB page cache for faster repeated reads\nPRAGMA mmap_size = 268435456; -- 256 MiB memory map window to cut syscalls\nPRAGMA wal_autocheckpoint = 1000;\nPRAGMA optimize;\n`;\n\nexport function vectors(dimension: number) {\n return `CREATE VIRTUAL TABLE vec_chunks USING vec0(\n source_id TEXT PARTITION KEY,\n document_id TEXT,\n embedding FLOAT[${dimension}] DISTANCE_METRIC=cosine,\n +content TEXT -- auxiliary payload (not filterable) for reconstruction\n)`;\n}\n\nexport const statements = [\n `CREATE TABLE sources (\n source_id TEXT PRIMARY KEY,\n updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n expires_at TEXT\n) STRICT`,\n `CREATE TABLE documents (\n id TEXT PRIMARY KEY,\n source_id TEXT NOT NULL,\n cid TEXT NOT NULL,\n metadata TEXT, -- JSON blob with arbitrary document metadata\n updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n FOREIGN KEY (source_id) REFERENCES sources(source_id) ON DELETE CASCADE\n) STRICT`,\n // Indexes to accelerate lookups / maintenance\n `CREATE INDEX idx_documents_source ON documents(source_id)`,\n `CREATE INDEX idx_documents_source_updated ON documents(source_id, updated_at)`,\n `CREATE INDEX idx_documents_cid ON documents(cid)`,\n `CREATE INDEX idx_sources_expires_at ON sources(expires_at) WHERE expires_at IS NOT NULL`,\n // emulate cascade for the virtual table\n `CREATE TRIGGER trg_documents_delete_vec\nAFTER DELETE ON documents\nBEGIN\n DELETE FROM vec_chunks\n WHERE source_id = OLD.source_id\n AND document_id = OLD.id;\nEND`,\n];\n", "import { openWithVectors } from './open.ts';\nimport { SQLiteStore } from './sqlite.ts';\n\nexport function nodeSQLite(dbName: string, dimension: number) {\n return new SQLiteStore(openWithVectors(dbName), dimension);\n}\n"],
|
|
5
|
-
"mappings": ";AA6BO,SAAS,mBAAmB,QAAgB;AACjD,QAAM,OAAO,OAAO;AACpB,QAAM,CAAC,WAAW,GAAG,IAAI,IAAI,MAAM,QAAQ,IAAI,IAAI,OAAO,CAAC;AAC3D,QAAM,aAAa,KAAK,GAAG,EAAE;AAC7B,MAAI,eAAe,QAAW;AAC5B,UAAM,IAAI,MAAM,2BAA2B,KAAK,UAAU,IAAI,CAAC,EAAE;AAAA,EACnE;AACA,QAAM,cAAc,YAAY;AAChC,MAAI,OAAO,KAAK,WAAW,aAAa;AACtC,UAAM,IAAI;AAAA,MACR,6BAA6B,OAAO,KAAK,MAAM,cAAc,WAAW,WAAW,SAAS,YAAY,UAAU;AAAA,IACpH;AAAA,EACF;AACA,QAAM,OAAO,OAAO;AACpB,MAAI,EAAE,gBAAgB,eAAe;AACnC,UAAM,IAAI,MAAM,oCAAoC,OAAO,IAAI,EAAE;AAAA,EACnE;AAEA,QAAM,aAA6B,CAAC;AACpC,QAAM,aAAa;AACnB,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,IAAI;AAClB,eAAW,KAAK,KAAK,SAAS,OAAO,QAAQ,UAAU,CAAC;AAAA,EAC1D;AACA,SAAO,EAAE,YAAY,WAAW;AAClC;;;AC9CO,IAAM,kBAAkB;AAAA,EAC7B,eAAe;AAAA,EACf,WAAW;AAAA,EACX,cAAc;AAAA,EACd,YAAY;AAAA,EACZ,eAAe;AAAA,EACf,YAAY;AAAA,EACZ,WAAW;AACb;AAIA,IAAM,oBAAoD;AAAA,EACxD,CAAC,gBAAgB,aAAa,GAAG;AAAA,EACjC,CAAC,gBAAgB,SAAS,GAAG;AAAA,EAC7B,CAAC,gBAAgB,YAAY,GAAG;AAAA,EAChC,CAAC,gBAAgB,UAAU,GAAG;AAAA,EAC9B,CAAC,gBAAgB,aAAa,GAAG;AAAA,EACjC,CAAC,gBAAgB,UAAU,GAAG;AAAA,EAC9B,CAAC,gBAAgB,SAAS,GAAG;AAC/B;AAaO,SAAS,UAAU,UAA4B,CAAC,GAAa;AAClE,QAAM;AAAA,IACJ,OAAO,UAAU,gBAAgB;AAAA,IACjC,YAAY;AAAA,IACZ;AAAA,EACF,IAAI;AAEJ,MAAI,CAAC,OAAO,UAAU,SAAS,KAAK,aAAa,GAAG;AAClD,UAAM,IAAI,WAAW,sCAAsC;AAAA,EAC7D;AAEA,MAAI,eAA0D;AAC9D,QAAM,WAAW,MAAM;AACrB,QAAI,CAAC,cAAc;AACjB,qBAAe,OAAO,2BAA2B,EAAE;AAAA,QAAK,CAAC,EAAE,SAAS,MAClE,SAAS,sBAAsB,kBAAkB,OAAO,GAAG;AAAA,UACzD,WAAW;AAAA,QACb,CAAC;AAAA,MACH;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAEA,SAAO,OAAO,cAAwB;AACpC,QAAI,UAAU,WAAW,GAAG;AAC1B,aAAO,EAAE,YAAY,CAAC,GAAG,YAAY,EAAE;AAAA,IACzC;AAEA,UAAM,QAAQ,MAAM,SAAS;AAC7B,UAAM,aAA6B,CAAC;AACpC,QAAI,aAAa;AAEjB,aAAS,QAAQ,GAAG,QAAQ,UAAU,QAAQ,SAAS,WAAW;AAChE,YAAM,SAAS,MAAM,MAAM,UAAU,MAAM,OAAO,QAAQ,SAAS,GAAG;AAAA,QACpE,SAAS;AAAA,QACT,WAAW;AAAA,MACb,CAAC;AACD,YAAM,QAAQ,mBAAmB,MAAM;AACvC,qBAAe,MAAM;AACrB,iBAAW,KAAK,GAAG,MAAM,UAAU;AAAA,IACrC;AAEA,WAAO,EAAE,YAAY,WAAW;AAAA,EAClC;AACF;;;ACrFA;AAAA,EACE;AAAA,EACA;AAAA,OACK;;;ACHP,SAAS,YAAY;AAEd,SAAS,IAAI,SAAiB;AACnC,SAAO,KAAK,UAAU,OAAO;AAC/B;;;ADaA,eAAsB,OACpB,QACA,UACA;AACA,QAAM,WAAW,OAAO,YAAY;AACpC,QAAM,WAAW,OAAO;AACxB,QAAM,WAAW,OAAO,UAAU,QAAQ;AAC1C,QAAM,aAAa,OAAO,OAAO,SAAS;AAE1C,mBAAiB,MAAM,UAAU;AAC/B,eAAW,GAAG,EAAE;AAChB,UAAM,UAAU,MAAM,GAAG,QAAQ;AACjC,QAAI,CAAC,QAAQ,KAAK,GAAG;AAEnB;AAAA,IACF;AACA,UAAM,OAAO,MAAM;AAAA,MACjB,OAAO,UAAU;AAAA,MACjB;AAAA,QACE,IAAI,GAAG;AAAA,QACP,KAAK,IAAI,OAAO;AAAA,QAChB,UAAU,GAAG;AAAA,QACb,SAAS,mBAAmB;AAE1B,gBAAM,SAAS,MAAM,SAAS,GAAG,IAAI,OAAO;AAC5C,gBAAM,YAAY;AAClB,mBAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,WAAW;AACjD,kBAAM,QAAQ,OAAO,MAAM,GAAG,IAAI,SAAS;AAC3C,kBAAM,EAAE,WAAW,IAAI,MAAM,SAAS,KAAK;AAC3C,qBAAS,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AAC1C,oBAAM;AAAA,gBACJ,SAAS,MAAM,CAAC;AAAA,gBAChB,WAAW,WAAW,CAAC;AAAA,cACzB;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,OAAO,WAAsB;AACpC,MAAI,UAAU,eAAe,UAAW,QAAO;AAC/C,MAAI,UAAU,iBAAiB,OAAW,QAAO;AACjD,SAAO,IAAI,KAAK,UAAU,MAAM,KAAK,IAAI,IAAI,UAAU,YAAY;AACrE;AAEA,SAAS,MAAM,IAAY,SAAiB;AAC1C,QAAM,WAAW,IAAI,qBAAqB;AAC1C,SAAO,SAAS,UAAU,OAAO;AACnC;AAeA,SAAS,kBAAkB,OAAe;AACxC,SAAO,MAAM,QAAQ,SAAS,IAAI;AACpC;AAEA,SAAS,iBACP,iBACA,QAC6B;AAC7B,MAAI,CAAC,OAAO,QAAQ;AAClB,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,oBAAoB,kBAAkB,eAAe;AAC3D,QAAM,YAAyC,CAAC;AAChD,MAAI,eAAe;AAEnB,aAAW,SAAS,QAAQ;AAC1B,UAAM,kBAAkB,kBAAkB,KAAK;AAC/C,UAAM,eAAe,gBAAgB,KAAK;AAE1C,UAAM,OAAO,CAAC,QAAgB,cAC5B,SAAS,kBAAkB,QAAQ,QAAQ,SAAS,IAAI;AAE1D,QAAI,aAAa,KAAK,iBAAiB,YAAY;AACnD,QAAI,aAAa;AAEjB,QAAI,eAAe,MAAM,cAAc;AACrC,mBAAa,KAAK,cAAc,YAAY;AAC5C,mBAAa;AAAA,IACf;AAEA,QAAI,eAAe,IAAI;AACrB,mBAAa,KAAK,iBAAiB,CAAC;AACpC,mBAAa;AAAA,IACf;AAEA,QAAI,eAAe,MAAM,cAAc;AACrC,mBAAa,KAAK,cAAc,CAAC;AACjC,mBAAa;AAAA,IACf;AAEA,QAAI,eAAe,IAAI;AACrB,gBAAU,KAAK,IAAI;AACnB;AAAA,IACF;AAEA,UAAM,SAAS,kBAAkB,MAAM,GAAG,UAAU;AACpD,UAAM,cAAc,OAAO,MAAM,IAAI;AACrC,UAAM,YAAY,YAAY;AAC9B,UAAM,cAAc,OAAO,SAAS,OAAO,YAAY,IAAI;AAE3D,UAAM,QAAQ,WAAW,MAAM,IAAI;AACnC,UAAM,UAAU,YAAY,MAAM,SAAS;AAC3C,UAAM,YACJ,MAAM,WAAW,IACb,cAAc,MAAM,CAAC,EAAE,SACvB,WAAW,SAAS,WAAW,YAAY,IAAI;AAErD,cAAU,KAAK,EAAE,WAAW,aAAa,SAAS,UAAU,CAAC;AAC7D,mBAAe,aAAa,WAAW;AAAA,EACzC;AAEA,SAAO;AACT;AAEA,SAAS,yBACP,iBACA,QAC0B;AAC1B,QAAM,YAAY,iBAAiB,iBAAiB,MAAM;AAC1D,SAAO,OAAO,IAAI,CAAC,SAAS,WAAW;AAAA,IACrC;AAAA,IACA;AAAA,IACA,UAAU,UAAU,KAAK,KAAK;AAAA,EAChC,EAAE;AACJ;AAEA,eAAsB,6BACpB,IACA,SACmC;AACnC,QAAM,WAAW,+BAA+B,aAAa,MAAM;AAAA,IACjE,WAAW;AAAA,IACX,cAAc;AAAA,EAChB,CAAC;AACD,QAAM,OAAO,MAAM,SAAS,gBAAgB,CAAC,OAAO,CAAC;AACrD,QAAM,SAAS,KAAK,IAAI,CAAC,MAAM,EAAE,WAAW;AAC5C,SAAO,yBAAyB,SAAS,MAAM;AACjD;AAEA,eAAsB,gBAAgB,IAAY,SAAiB;AACjE,QAAM,SAAS,MAAM,6BAA6B,IAAI,OAAO;AAC7D,SAAO,OAAO,IAAI,CAAC,UAAU,MAAM,OAAO;AAC5C;;;AEjLA,OAA0B;AAI1B,eAAsB,iBACpB,OACA,QACA;AAEA,QAAM,EAAE,UAAU,IAAI;AACtB,MAAI,eAAe;AAEnB,MAAI,UAAU,eAAe,SAAS;AACpC,QAAI,MAAM,OAAO,MAAM,aAAa,OAAO,UAAU,QAAQ,GAAG;AAC9D,cAAQ;AAAA,QACN,iCAAiC,OAAO,UAAU,QAAQ;AAAA,MAC5D;AACA,qBAAe;AAAA,IACjB;AAAA,EACF,WAAW,UAAU,eAAe,WAAW;AAC7C,UAAM,eAAe,MAAM,OAAO,MAAM;AAAA,MACtC,OAAO,UAAU;AAAA,IACnB;AACA,QACE,gBACA,CAAE,MAAM,OAAO,MAAM;AAAA,MACnB,OAAO,UAAU;AAAA,MACjB,IAAI,KAAK,UAAU,MAAM,KAAK,IAAI,CAAC;AAAA,IACrC,GACA;AACA,cAAQ;AAAA,QACN,iCAAiC,OAAO,UAAU,QAAQ;AAAA,MAC5D;AACA,qBAAe;AAAA,IACjB;AAAA,EACF;AAEA,MAAI,cAAc;AAChB,UAAM,OAAO,MAAM;AAAA,EACrB;AACA,SAAO,OAAO,MACX;AAAA,IACC;AAAA,IACA,EAAE,UAAU,OAAO,UAAU,UAAU,MAAM,GAAG;AAAA,IAChD,OAAO;AAAA,EACT,EACC;AAAA,IAAK,CAAC,YACL,QAAQ,IAAI,CAAC,QAAQ;AAAA,MACnB,GAAG;AAAA,MACH,YAAY,IAAI,GAAG;AAAA,MACnB,UAAU,GAAG;AAAA,IACf,EAAE;AAAA,EACJ;AACJ;;;ACrDA,SAAS,eAAe,YAAY;AACpC,YAAY,eAAe;AAMpB,SAAS,gBAAgB,MAAc;AAC5C,QAAM,KAAK,KAAK,MAAM;AAAA,IACpB,gBAAgB;AAAA,IAChB,SAAS,IAAI,cAAc,EAAE,UAAU,MAAM,CAAC;AAAA,EAChD,CAAC;AACD,KAAG,cAAwB,0BAAgB,CAAC;AAC5C,SAAO;AACT;;;ACdA,SAAS,mBAAmB;AAC5B,SAAS,mBAAmB;AAC5B,OAAO,YAAY;;;ACGZ,IAAM,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AASd,SAAS,QAAQ,WAAmB;AACzC,SAAO;AAAA;AAAA;AAAA,oBAGW,SAAS;AAAA;AAAA;AAG7B;AAEO,IAAM,aAAa;AAAA,EACxB;AAAA;AAAA;AAAA;AAAA;AAAA,EAKA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAOF;;;ADlCA,IAAM,gBAAgB;AAEtB,IAAM,SAAS,IAAI,YAAY,YAAY,EAAE,MAAM,OAAO,CAAC;AAEpD,IAAM,cAAN,MAAmC;AAAA,EAC/B;AAAA,EACT,YAAY,IAAkB,WAAmB;AAC/C,SAAK,MAAM;AACX,SAAK,IAAI,KAAK,KAAK;AACnB,UAAM,SAAS,IAAI,YAAY,CAAC,QAAQ,SAAS,CAAC,GAAG;AAAA,MACnD,MAAM;AAAA,MACN,aAAa,MAAM,gBAAgB,UAAU;AAAA,IAC/C,CAAC;AACD;AAAA,MACE,KAAK;AAAA,MACL,MAAM;AACJ,cAAM,QAAQ,OAAO,MAAM,KAAK,GAAG,MAAM;AACzC,eAAO,MAAM,KAAK,GAAG;AAKrB,YAAI,MAAO,MAAK,IAAI,KAAK,qBAAqB;AAAA,MAChD;AAAA,MACA,EAAE,MAAM,YAAY;AAAA,IACtB;AACA,WAAO,MAAM,KAAK,GAAG;AAAA,EACvB;AAAA,EAEA,kBAAkB,QAKf;AACD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAS9B;AACA,WAAO,KAAK;AAAA,MACV,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO;AAAA,IACT;AAAA,EACF;AAAA,EACA,gBAAgB,QAIb;AACD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAQ9B;AACA,WAAO,KAAK,IAAI,OAAO,UAAU,OAAO,WAAW,OAAO,CAAC;AAAA,EAC7D;AAAA,EACA,MAAM,OAAO,OAAe,SAAwB,UAAoB;AACtE,UAAM,EAAE,WAAW,IAAI,MAAM,SAAS,CAAC,KAAK,CAAC;AAC7C,QAAI,CAAC,WAAW,QAAQ;AACtB,aAAO,CAAC;AAAA,IACV;AACA,UAAM,aAAa,aAAa,WAAW,CAAC,CAAC;AAC7C,UAAM,OAAO,QAAQ;AAErB,QAAI,QAAQ,YAAY;AACtB,YAAMA,QAAO,KAAK,kBAAkB;AAAA,QAClC,UAAU,QAAQ;AAAA,QAClB,YAAY,QAAQ;AAAA,QACpB,WAAW;AAAA,QACX,GAAG,QAAQ;AAAA,MACb,CAAC;AACD,aAAOA,MAAK,IAAI,cAAc;AAAA,IAChC;AAEA,UAAM,OAAO,KAAK,gBAAgB;AAAA,MAChC,UAAU,QAAQ;AAAA,MAClB,WAAW;AAAA,MACX,GAAG,QAAQ;AAAA,IACb,CAAC;AACD,WAAO,KAAK,IAAI,cAAc;AAAA,EAChC;AAAA,EACA,cAAc,QAA8B;AAC1C,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA,IAG9B;AACA,WAAO,KAAK,IAAI,OAAO,QAAQ;AAAA,EACjC;AAAA,EACA,wBAAwB,QAAiD;AACvE,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,IAK9B;AACA,WAAO,KAAK,IAAI,OAAO,UAAU,OAAO,SAAS;AAAA,EACnD;AAAA,EACA,UAAU,QAKP;AACD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,OAQ3B;AACH,WAAO,KAAK;AAAA,MACV,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO,WAAW,KAAK,UAAU,OAAO,QAAQ,IAAI;AAAA,IACtD;AAAA,EACF;AAAA,EACA,UAAU,QAAkD;AAC1D,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA,IAG9B;AACA,WAAO,CAAC,UAAiB;AACvB,WAAK;AAAA,QACH,OAAO;AAAA,QACP,OAAO;AAAA,QACP,MAAM;AAAA,QACN,aAAa,MAAM,SAAS;AAAA,MAC9B;AAAA,IACF;AAAA,EACF;AAAA,EACA,OAAO,QAAkD;AACvD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA,IAE9B;AACA,WAAO,KAAK,IAAI,OAAO,UAAU,OAAO,UAAU;AAAA,EACpD;AAAA,EACA,aAAa,UAAkB;AAC7B,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA,IAE9B;AACA,UAAM,MAAM,KAAK,IAAI,QAAQ;AAC7B,WAAO,QAAQ,GAAG;AAAA,EACpB;AAAA,EACA,cAAc,UAAkB,KAAW;AACzC,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAM9B;AACA,UAAM,MAAM,KAAK,IAAI,UAAU,IAAI,YAAY,CAAC;AAChD,WAAO,QAAQ,GAAG;AAAA,EACpB;AAAA,EACA,gBAAgB,UAAkB,YAAkB;AAClD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA,IAI9B;AACA,SAAK,IAAI,WAAW,YAAY,GAAG,QAAQ;AAAA,EAC7C;AAAA,EACA,MAAM,MAAM,UAAkB,QAAgB,YAAmB;AAC/D,QAAI,YAAY;AACd,WAAK,wBAAwB;AAAA,QAC3B,WAAW,WAAW,YAAY;AAAA,QAClC;AAAA,MACF,CAAC;AAAA,IACH,OAAO;AACL,WAAK,cAAc,EAAE,SAAS,CAAC;AAAA,IACjC;AAEA,UAAM,OAAO,KAAK,UAAU;AAAA,MAC1B,YAAY,OAAO;AAAA,MACnB;AAAA,MACA,KAAK,OAAO;AAAA,MACZ,UAAU,OAAO;AAAA,IACnB,CAAC;AACD,UAAM,UAAU,KAAK,UAAU;AAE/B,QAAI,CAAC,SAAS;AACZ;AAAA,IACF;AAEA,UAAM,SAAS,KAAK,UAAU,EAAE,UAAU,YAAY,OAAO,GAAG,CAAC;AAEjE;AAAA,MACE,KAAK;AAAA,MACL,MAAM;AACJ,aAAK,OAAO,EAAE,UAAU,YAAY,OAAO,GAAG,CAAC;AAAA,MACjD;AAAA,MACA,EAAE,MAAM,YAAY;AAAA,IACtB;AAEA,UAAM,YAAY;AAClB,QAAI,QAAiB,CAAC;AACtB,UAAM,QAAQ,MAAM;AAClB,UAAI,CAAC,MAAM,OAAQ;AACnB;AAAA,QACE,KAAK;AAAA,QACL,MAAM;AACJ,mBAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,mBAAO,MAAM,CAAC,CAAC;AAAA,UACjB;AAAA,QACF;AAAA,QACA,EAAE,MAAM,YAAY;AAAA,MACtB;AACA,cAAQ,CAAC;AAAA,IACX;AAEA,qBAAiB,SAAS,OAAO,QAAQ,GAAG;AAC1C,YAAM,KAAK,KAAK;AAChB,UAAI,MAAM,UAAU,UAAW,OAAM;AAAA,IACvC;AAEA,UAAM;AAAA,EACR;AACF;AAEO,SAAS,aAAa,QAAyC;AACpE,MAAI,kBAAkB,cAAc;AAElC,UAAM,SAAS,IAAI,aAAa,OAAO,MAAM;AAC7C,WAAO,IAAI,MAAM;AACjB,WAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAClC;AACA,QAAM,aAAa,IAAI,aAAa,MAAM;AAC1C,SAAO,OAAO,KAAK,WAAW,MAAM;AACtC;AAEA,SAAS,eAAe,KAA4B;AAClD,MACE,OAAO,QAAQ,YACf,QAAQ,QACR,EAAE,aAAa,QACf,OAAO,IAAI,YAAY,YACvB,EAAE,cAAc,QAChB,OAAO,IAAI,aAAa,YACxB,EAAE,iBAAiB,QACnB,OAAO,IAAI,gBAAgB,YAC3B,EAAE,cAAc,QACf,IAAI,aAAa,QAAQ,OAAO,IAAI,aAAa,UAClD;AACA,UAAM,IAAI,MAAM,8BAA8B;AAAA,EAChD;AACA,SAAO;AAAA,IACL,SAAS,IAAI;AAAA,IACb,UAAU,IAAI;AAAA,IACd,aAAa,IAAI;AAAA,IACjB,UAAU,cAAc,IAAI,QAAQ;AAAA,EACtC;AACF;AAEA,SAAS,cAAc,OAAsD;AAC3E,MAAI,UAAU,KAAM,QAAO;AAC3B,MAAI;AACF,UAAM,SAAkB,KAAK,MAAM,KAAK;AACxC,WAAO,SAAS,MAAM,IAAI,SAAS;AAAA,EACrC,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,SAAS,SAAS,OAAkD;AAClE,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;;;AErSO,SAAS,WAAW,QAAgB,WAAmB;AAC5D,SAAO,IAAI,YAAY,gBAAgB,MAAM,GAAG,SAAS;AAC3D;",
|
|
4
|
+
"sourcesContent": ["import type {\n FeatureExtractionPipeline,\n Tensor,\n} from '@huggingface/transformers';\n\nimport type { Embedder } from '../stores/store.ts';\n\nexport type FeatureExtractionFn = FeatureExtractionPipeline;\n\nexport interface EmbeddingOptions {\n extractorFn: () =>\n Promise<FeatureExtractionPipeline> | FeatureExtractionPipeline;\n pooling?: 'mean' | 'cls';\n normalize?: boolean;\n}\n\nasync function textEmbeddings(\n inputs: string | string[],\n { extractorFn, pooling, normalize }: EmbeddingOptions,\n) {\n inputs = (Array.isArray(inputs) ? inputs : [inputs]).map((it) => it.trim());\n const extractor = await extractorFn();\n const tensor = await extractor(inputs, {\n pooling: pooling ?? 'mean',\n normalize: normalize ?? true,\n });\n return tensorToEmbeddings(tensor);\n}\n\nexport function tensorToEmbeddings(tensor: Tensor) {\n const dims = tensor.dims;\n const [batchSize, ...rest] = Array.isArray(dims) ? dims : [];\n const hiddenSize = rest.at(-1);\n if (hiddenSize === undefined) {\n throw new Error(`Unexpected tensor dims: ${JSON.stringify(dims)}`);\n }\n const expectedLen = batchSize * hiddenSize;\n if (tensor.data.length !== expectedLen) {\n throw new Error(\n `Data length mismatch: got ${tensor.data.length}, expected ${expectedLen} (batch=${batchSize}, hidden=${hiddenSize})`,\n );\n }\n const data = tensor.data;\n if (!(data instanceof Float32Array)) {\n throw new Error(`Expected float32 embeddings, got ${tensor.type}`);\n }\n // Reuse the underlying typed array without copying; we'll copy when creating Buffer for SQLite\n const embeddings: Float32Array[] = [];\n const dimensions = hiddenSize;\n for (let i = 0; i < batchSize; i++) {\n const start = i * hiddenSize;\n embeddings.push(data.subarray(start, start + hiddenSize));\n }\n return { embeddings, dimensions };\n}\n\nexport function huggingface(\n extractorFn: () =>\n Promise<FeatureExtractionPipeline> | FeatureExtractionPipeline,\n): Embedder {\n return (documents) => textEmbeddings(documents, { extractorFn });\n}\n", "import type { FeatureExtractionPipeline } from '@huggingface/transformers';\n\nimport type { Embedder } from '../stores/store.ts';\nimport { tensorToEmbeddings } from './huggingface.ts';\n\n/**\n * Stable model identifiers retained from the former `fastembed` adapter.\n */\nexport const EmbeddingModels = {\n AllMiniLML6V2: 'fast-all-MiniLM-L6-v2',\n BGEBaseEN: 'fast-bge-base-en',\n BGEBaseENV15: 'fast-bge-base-en-v1.5',\n BGESmallEN: 'fast-bge-small-en',\n BGESmallENV15: 'fast-bge-small-en-v1.5',\n BGESmallZH: 'fast-bge-small-zh-v1.5',\n MLE5Large: 'fast-multilingual-e5-large',\n} as const;\nexport type EmbeddingModel =\n (typeof EmbeddingModels)[keyof typeof EmbeddingModels];\n\nconst transformerModels: Record<EmbeddingModel, string> = {\n [EmbeddingModels.AllMiniLML6V2]: 'Xenova/all-MiniLM-L6-v2',\n [EmbeddingModels.BGEBaseEN]: 'Xenova/bge-base-en',\n [EmbeddingModels.BGEBaseENV15]: 'Xenova/bge-base-en-v1.5',\n [EmbeddingModels.BGESmallEN]: 'Xenova/bge-small-en',\n [EmbeddingModels.BGESmallENV15]: 'Xenova/bge-small-en-v1.5',\n [EmbeddingModels.BGESmallZH]: 'Xenova/bge-small-zh-v1.5',\n [EmbeddingModels.MLE5Large]: 'Xenova/multilingual-e5-large',\n};\n\nexport interface FastEmbedOptions {\n model?: EmbeddingModel;\n batchSize?: number;\n cacheDir?: string;\n}\n\n/**\n * Local feature extraction backed by Transformers.js. The historical function\n * and model identifiers remain stable while avoiding FastEmbed's archived tar\n * extraction dependency.\n */\nexport function fastembed(options: FastEmbedOptions = {}): Embedder {\n const {\n model: modelId = EmbeddingModels.BGESmallENV15,\n batchSize = 256,\n cacheDir,\n } = options;\n\n if (!Number.isInteger(batchSize) || batchSize <= 0) {\n throw new RangeError('batchSize must be a positive integer');\n }\n\n let modelPromise: Promise<FeatureExtractionPipeline> | null = null;\n const getModel = () => {\n if (!modelPromise) {\n modelPromise = import('@huggingface/transformers').then(({ pipeline }) =>\n pipeline('feature-extraction', transformerModels[modelId], {\n cache_dir: cacheDir,\n }),\n );\n }\n return modelPromise;\n };\n\n return async (documents: string[]) => {\n if (documents.length === 0) {\n return { embeddings: [], dimensions: 0 };\n }\n\n const model = await getModel();\n const embeddings: Float32Array[] = [];\n let dimensions = 0;\n\n for (let start = 0; start < documents.length; start += batchSize) {\n const tensor = await model(documents.slice(start, start + batchSize), {\n pooling: 'mean',\n normalize: true,\n });\n const batch = tensorToEmbeddings(tensor);\n dimensions ||= batch.dimensions;\n embeddings.push(...batch.embeddings);\n }\n\n return { embeddings, dimensions };\n };\n}\n", "import {\n MarkdownTextSplitter,\n RecursiveCharacterTextSplitter,\n} from '@langchain/textsplitters';\n\nimport type { Connector } from './connectors/connector.ts';\nimport type { Splitter } from './pipeline.ts';\nimport { cid } from './stores/cid.ts';\nimport type { Embedder, Store } from './stores/store.ts';\n\nexport interface IngestionConfig {\n connector: Connector;\n store: Store;\n splitter?: Splitter;\n embedder: Embedder;\n}\n\nexport async function ingest(\n config: IngestionConfig,\n callback?: (it: string) => void,\n) {\n const splitter = config.splitter ?? split;\n const embedder = config.embedder;\n const corpuses = config.connector.sources();\n const expiryDate = expiry(config.connector);\n\n for await (const it of corpuses) {\n callback?.(it.id);\n const content = await it.content();\n if (!content.trim()) {\n // skip empty files\n continue;\n }\n await config.store.index(\n config.connector.sourceId,\n {\n id: it.id,\n cid: cid(content),\n metadata: it.metadata,\n chunker: async function* () {\n // Embed in small batches to control memory usage\n const values = await splitter(it.id, content);\n const batchSize = 40;\n for (let i = 0; i < values.length; i += batchSize) {\n const batch = values.slice(i, i + batchSize);\n const { embeddings } = await embedder(batch);\n for (let j = 0; j < embeddings.length; j++) {\n yield {\n content: batch[j],\n embedding: embeddings[j],\n };\n }\n }\n },\n },\n expiryDate,\n );\n }\n}\n\nfunction expiry(connector: Connector) {\n if (connector.ingestWhen !== 'expired') return undefined;\n if (connector.expiresAfter === undefined) return undefined;\n return new Date(connector.clock.Date.now() + connector.expiresAfter);\n}\n\nfunction split(id: string, content: string) {\n const splitter = new MarkdownTextSplitter();\n return splitter.splitText(content);\n}\n\nexport type ChunkPosition = {\n startLine: number;\n startColumn: number;\n endLine: number;\n endColumn: number;\n};\n\nexport type SplitChunkWithPosition = {\n content: string;\n position: ChunkPosition | null;\n index: number;\n};\n\nfunction normalizeNewlines(value: string) {\n return value.replace(/\\r\\n/g, '\\n');\n}\n\nfunction computePositions(\n originalContent: string,\n chunks: string[],\n): Array<ChunkPosition | null> {\n if (!chunks.length) {\n return [];\n }\n\n const normalizedContent = normalizeNewlines(originalContent);\n const positions: Array<ChunkPosition | null> = [];\n let searchOffset = 0;\n\n for (const chunk of chunks) {\n const normalizedChunk = normalizeNewlines(chunk);\n const trimmedChunk = normalizedChunk.trim();\n\n const seek = (needle: string, fromIndex: number) =>\n needle ? normalizedContent.indexOf(needle, fromIndex) : -1;\n\n let matchIndex = seek(normalizedChunk, searchOffset);\n let matchValue = normalizedChunk;\n\n if (matchIndex === -1 && trimmedChunk) {\n matchIndex = seek(trimmedChunk, searchOffset);\n matchValue = trimmedChunk;\n }\n\n if (matchIndex === -1) {\n matchIndex = seek(normalizedChunk, 0);\n matchValue = normalizedChunk;\n }\n\n if (matchIndex === -1 && trimmedChunk) {\n matchIndex = seek(trimmedChunk, 0);\n matchValue = trimmedChunk;\n }\n\n if (matchIndex === -1) {\n positions.push(null);\n continue;\n }\n\n const before = normalizedContent.slice(0, matchIndex);\n const beforeLines = before.split('\\n');\n const startLine = beforeLines.length;\n const startColumn = before.length - before.lastIndexOf('\\n');\n\n const lines = matchValue.split('\\n');\n const endLine = startLine + lines.length - 1;\n const endColumn =\n lines.length === 1\n ? startColumn + lines[0].length\n : matchValue.length - matchValue.lastIndexOf('\\n');\n\n positions.push({ startLine, startColumn, endLine, endColumn });\n searchOffset = matchIndex + matchValue.length;\n }\n\n return positions;\n}\n\nfunction buildChunksWithPositions(\n originalContent: string,\n chunks: string[],\n): SplitChunkWithPosition[] {\n const positions = computePositions(originalContent, chunks);\n return chunks.map((content, index) => ({\n content,\n index,\n position: positions[index] ?? null,\n }));\n}\n\nexport async function splitTypeScriptWithPositions(\n id: string,\n content: string,\n): Promise<SplitChunkWithPosition[]> {\n const splitter = RecursiveCharacterTextSplitter.fromLanguage('js', {\n chunkSize: 512,\n chunkOverlap: 100,\n });\n const docs = await splitter.createDocuments([content]);\n const chunks = docs.map((d) => d.pageContent);\n return buildChunksWithPositions(content, chunks);\n}\n\nexport async function splitTypeScript(id: string, content: string) {\n const chunks = await splitTypeScriptWithPositions(id, content);\n return chunks.map((chunk) => chunk.content);\n}\n", "import { hash } from 'node:crypto';\n\nexport function cid(content: string) {\n return hash('sha256', content);\n}\n", "import { embedMany } from 'ai';\n\nimport { type IngestionConfig, ingest } from './ingest.ts';\n\nexport async function similaritySearch(\n query: string,\n config: Omit<IngestionConfig, 'splitter'>,\n) {\n // Ingest if needed, then perform vector similarity search via the configured store\n const { connector } = config;\n let shouldIngest = true;\n\n if (connector.ingestWhen === 'never') {\n if (await config.store.sourceExists(config.connector.sourceId)) {\n console.log(\n `Skipping ingestion for source ${config.connector.sourceId} (ingestWhen=never and source exists)`,\n );\n shouldIngest = false;\n }\n } else if (connector.ingestWhen === 'expired') {\n const sourceExists = await config.store.sourceExists(\n config.connector.sourceId,\n );\n if (\n sourceExists &&\n !(await config.store.sourceExpired(\n config.connector.sourceId,\n new Date(connector.clock.Date.now()),\n ))\n ) {\n console.log(\n `Skipping ingestion for source ${config.connector.sourceId} (ingestWhen=expired and source not expired)`,\n );\n shouldIngest = false;\n }\n }\n\n if (shouldIngest) {\n await ingest(config);\n }\n return config.store\n .search(\n query,\n { sourceId: config.connector.sourceId, topN: 50 },\n config.embedder,\n )\n .then((results) =>\n results.map((it) => ({\n ...it,\n similarity: 1 - it.distance,\n distance: it.distance,\n })),\n );\n}\n", "import { WriteAheadLog, open } from '@zukhruf/sqlite-open';\nimport * as sqliteVec from 'sqlite-vec';\n\n// The store's statements use the vec0 module, so every connection that runs\n// them, the dev database of the declaration included, loads sqlite-vec. The\n// rows of vectors are large, so a new file gets 32 KiB pages; the write-ahead\n// log fixes the page size, so the journal sets it before it.\nexport function openWithVectors(path: string) {\n const db = open(path, {\n allowExtension: true,\n journal: new WriteAheadLog({ pageSize: 32768 }),\n });\n db.loadExtension(sqliteVec.getLoadablePath());\n return db;\n}\n", "import { Declaration } from '@zukhruf/sqlite-drift';\nimport { transaction } from '@zukhruf/sqlite-transaction';\nimport { Vector } from '@zukhruf/vectors';\nimport { toSql } from '@zukhruf/vectors/sqlite';\nimport dedent from 'dedent';\nimport type { DatabaseSync } from 'node:sqlite';\n\nimport type {\n Chunk,\n Corpus,\n Embedder,\n SearchOptions,\n SearchResult,\n Store,\n} from '../store.ts';\nimport { SETUP, statements, vectors } from './ddl.sqlite.ts';\nimport { openWithVectors } from './open.ts';\n\nconst DEFAULT_TOP_N = 10;\n\nconst tables = new Declaration(statements, { rows: 'keep' });\n\nexport class SQLiteStore implements Store {\n readonly #db: DatabaseSync;\n constructor(db: DatabaseSync, dimension: number) {\n this.#db = db;\n this.#db.exec(SETUP);\n const chunks = new Declaration([vectors(dimension)], {\n rows: 'replace',\n devDatabase: () => openWithVectors(':memory:'),\n });\n transaction(\n this.#db,\n () => {\n const stale = chunks.drift(this.#db) === 'stale';\n chunks.apply(this.#db);\n // A replaced vec_chunks is empty, yet upsertDoc skips a document whose\n // cid is unchanged and similaritySearch skips a source that exists.\n // Deleting the sources cascades to their documents, so the next\n // ingest embeds every document again.\n if (stale) this.#db.exec('DELETE FROM sources');\n },\n { mode: 'immediate' },\n );\n tables.apply(this.#db);\n }\n\n #searchByDocument(inputs: {\n sourceId: string;\n documentId: string;\n embedding: Uint8Array;\n k: number;\n }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT v.content, v.distance, v.document_id, d.metadata\n\t\t\t\tFROM vec_chunks v\n\t\t\t\tJOIN documents d ON d.id = v.document_id\n\t\t\t\tWHERE v.source_id = ?\n\t\t\t\t\tAND v.document_id = ?\n\t\t\t\t\tAND v.embedding MATCH vec_normalize(vec_f32(?))\n\t\t\t\t\tAND v.k = ?\n\t\t\t\tORDER BY v.distance ASC\n\t\t\t`);\n return stmt.all(\n inputs.sourceId,\n inputs.documentId,\n inputs.embedding,\n inputs.k,\n );\n }\n #searchBySource(inputs: {\n sourceId: string;\n embedding: Uint8Array;\n k: number;\n }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT v.content, v.distance, v.document_id, d.metadata\n\t\t\t\tFROM vec_chunks v\n\t\t\t\tJOIN documents d ON d.id = v.document_id\n\t\t\t\tWHERE v.source_id = ?\n\t\t\t\t\tAND v.embedding MATCH vec_normalize(vec_f32(?))\n\t\t\t\t\tAND v.k = ?\n\t\t\t\tORDER BY v.distance ASC\n\t\t\t`);\n return stmt.all(inputs.sourceId, inputs.embedding, inputs.k);\n }\n async search(query: string, options: SearchOptions, embedder: Embedder) {\n const { embeddings } = await embedder([query]);\n if (!embeddings.length) {\n return [];\n }\n const vectorBlob = toSql(Vector.from(embeddings[0]));\n const topN = options.topN;\n\n if (options.documentId) {\n const rows = this.#searchByDocument({\n sourceId: options.sourceId,\n documentId: options.documentId,\n embedding: vectorBlob,\n k: topN ?? DEFAULT_TOP_N,\n });\n return rows.map(toSearchResult);\n }\n\n const rows = this.#searchBySource({\n sourceId: options.sourceId,\n embedding: vectorBlob,\n k: topN ?? DEFAULT_TOP_N,\n });\n return rows.map(toSearchResult);\n }\n #upsertSource(inputs: { sourceId: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tINSERT INTO sources (source_id) VALUES (?)\n\t\t\t\tON CONFLICT(source_id) DO UPDATE SET updated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now')\n\t\t\t`);\n return stmt.run(inputs.sourceId);\n }\n #upsertSourceWithExpiry(inputs: { sourceId: string; expiresAt: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tINSERT INTO sources (source_id, expires_at) VALUES (?, ?)\n\t\t\t\tON CONFLICT(source_id) DO UPDATE SET\n\t\t\t\t\tupdated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now'),\n\t\t\t\t\texpires_at=excluded.expires_at\n\t\t\t`);\n return stmt.run(inputs.sourceId, inputs.expiresAt);\n }\n upsertDoc(inputs: {\n documentId: string;\n sourceId: string;\n cid: string;\n metadata?: Record<string, any>;\n }) {\n const stmt = this.#db.prepare(dedent`\n INSERT INTO documents (id, source_id, cid, metadata)\n VALUES (?, ?, ?, json(?))\n ON CONFLICT(id) DO UPDATE SET\n cid=excluded.cid,\n metadata=COALESCE(excluded.metadata, documents.metadata),\n updated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now')\n WHERE documents.cid != excluded.cid;\n `);\n return stmt.run(\n inputs.documentId,\n inputs.sourceId,\n inputs.cid,\n inputs.metadata ? JSON.stringify(inputs.metadata) : null,\n );\n }\n insertDoc(inputs: { sourceId: string; documentId: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tINSERT INTO vec_chunks (source_id, document_id, content, embedding)\n\t\t\t\tVALUES (?, ?, ?, vec_normalize(vec_f32(?)))\n\t\t\t`);\n return (chunk: Chunk) => {\n stmt.run(\n inputs.sourceId,\n inputs.documentId,\n chunk.content,\n toSql(Vector.from(chunk.embedding)),\n );\n };\n }\n delete(inputs: { sourceId: string; documentId: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tDELETE FROM vec_chunks WHERE source_id = ? AND document_id = ?\n\t\t\t`);\n return stmt.run(inputs.sourceId, inputs.documentId);\n }\n sourceExists(sourceId: string) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT 1 FROM sources WHERE source_id = ? LIMIT 1\n\t\t\t`);\n const row = stmt.get(sourceId);\n return Boolean(row);\n }\n sourceExpired(sourceId: string, now: Date) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT 1 FROM sources\n\t\t\t\tWHERE source_id = ?\n\t\t\t\t\tAND expires_at IS NOT NULL\n\t\t\t\t\tAND expires_at <= ?\n\t\t\t\tLIMIT 1\n\t\t\t`);\n const row = stmt.get(sourceId, now.toISOString());\n return Boolean(row);\n }\n setSourceExpiry(sourceId: string, expiryDate: Date) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tUPDATE sources\n\t\t\t\tSET expires_at = ?, updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')\n\t\t\t\tWHERE source_id = ?\n\t\t\t`);\n stmt.run(expiryDate.toISOString(), sourceId);\n }\n async index(sourceId: string, corpus: Corpus, expiryDate?: Date) {\n if (expiryDate) {\n this.#upsertSourceWithExpiry({\n expiresAt: expiryDate.toISOString(),\n sourceId,\n });\n } else {\n this.#upsertSource({ sourceId });\n }\n\n const info = this.upsertDoc({\n documentId: corpus.id,\n sourceId,\n cid: corpus.cid,\n metadata: corpus.metadata,\n });\n const changed = info.changes > 0;\n\n if (!changed) {\n return;\n }\n\n const insert = this.insertDoc({ sourceId, documentId: corpus.id });\n // Delete previous rows once before inserting\n transaction(\n this.#db,\n () => {\n this.delete({ sourceId, documentId: corpus.id });\n },\n { mode: 'immediate' },\n );\n\n const batchSize = 32;\n let batch: Chunk[] = [];\n const flush = () => {\n if (!batch.length) return;\n transaction(\n this.#db,\n () => {\n for (let i = 0; i < batch.length; i++) {\n insert(batch[i]);\n }\n },\n { mode: 'immediate' },\n );\n batch = [];\n };\n\n for await (const chunk of corpus.chunker()) {\n batch.push(chunk);\n if (batch.length >= batchSize) flush();\n }\n // flush any remaining\n flush();\n }\n}\n\nfunction toSearchResult(row: unknown): SearchResult {\n if (\n typeof row !== 'object' ||\n row === null ||\n !('content' in row) ||\n typeof row.content !== 'string' ||\n !('distance' in row) ||\n typeof row.distance !== 'number' ||\n !('document_id' in row) ||\n typeof row.document_id !== 'string' ||\n !('metadata' in row) ||\n (row.metadata !== null && typeof row.metadata !== 'string')\n ) {\n throw new Error('Unexpected vector search row');\n }\n return {\n content: row.content,\n distance: row.distance,\n document_id: row.document_id,\n metadata: parseMetadata(row.metadata),\n };\n}\n\nfunction parseMetadata(value: string | null): Record<string, unknown> | null {\n if (value === null) return null;\n try {\n const parsed: unknown = JSON.parse(value);\n return isRecord(parsed) ? parsed : null;\n } catch {\n return null;\n }\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n return typeof value === 'object' && value !== null && !Array.isArray(value);\n}\n", "// Embedding store schema for SQLite. SQLite stores the text of each CREATE and\n// @zukhruf/sqlite-drift compares it, so the text after each object name must\n// stay byte-identical to what earlier releases wrote, inline comments included.\n\n// Session-level PRAGMAs, run on each connection before the schema is applied.\nexport const SETUP = `\nPRAGMA synchronous = NORMAL;\nPRAGMA temp_store = MEMORY;\nPRAGMA cache_size = -131072; -- ~128 MiB page cache for faster repeated reads\nPRAGMA mmap_size = 268435456; -- 256 MiB memory map window to cut syscalls\nPRAGMA wal_autocheckpoint = 1000;\nPRAGMA optimize;\n`;\n\nexport function vectors(dimension: number) {\n return `CREATE VIRTUAL TABLE vec_chunks USING vec0(\n source_id TEXT PARTITION KEY,\n document_id TEXT,\n embedding FLOAT[${dimension}] DISTANCE_METRIC=cosine,\n +content TEXT -- auxiliary payload (not filterable) for reconstruction\n)`;\n}\n\nexport const statements = [\n `CREATE TABLE sources (\n source_id TEXT PRIMARY KEY,\n updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n expires_at TEXT\n) STRICT`,\n `CREATE TABLE documents (\n id TEXT PRIMARY KEY,\n source_id TEXT NOT NULL,\n cid TEXT NOT NULL,\n metadata TEXT, -- JSON blob with arbitrary document metadata\n updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n FOREIGN KEY (source_id) REFERENCES sources(source_id) ON DELETE CASCADE\n) STRICT`,\n // Indexes to accelerate lookups / maintenance\n `CREATE INDEX idx_documents_source ON documents(source_id)`,\n `CREATE INDEX idx_documents_source_updated ON documents(source_id, updated_at)`,\n `CREATE INDEX idx_documents_cid ON documents(cid)`,\n `CREATE INDEX idx_sources_expires_at ON sources(expires_at) WHERE expires_at IS NOT NULL`,\n // emulate cascade for the virtual table\n `CREATE TRIGGER trg_documents_delete_vec\nAFTER DELETE ON documents\nBEGIN\n DELETE FROM vec_chunks\n WHERE source_id = OLD.source_id\n AND document_id = OLD.id;\nEND`,\n];\n", "import { openWithVectors } from './open.ts';\nimport { SQLiteStore } from './sqlite.ts';\n\nexport function nodeSQLite(dbName: string, dimension: number) {\n return new SQLiteStore(openWithVectors(dbName), dimension);\n}\n"],
|
|
5
|
+
"mappings": ";AA6BO,SAAS,mBAAmB,QAAgB;AACjD,QAAM,OAAO,OAAO;AACpB,QAAM,CAAC,WAAW,GAAG,IAAI,IAAI,MAAM,QAAQ,IAAI,IAAI,OAAO,CAAC;AAC3D,QAAM,aAAa,KAAK,GAAG,EAAE;AAC7B,MAAI,eAAe,QAAW;AAC5B,UAAM,IAAI,MAAM,2BAA2B,KAAK,UAAU,IAAI,CAAC,EAAE;AAAA,EACnE;AACA,QAAM,cAAc,YAAY;AAChC,MAAI,OAAO,KAAK,WAAW,aAAa;AACtC,UAAM,IAAI;AAAA,MACR,6BAA6B,OAAO,KAAK,MAAM,cAAc,WAAW,WAAW,SAAS,YAAY,UAAU;AAAA,IACpH;AAAA,EACF;AACA,QAAM,OAAO,OAAO;AACpB,MAAI,EAAE,gBAAgB,eAAe;AACnC,UAAM,IAAI,MAAM,oCAAoC,OAAO,IAAI,EAAE;AAAA,EACnE;AAEA,QAAM,aAA6B,CAAC;AACpC,QAAM,aAAa;AACnB,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,IAAI;AAClB,eAAW,KAAK,KAAK,SAAS,OAAO,QAAQ,UAAU,CAAC;AAAA,EAC1D;AACA,SAAO,EAAE,YAAY,WAAW;AAClC;;;AC9CO,IAAM,kBAAkB;AAAA,EAC7B,eAAe;AAAA,EACf,WAAW;AAAA,EACX,cAAc;AAAA,EACd,YAAY;AAAA,EACZ,eAAe;AAAA,EACf,YAAY;AAAA,EACZ,WAAW;AACb;AAIA,IAAM,oBAAoD;AAAA,EACxD,CAAC,gBAAgB,aAAa,GAAG;AAAA,EACjC,CAAC,gBAAgB,SAAS,GAAG;AAAA,EAC7B,CAAC,gBAAgB,YAAY,GAAG;AAAA,EAChC,CAAC,gBAAgB,UAAU,GAAG;AAAA,EAC9B,CAAC,gBAAgB,aAAa,GAAG;AAAA,EACjC,CAAC,gBAAgB,UAAU,GAAG;AAAA,EAC9B,CAAC,gBAAgB,SAAS,GAAG;AAC/B;AAaO,SAAS,UAAU,UAA4B,CAAC,GAAa;AAClE,QAAM;AAAA,IACJ,OAAO,UAAU,gBAAgB;AAAA,IACjC,YAAY;AAAA,IACZ;AAAA,EACF,IAAI;AAEJ,MAAI,CAAC,OAAO,UAAU,SAAS,KAAK,aAAa,GAAG;AAClD,UAAM,IAAI,WAAW,sCAAsC;AAAA,EAC7D;AAEA,MAAI,eAA0D;AAC9D,QAAM,WAAW,MAAM;AACrB,QAAI,CAAC,cAAc;AACjB,qBAAe,OAAO,2BAA2B,EAAE;AAAA,QAAK,CAAC,EAAE,SAAS,MAClE,SAAS,sBAAsB,kBAAkB,OAAO,GAAG;AAAA,UACzD,WAAW;AAAA,QACb,CAAC;AAAA,MACH;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAEA,SAAO,OAAO,cAAwB;AACpC,QAAI,UAAU,WAAW,GAAG;AAC1B,aAAO,EAAE,YAAY,CAAC,GAAG,YAAY,EAAE;AAAA,IACzC;AAEA,UAAM,QAAQ,MAAM,SAAS;AAC7B,UAAM,aAA6B,CAAC;AACpC,QAAI,aAAa;AAEjB,aAAS,QAAQ,GAAG,QAAQ,UAAU,QAAQ,SAAS,WAAW;AAChE,YAAM,SAAS,MAAM,MAAM,UAAU,MAAM,OAAO,QAAQ,SAAS,GAAG;AAAA,QACpE,SAAS;AAAA,QACT,WAAW;AAAA,MACb,CAAC;AACD,YAAM,QAAQ,mBAAmB,MAAM;AACvC,qBAAe,MAAM;AACrB,iBAAW,KAAK,GAAG,MAAM,UAAU;AAAA,IACrC;AAEA,WAAO,EAAE,YAAY,WAAW;AAAA,EAClC;AACF;;;ACrFA;AAAA,EACE;AAAA,EACA;AAAA,OACK;;;ACHP,SAAS,YAAY;AAEd,SAAS,IAAI,SAAiB;AACnC,SAAO,KAAK,UAAU,OAAO;AAC/B;;;ADaA,eAAsB,OACpB,QACA,UACA;AACA,QAAM,WAAW,OAAO,YAAY;AACpC,QAAM,WAAW,OAAO;AACxB,QAAM,WAAW,OAAO,UAAU,QAAQ;AAC1C,QAAM,aAAa,OAAO,OAAO,SAAS;AAE1C,mBAAiB,MAAM,UAAU;AAC/B,eAAW,GAAG,EAAE;AAChB,UAAM,UAAU,MAAM,GAAG,QAAQ;AACjC,QAAI,CAAC,QAAQ,KAAK,GAAG;AAEnB;AAAA,IACF;AACA,UAAM,OAAO,MAAM;AAAA,MACjB,OAAO,UAAU;AAAA,MACjB;AAAA,QACE,IAAI,GAAG;AAAA,QACP,KAAK,IAAI,OAAO;AAAA,QAChB,UAAU,GAAG;AAAA,QACb,SAAS,mBAAmB;AAE1B,gBAAM,SAAS,MAAM,SAAS,GAAG,IAAI,OAAO;AAC5C,gBAAM,YAAY;AAClB,mBAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,WAAW;AACjD,kBAAM,QAAQ,OAAO,MAAM,GAAG,IAAI,SAAS;AAC3C,kBAAM,EAAE,WAAW,IAAI,MAAM,SAAS,KAAK;AAC3C,qBAAS,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AAC1C,oBAAM;AAAA,gBACJ,SAAS,MAAM,CAAC;AAAA,gBAChB,WAAW,WAAW,CAAC;AAAA,cACzB;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,OAAO,WAAsB;AACpC,MAAI,UAAU,eAAe,UAAW,QAAO;AAC/C,MAAI,UAAU,iBAAiB,OAAW,QAAO;AACjD,SAAO,IAAI,KAAK,UAAU,MAAM,KAAK,IAAI,IAAI,UAAU,YAAY;AACrE;AAEA,SAAS,MAAM,IAAY,SAAiB;AAC1C,QAAM,WAAW,IAAI,qBAAqB;AAC1C,SAAO,SAAS,UAAU,OAAO;AACnC;AAeA,SAAS,kBAAkB,OAAe;AACxC,SAAO,MAAM,QAAQ,SAAS,IAAI;AACpC;AAEA,SAAS,iBACP,iBACA,QAC6B;AAC7B,MAAI,CAAC,OAAO,QAAQ;AAClB,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,oBAAoB,kBAAkB,eAAe;AAC3D,QAAM,YAAyC,CAAC;AAChD,MAAI,eAAe;AAEnB,aAAW,SAAS,QAAQ;AAC1B,UAAM,kBAAkB,kBAAkB,KAAK;AAC/C,UAAM,eAAe,gBAAgB,KAAK;AAE1C,UAAM,OAAO,CAAC,QAAgB,cAC5B,SAAS,kBAAkB,QAAQ,QAAQ,SAAS,IAAI;AAE1D,QAAI,aAAa,KAAK,iBAAiB,YAAY;AACnD,QAAI,aAAa;AAEjB,QAAI,eAAe,MAAM,cAAc;AACrC,mBAAa,KAAK,cAAc,YAAY;AAC5C,mBAAa;AAAA,IACf;AAEA,QAAI,eAAe,IAAI;AACrB,mBAAa,KAAK,iBAAiB,CAAC;AACpC,mBAAa;AAAA,IACf;AAEA,QAAI,eAAe,MAAM,cAAc;AACrC,mBAAa,KAAK,cAAc,CAAC;AACjC,mBAAa;AAAA,IACf;AAEA,QAAI,eAAe,IAAI;AACrB,gBAAU,KAAK,IAAI;AACnB;AAAA,IACF;AAEA,UAAM,SAAS,kBAAkB,MAAM,GAAG,UAAU;AACpD,UAAM,cAAc,OAAO,MAAM,IAAI;AACrC,UAAM,YAAY,YAAY;AAC9B,UAAM,cAAc,OAAO,SAAS,OAAO,YAAY,IAAI;AAE3D,UAAM,QAAQ,WAAW,MAAM,IAAI;AACnC,UAAM,UAAU,YAAY,MAAM,SAAS;AAC3C,UAAM,YACJ,MAAM,WAAW,IACb,cAAc,MAAM,CAAC,EAAE,SACvB,WAAW,SAAS,WAAW,YAAY,IAAI;AAErD,cAAU,KAAK,EAAE,WAAW,aAAa,SAAS,UAAU,CAAC;AAC7D,mBAAe,aAAa,WAAW;AAAA,EACzC;AAEA,SAAO;AACT;AAEA,SAAS,yBACP,iBACA,QAC0B;AAC1B,QAAM,YAAY,iBAAiB,iBAAiB,MAAM;AAC1D,SAAO,OAAO,IAAI,CAAC,SAAS,WAAW;AAAA,IACrC;AAAA,IACA;AAAA,IACA,UAAU,UAAU,KAAK,KAAK;AAAA,EAChC,EAAE;AACJ;AAEA,eAAsB,6BACpB,IACA,SACmC;AACnC,QAAM,WAAW,+BAA+B,aAAa,MAAM;AAAA,IACjE,WAAW;AAAA,IACX,cAAc;AAAA,EAChB,CAAC;AACD,QAAM,OAAO,MAAM,SAAS,gBAAgB,CAAC,OAAO,CAAC;AACrD,QAAM,SAAS,KAAK,IAAI,CAAC,MAAM,EAAE,WAAW;AAC5C,SAAO,yBAAyB,SAAS,MAAM;AACjD;AAEA,eAAsB,gBAAgB,IAAY,SAAiB;AACjE,QAAM,SAAS,MAAM,6BAA6B,IAAI,OAAO;AAC7D,SAAO,OAAO,IAAI,CAAC,UAAU,MAAM,OAAO;AAC5C;;;AEjLA,OAA0B;AAI1B,eAAsB,iBACpB,OACA,QACA;AAEA,QAAM,EAAE,UAAU,IAAI;AACtB,MAAI,eAAe;AAEnB,MAAI,UAAU,eAAe,SAAS;AACpC,QAAI,MAAM,OAAO,MAAM,aAAa,OAAO,UAAU,QAAQ,GAAG;AAC9D,cAAQ;AAAA,QACN,iCAAiC,OAAO,UAAU,QAAQ;AAAA,MAC5D;AACA,qBAAe;AAAA,IACjB;AAAA,EACF,WAAW,UAAU,eAAe,WAAW;AAC7C,UAAM,eAAe,MAAM,OAAO,MAAM;AAAA,MACtC,OAAO,UAAU;AAAA,IACnB;AACA,QACE,gBACA,CAAE,MAAM,OAAO,MAAM;AAAA,MACnB,OAAO,UAAU;AAAA,MACjB,IAAI,KAAK,UAAU,MAAM,KAAK,IAAI,CAAC;AAAA,IACrC,GACA;AACA,cAAQ;AAAA,QACN,iCAAiC,OAAO,UAAU,QAAQ;AAAA,MAC5D;AACA,qBAAe;AAAA,IACjB;AAAA,EACF;AAEA,MAAI,cAAc;AAChB,UAAM,OAAO,MAAM;AAAA,EACrB;AACA,SAAO,OAAO,MACX;AAAA,IACC;AAAA,IACA,EAAE,UAAU,OAAO,UAAU,UAAU,MAAM,GAAG;AAAA,IAChD,OAAO;AAAA,EACT,EACC;AAAA,IAAK,CAAC,YACL,QAAQ,IAAI,CAAC,QAAQ;AAAA,MACnB,GAAG;AAAA,MACH,YAAY,IAAI,GAAG;AAAA,MACnB,UAAU,GAAG;AAAA,IACf,EAAE;AAAA,EACJ;AACJ;;;ACrDA,SAAS,eAAe,YAAY;AACpC,YAAY,eAAe;AAMpB,SAAS,gBAAgB,MAAc;AAC5C,QAAM,KAAK,KAAK,MAAM;AAAA,IACpB,gBAAgB;AAAA,IAChB,SAAS,IAAI,cAAc,EAAE,UAAU,MAAM,CAAC;AAAA,EAChD,CAAC;AACD,KAAG,cAAwB,0BAAgB,CAAC;AAC5C,SAAO;AACT;;;ACdA,SAAS,mBAAmB;AAC5B,SAAS,mBAAmB;AAC5B,SAAS,cAAc;AACvB,SAAS,aAAa;AACtB,OAAO,YAAY;;;ACCZ,IAAM,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AASd,SAAS,QAAQ,WAAmB;AACzC,SAAO;AAAA;AAAA;AAAA,oBAGW,SAAS;AAAA;AAAA;AAG7B;AAEO,IAAM,aAAa;AAAA,EACxB;AAAA;AAAA;AAAA;AAAA;AAAA,EAKA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAOF;;;ADhCA,IAAM,gBAAgB;AAEtB,IAAM,SAAS,IAAI,YAAY,YAAY,EAAE,MAAM,OAAO,CAAC;AAEpD,IAAM,cAAN,MAAmC;AAAA,EAC/B;AAAA,EACT,YAAY,IAAkB,WAAmB;AAC/C,SAAK,MAAM;AACX,SAAK,IAAI,KAAK,KAAK;AACnB,UAAM,SAAS,IAAI,YAAY,CAAC,QAAQ,SAAS,CAAC,GAAG;AAAA,MACnD,MAAM;AAAA,MACN,aAAa,MAAM,gBAAgB,UAAU;AAAA,IAC/C,CAAC;AACD;AAAA,MACE,KAAK;AAAA,MACL,MAAM;AACJ,cAAM,QAAQ,OAAO,MAAM,KAAK,GAAG,MAAM;AACzC,eAAO,MAAM,KAAK,GAAG;AAKrB,YAAI,MAAO,MAAK,IAAI,KAAK,qBAAqB;AAAA,MAChD;AAAA,MACA,EAAE,MAAM,YAAY;AAAA,IACtB;AACA,WAAO,MAAM,KAAK,GAAG;AAAA,EACvB;AAAA,EAEA,kBAAkB,QAKf;AACD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAS9B;AACA,WAAO,KAAK;AAAA,MACV,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO;AAAA,IACT;AAAA,EACF;AAAA,EACA,gBAAgB,QAIb;AACD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAQ9B;AACA,WAAO,KAAK,IAAI,OAAO,UAAU,OAAO,WAAW,OAAO,CAAC;AAAA,EAC7D;AAAA,EACA,MAAM,OAAO,OAAe,SAAwB,UAAoB;AACtE,UAAM,EAAE,WAAW,IAAI,MAAM,SAAS,CAAC,KAAK,CAAC;AAC7C,QAAI,CAAC,WAAW,QAAQ;AACtB,aAAO,CAAC;AAAA,IACV;AACA,UAAM,aAAa,MAAM,OAAO,KAAK,WAAW,CAAC,CAAC,CAAC;AACnD,UAAM,OAAO,QAAQ;AAErB,QAAI,QAAQ,YAAY;AACtB,YAAMA,QAAO,KAAK,kBAAkB;AAAA,QAClC,UAAU,QAAQ;AAAA,QAClB,YAAY,QAAQ;AAAA,QACpB,WAAW;AAAA,QACX,GAAG,QAAQ;AAAA,MACb,CAAC;AACD,aAAOA,MAAK,IAAI,cAAc;AAAA,IAChC;AAEA,UAAM,OAAO,KAAK,gBAAgB;AAAA,MAChC,UAAU,QAAQ;AAAA,MAClB,WAAW;AAAA,MACX,GAAG,QAAQ;AAAA,IACb,CAAC;AACD,WAAO,KAAK,IAAI,cAAc;AAAA,EAChC;AAAA,EACA,cAAc,QAA8B;AAC1C,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA,IAG9B;AACA,WAAO,KAAK,IAAI,OAAO,QAAQ;AAAA,EACjC;AAAA,EACA,wBAAwB,QAAiD;AACvE,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,IAK9B;AACA,WAAO,KAAK,IAAI,OAAO,UAAU,OAAO,SAAS;AAAA,EACnD;AAAA,EACA,UAAU,QAKP;AACD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,OAQ3B;AACH,WAAO,KAAK;AAAA,MACV,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO,WAAW,KAAK,UAAU,OAAO,QAAQ,IAAI;AAAA,IACtD;AAAA,EACF;AAAA,EACA,UAAU,QAAkD;AAC1D,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA,IAG9B;AACA,WAAO,CAAC,UAAiB;AACvB,WAAK;AAAA,QACH,OAAO;AAAA,QACP,OAAO;AAAA,QACP,MAAM;AAAA,QACN,MAAM,OAAO,KAAK,MAAM,SAAS,CAAC;AAAA,MACpC;AAAA,IACF;AAAA,EACF;AAAA,EACA,OAAO,QAAkD;AACvD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA,IAE9B;AACA,WAAO,KAAK,IAAI,OAAO,UAAU,OAAO,UAAU;AAAA,EACpD;AAAA,EACA,aAAa,UAAkB;AAC7B,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA,IAE9B;AACA,UAAM,MAAM,KAAK,IAAI,QAAQ;AAC7B,WAAO,QAAQ,GAAG;AAAA,EACpB;AAAA,EACA,cAAc,UAAkB,KAAW;AACzC,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAM9B;AACA,UAAM,MAAM,KAAK,IAAI,UAAU,IAAI,YAAY,CAAC;AAChD,WAAO,QAAQ,GAAG;AAAA,EACpB;AAAA,EACA,gBAAgB,UAAkB,YAAkB;AAClD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA,IAI9B;AACA,SAAK,IAAI,WAAW,YAAY,GAAG,QAAQ;AAAA,EAC7C;AAAA,EACA,MAAM,MAAM,UAAkB,QAAgB,YAAmB;AAC/D,QAAI,YAAY;AACd,WAAK,wBAAwB;AAAA,QAC3B,WAAW,WAAW,YAAY;AAAA,QAClC;AAAA,MACF,CAAC;AAAA,IACH,OAAO;AACL,WAAK,cAAc,EAAE,SAAS,CAAC;AAAA,IACjC;AAEA,UAAM,OAAO,KAAK,UAAU;AAAA,MAC1B,YAAY,OAAO;AAAA,MACnB;AAAA,MACA,KAAK,OAAO;AAAA,MACZ,UAAU,OAAO;AAAA,IACnB,CAAC;AACD,UAAM,UAAU,KAAK,UAAU;AAE/B,QAAI,CAAC,SAAS;AACZ;AAAA,IACF;AAEA,UAAM,SAAS,KAAK,UAAU,EAAE,UAAU,YAAY,OAAO,GAAG,CAAC;AAEjE;AAAA,MACE,KAAK;AAAA,MACL,MAAM;AACJ,aAAK,OAAO,EAAE,UAAU,YAAY,OAAO,GAAG,CAAC;AAAA,MACjD;AAAA,MACA,EAAE,MAAM,YAAY;AAAA,IACtB;AAEA,UAAM,YAAY;AAClB,QAAI,QAAiB,CAAC;AACtB,UAAM,QAAQ,MAAM;AAClB,UAAI,CAAC,MAAM,OAAQ;AACnB;AAAA,QACE,KAAK;AAAA,QACL,MAAM;AACJ,mBAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,mBAAO,MAAM,CAAC,CAAC;AAAA,UACjB;AAAA,QACF;AAAA,QACA,EAAE,MAAM,YAAY;AAAA,MACtB;AACA,cAAQ,CAAC;AAAA,IACX;AAEA,qBAAiB,SAAS,OAAO,QAAQ,GAAG;AAC1C,YAAM,KAAK,KAAK;AAChB,UAAI,MAAM,UAAU,UAAW,OAAM;AAAA,IACvC;AAEA,UAAM;AAAA,EACR;AACF;AAEA,SAAS,eAAe,KAA4B;AAClD,MACE,OAAO,QAAQ,YACf,QAAQ,QACR,EAAE,aAAa,QACf,OAAO,IAAI,YAAY,YACvB,EAAE,cAAc,QAChB,OAAO,IAAI,aAAa,YACxB,EAAE,iBAAiB,QACnB,OAAO,IAAI,gBAAgB,YAC3B,EAAE,cAAc,QACf,IAAI,aAAa,QAAQ,OAAO,IAAI,aAAa,UAClD;AACA,UAAM,IAAI,MAAM,8BAA8B;AAAA,EAChD;AACA,SAAO;AAAA,IACL,SAAS,IAAI;AAAA,IACb,UAAU,IAAI;AAAA,IACd,aAAa,IAAI;AAAA,IACjB,UAAU,cAAc,IAAI,QAAQ;AAAA,EACtC;AACF;AAEA,SAAS,cAAc,OAAsD;AAC3E,MAAI,UAAU,KAAM,QAAO;AAC3B,MAAI;AACF,UAAM,SAAkB,KAAK,MAAM,KAAK;AACxC,WAAO,SAAS,MAAM,IAAI,SAAS;AAAA,EACrC,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,SAAS,SAAS,OAAkD;AAClE,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;;;AE5RO,SAAS,WAAW,QAAgB,WAAmB;AAC5D,SAAO,IAAI,YAAY,gBAAgB,MAAM,GAAG,SAAS;AAC3D;",
|
|
6
6
|
"names": ["rows"]
|
|
7
7
|
}
|
|
@@ -23,5 +23,4 @@ export declare class SQLiteStore implements Store {
|
|
|
23
23
|
setSourceExpiry(sourceId: string, expiryDate: Date): void;
|
|
24
24
|
index(sourceId: string, corpus: Corpus, expiryDate?: Date): Promise<void>;
|
|
25
25
|
}
|
|
26
|
-
export declare function vectorToBlob(vector: number[] | Float32Array): Buffer;
|
|
27
26
|
//# sourceMappingURL=sqlite.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sqlite.d.ts","sourceRoot":"","sources":["../../../../src/lib/stores/sqlite/sqlite.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"sqlite.d.ts","sourceRoot":"","sources":["../../../../src/lib/stores/sqlite/sqlite.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,KAAK,EACV,KAAK,EACL,MAAM,EACN,QAAQ,EACR,aAAa,EACb,YAAY,EACZ,KAAK,EACN,MAAM,aAAa,CAAC;AAQrB,qBAAa,WAAY,YAAW,KAAK;;gBAE3B,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM;IA8DzC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ;IAyCtE,SAAS,CAAC,MAAM,EAAE;QAChB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,GAAG,EAAE,MAAM,CAAC;QACZ,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KAChC;IAiBD,SAAS,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,IAKhD,OAAO,KAAK;IAStB,MAAM,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE;IAMvD,YAAY,CAAC,QAAQ,EAAE,MAAM;IAO7B,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI;IAWzC,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI;IAQ5C,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,IAAI;CAuDhE"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@deepagents/retrieval",
|
|
3
|
-
"version": "7.
|
|
3
|
+
"version": "7.7.1",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -41,25 +41,26 @@
|
|
|
41
41
|
},
|
|
42
42
|
"dependencies": {
|
|
43
43
|
"@ai-sdk/tool-web-fetch": "^1.0.16",
|
|
44
|
+
"@huggingface/transformers": "^4.2.0",
|
|
45
|
+
"@langchain/core": "^1.2.3",
|
|
44
46
|
"@langchain/textsplitters": "^1.0.1",
|
|
45
|
-
"@zukhruf/sqlite-drift": "^0.8.22",
|
|
46
|
-
"@zukhruf/sqlite-open": "^0.8.22",
|
|
47
|
-
"@zukhruf/sqlite-transaction": "^0.8.22",
|
|
48
|
-
"execa": "^10.0.0",
|
|
49
47
|
"@linear/sdk": "^88.2.0",
|
|
48
|
+
"@zukhruf/sqlite-drift": "^0.8.30",
|
|
49
|
+
"@zukhruf/sqlite-open": "^0.8.30",
|
|
50
|
+
"@zukhruf/sqlite-transaction": "^0.8.30",
|
|
51
|
+
"@zukhruf/vectors": "^0.8.32",
|
|
52
|
+
"ai": "^7.0.166",
|
|
53
|
+
"dedent": "1.7.2",
|
|
54
|
+
"execa": "^10.0.0",
|
|
50
55
|
"fast-glob": "^3.3.3",
|
|
51
56
|
"ignore": "^7.0.6",
|
|
52
|
-
"unpdf": "^1.6.2",
|
|
53
|
-
"rss-parser": "^3.13.0",
|
|
54
|
-
"@huggingface/transformers": "^4.2.0",
|
|
55
57
|
"langchain": "^1.5.3",
|
|
56
|
-
"
|
|
58
|
+
"rss-parser": "^3.13.0",
|
|
57
59
|
"sqlite-vec": "^0.1.9",
|
|
58
|
-
"
|
|
59
|
-
"@langchain/core": "^1.2.3"
|
|
60
|
+
"unpdf": "^1.6.2"
|
|
60
61
|
},
|
|
61
62
|
"devDependencies": {
|
|
62
63
|
"@sinonjs/fake-timers": "^15.4.0",
|
|
63
|
-
"@zukhruf/fs": "^0.8.
|
|
64
|
+
"@zukhruf/fs": "^0.8.30"
|
|
64
65
|
}
|
|
65
66
|
}
|