@deepagents/retrieval 7.1.0 → 7.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/stores/sqlite/sqlite.d.ts.map +1 -1
- package/dist/lib/stores/sqlite/sqlite.js +5 -2
- package/dist/lib/stores/sqlite/sqlite.js.map +2 -2
- package/dist/lib/stores/sqlite/sqlite.sql.d.ts +1 -1
- package/dist/lib/stores/sqlite/sqlite.sql.d.ts.map +1 -1
- package/dist/lib/stores/sqlite/sqlite.sql.js +1 -0
- package/dist/lib/stores/sqlite/sqlite.sql.js.map +2 -2
- package/package.json +3 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sqlite.d.ts","sourceRoot":"","sources":["../../../../src/lib/stores/sqlite/sqlite.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,KAAK,EACL,MAAM,EACN,QAAQ,EACR,aAAa,EACb,YAAY,EACZ,KAAK,EACN,MAAM,aAAa,CAAC;AAKrB,UAAU,EAAE;IACV,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK;QACxB,GAAG,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,CAAC;QAC7B,GAAG,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,EAAE,CAAC;QAC/B,GAAG,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,CAAC;KAC9B,CAAC;IACF,IAAI,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAC7B;AAED,qBAAa,WAAY,YAAW,KAAK;;gBAE3B,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM;
|
|
1
|
+
{"version":3,"file":"sqlite.d.ts","sourceRoot":"","sources":["../../../../src/lib/stores/sqlite/sqlite.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,KAAK,EACL,MAAM,EACN,QAAQ,EACR,aAAa,EACb,YAAY,EACZ,KAAK,EACN,MAAM,aAAa,CAAC;AAKrB,UAAU,EAAE;IACV,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK;QACxB,GAAG,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,CAAC;QAC7B,GAAG,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,EAAE,CAAC;QAC/B,GAAG,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,CAAC;KAC9B,CAAC;IACF,IAAI,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAC7B;AAED,qBAAa,WAAY,YAAW,KAAK;;gBAE3B,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM;IA4D/B,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ;IAyCtE,SAAS,CAAC,MAAM,EAAE;QAChB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,GAAG,EAAE,MAAM,CAAC;QACZ,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KAChC;IAiBD,SAAS,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,IAKhD,OAAO,KAAK;IAStB,MAAM,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE;IAMvD,YAAY,CAAC,QAAQ,EAAE,MAAM;IAO7B,aAAa,CAAC,QAAQ,EAAE,MAAM;IAW9B,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI;IAQ5C,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,IAAI;CA+ChE;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,YAAY,GAAG,MAAM,CASpE"}
|
|
@@ -10,12 +10,15 @@ class SQLiteStore {
|
|
|
10
10
|
this.#db.exec(compiled({ DIMENSION: dimension }));
|
|
11
11
|
}
|
|
12
12
|
#transaction(callback) {
|
|
13
|
+
this.#db.exec("BEGIN IMMEDIATE");
|
|
13
14
|
try {
|
|
14
|
-
this.#db.exec("BEGIN IMMEDIATE");
|
|
15
15
|
callback();
|
|
16
16
|
this.#db.exec("COMMIT");
|
|
17
17
|
} catch (error) {
|
|
18
|
-
|
|
18
|
+
try {
|
|
19
|
+
this.#db.exec("ROLLBACK");
|
|
20
|
+
} catch {
|
|
21
|
+
}
|
|
19
22
|
throw error;
|
|
20
23
|
}
|
|
21
24
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/lib/stores/sqlite/sqlite.ts"],
|
|
4
|
-
"sourcesContent": ["import dedent from 'dedent';\nimport { template } from 'lodash-es';\n\nimport type {\n Chunk,\n Corpus,\n Embedder,\n SearchOptions,\n SearchResult,\n Store,\n} from '../store.js';\nimport sql from './sqlite.sql.js';\n\nconst DEFAULT_TOP_N = 10;\n\ninterface DB {\n prepare: (sql: string) => {\n run: (...args: any[]) => any;\n all: (...args: any[]) => any[];\n get: (...args: any[]) => any;\n };\n exec: (sql: string) => void;\n}\n\nexport class SQLiteStore implements Store {\n readonly #db: DB;\n constructor(db: DB, dimension: number) {\n this.#db = db;\n const compiled = template(sql);\n this.#db.exec(compiled({ DIMENSION: dimension }));\n }\n\n #transaction(callback: () => void) {\n
|
|
5
|
-
"mappings": "AAAA,OAAO,YAAY;AACnB,SAAS,gBAAgB;AAUzB,OAAO,SAAS;AAEhB,MAAM,gBAAgB;AAWf,MAAM,YAA6B;AAAA,EAC/B;AAAA,EACT,YAAY,IAAQ,WAAmB;AACrC,SAAK,MAAM;AACX,UAAM,WAAW,SAAS,GAAG;AAC7B,SAAK,IAAI,KAAK,SAAS,EAAE,WAAW,UAAU,CAAC,CAAC;AAAA,EAClD;AAAA,EAEA,aAAa,UAAsB;AACjC,
|
|
4
|
+
"sourcesContent": ["import dedent from 'dedent';\nimport { template } from 'lodash-es';\n\nimport type {\n Chunk,\n Corpus,\n Embedder,\n SearchOptions,\n SearchResult,\n Store,\n} from '../store.js';\nimport sql from './sqlite.sql.js';\n\nconst DEFAULT_TOP_N = 10;\n\ninterface DB {\n prepare: (sql: string) => {\n run: (...args: any[]) => any;\n all: (...args: any[]) => any[];\n get: (...args: any[]) => any;\n };\n exec: (sql: string) => void;\n}\n\nexport class SQLiteStore implements Store {\n readonly #db: DB;\n constructor(db: DB, dimension: number) {\n this.#db = db;\n const compiled = template(sql);\n this.#db.exec(compiled({ DIMENSION: dimension }));\n }\n\n #transaction(callback: () => void) {\n this.#db.exec('BEGIN IMMEDIATE');\n try {\n callback();\n this.#db.exec('COMMIT');\n } catch (error) {\n try {\n this.#db.exec('ROLLBACK');\n } catch {\n // Preserve the operation error when SQLite already ended the transaction.\n }\n throw error;\n }\n }\n\n #searchByDocument(inputs: {\n sourceId: string;\n documentId: string;\n embedding: Buffer<ArrayBufferLike>;\n k: number;\n }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT v.content, v.distance, v.document_id, d.metadata\n\t\t\t\tFROM vec_chunks v\n\t\t\t\tJOIN documents d ON d.id = v.document_id\n\t\t\t\tWHERE v.source_id = ?\n\t\t\t\t\tAND v.document_id = ?\n\t\t\t\t\tAND v.embedding MATCH vec_normalize(vec_f32(?))\n\t\t\t\t\tAND v.k = ?\n\t\t\t\tORDER BY v.distance ASC\n\t\t\t`);\n return stmt.all(\n inputs.sourceId,\n inputs.documentId,\n inputs.embedding,\n inputs.k,\n );\n }\n #searchBySource(inputs: {\n sourceId: string;\n embedding: Buffer<ArrayBufferLike>;\n k: number;\n }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT v.content, v.distance, v.document_id, d.metadata\n\t\t\t\tFROM vec_chunks v\n\t\t\t\tJOIN documents d ON d.id = v.document_id\n\t\t\t\tWHERE v.source_id = ?\n\t\t\t\t\tAND v.embedding MATCH vec_normalize(vec_f32(?))\n\t\t\t\t\tAND v.k = ?\n\t\t\t\tORDER BY v.distance ASC\n\t\t\t`);\n return stmt.all(inputs.sourceId, inputs.embedding, inputs.k);\n }\n async search(query: string, options: SearchOptions, embedder: Embedder) {\n const { embeddings } = await embedder([query]);\n if (!embeddings.length) {\n return [];\n }\n const vectorBlob = vectorToBlob(embeddings[0]);\n const topN = options.topN;\n\n if (options.documentId) {\n const rows = this.#searchByDocument({\n sourceId: options.sourceId,\n documentId: options.documentId,\n embedding: vectorBlob,\n k: topN ?? DEFAULT_TOP_N,\n });\n return rows.map(toSearchResult);\n }\n\n const rows = this.#searchBySource({\n sourceId: options.sourceId,\n embedding: vectorBlob,\n k: topN ?? DEFAULT_TOP_N,\n });\n return rows.map(toSearchResult);\n }\n #upsertSource(inputs: { sourceId: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tINSERT INTO sources (source_id) VALUES (?)\n\t\t\t\tON CONFLICT(source_id) DO UPDATE SET updated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now')\n\t\t\t`);\n return stmt.run(inputs.sourceId);\n }\n #upsertSourceWithExpiry(inputs: { sourceId: string; expiresAt: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tINSERT INTO sources (source_id, expires_at) VALUES (?, ?)\n\t\t\t\tON CONFLICT(source_id) DO UPDATE SET\n\t\t\t\t\tupdated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now',\n\t\t\t\t\texpires_at=excluded.expires_at\n\t\t\t`);\n return stmt.run(inputs.sourceId, inputs.expiresAt);\n }\n upsertDoc(inputs: {\n documentId: string;\n sourceId: string;\n cid: string;\n metadata?: Record<string, any>;\n }) {\n const stmt = this.#db.prepare(dedent`\n INSERT INTO documents (id, source_id, cid, metadata)\n VALUES (?, ?, ?, json(?))\n ON CONFLICT(id) DO UPDATE SET\n cid=excluded.cid,\n metadata=COALESCE(excluded.metadata, documents.metadata),\n updated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now')\n WHERE documents.cid != excluded.cid;\n `);\n return stmt.run(\n inputs.documentId,\n inputs.sourceId,\n inputs.cid,\n inputs.metadata ? JSON.stringify(inputs.metadata) : null,\n );\n }\n insertDoc(inputs: { sourceId: string; documentId: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tINSERT INTO vec_chunks (source_id, document_id, content, embedding)\n\t\t\t\tVALUES (?, ?, ?, vec_normalize(vec_f32(?)))\n\t\t\t`);\n return (chunk: Chunk) => {\n stmt.run(\n inputs.sourceId,\n inputs.documentId,\n chunk.content,\n vectorToBlob(chunk.embedding),\n );\n };\n }\n delete(inputs: { sourceId: string; documentId: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tDELETE FROM vec_chunks WHERE source_id = ? AND document_id = ?\n\t\t\t`);\n return stmt.run(inputs.sourceId, inputs.documentId);\n }\n sourceExists(sourceId: string) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT 1 FROM sources WHERE source_id = ? LIMIT 1\n\t\t\t`);\n const row = stmt.get(sourceId);\n return Boolean(row);\n }\n sourceExpired(sourceId: string) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT 1 FROM sources\n\t\t\t\tWHERE source_id = ?\n\t\t\t\t\tAND expires_at IS NOT NULL\n\t\t\t\t\tAND expires_at <= strftime('%Y-%m-%dT%H:%M:%fZ','now')\n\t\t\t\tLIMIT 1\n\t\t\t`);\n const row = stmt.run(sourceId);\n return Boolean(row);\n }\n setSourceExpiry(sourceId: string, expiryDate: Date) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tUPDATE sources\n\t\t\t\tSET expires_at = ?, updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')\n\t\t\t\tWHERE source_id = ?\n\t\t\t`);\n return stmt.run(expiryDate.toISOString(), sourceId);\n }\n async index(sourceId: string, corpus: Corpus, expiryDate?: Date) {\n if (expiryDate) {\n this.#upsertSourceWithExpiry({\n expiresAt: expiryDate.toISOString(),\n sourceId,\n });\n } else {\n this.#upsertSource({ sourceId });\n }\n\n const info = this.upsertDoc({\n documentId: corpus.id,\n sourceId,\n cid: corpus.cid,\n metadata: corpus.metadata,\n });\n const changed = info.changes > 0;\n\n if (!changed) {\n return;\n }\n\n const insert = this.insertDoc({ sourceId, documentId: corpus.id });\n // Delete previous rows once before inserting\n this.#transaction(() => {\n this.delete({ sourceId, documentId: corpus.id });\n });\n\n const batchSize = 32;\n let batch: Chunk[] = [];\n const flush = () => {\n if (!batch.length) return;\n this.#transaction(() => {\n for (let i = 0; i < batch.length; i++) {\n insert(batch[i]);\n }\n });\n batch = [];\n };\n\n for await (const chunk of corpus.chunker()) {\n batch.push(chunk);\n if (batch.length >= batchSize) flush();\n }\n // flush any remaining\n flush();\n }\n}\n\nexport function vectorToBlob(vector: number[] | Float32Array): Buffer {\n if (vector instanceof Float32Array) {\n // Copy into a fresh Buffer to avoid retaining references to a larger batch tensor buffer\n const copied = new Float32Array(vector.length);\n copied.set(vector);\n return Buffer.from(copied.buffer);\n }\n const floatArray = new Float32Array(vector);\n return Buffer.from(floatArray.buffer);\n}\n\nfunction toSearchResult(row: unknown): SearchResult {\n if (\n typeof row !== 'object' ||\n row === null ||\n !('content' in row) ||\n typeof row.content !== 'string' ||\n !('distance' in row) ||\n typeof row.distance !== 'number' ||\n !('document_id' in row) ||\n typeof row.document_id !== 'string' ||\n !('metadata' in row) ||\n (row.metadata !== null && typeof row.metadata !== 'string')\n ) {\n throw new Error('Unexpected vector search row');\n }\n return {\n content: row.content,\n distance: row.distance,\n document_id: row.document_id,\n metadata: parseMetadata(row.metadata),\n };\n}\n\nfunction parseMetadata(value: string | null): Record<string, unknown> | null {\n if (value === null) return null;\n try {\n const parsed: unknown = JSON.parse(value);\n return isRecord(parsed) ? parsed : null;\n } catch {\n return null;\n }\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n return typeof value === 'object' && value !== null && !Array.isArray(value);\n}\n"],
|
|
5
|
+
"mappings": "AAAA,OAAO,YAAY;AACnB,SAAS,gBAAgB;AAUzB,OAAO,SAAS;AAEhB,MAAM,gBAAgB;AAWf,MAAM,YAA6B;AAAA,EAC/B;AAAA,EACT,YAAY,IAAQ,WAAmB;AACrC,SAAK,MAAM;AACX,UAAM,WAAW,SAAS,GAAG;AAC7B,SAAK,IAAI,KAAK,SAAS,EAAE,WAAW,UAAU,CAAC,CAAC;AAAA,EAClD;AAAA,EAEA,aAAa,UAAsB;AACjC,SAAK,IAAI,KAAK,iBAAiB;AAC/B,QAAI;AACF,eAAS;AACT,WAAK,IAAI,KAAK,QAAQ;AAAA,IACxB,SAAS,OAAO;AACd,UAAI;AACF,aAAK,IAAI,KAAK,UAAU;AAAA,MAC1B,QAAQ;AAAA,MAER;AACA,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEA,kBAAkB,QAKf;AACD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAS9B;AACA,WAAO,KAAK;AAAA,MACV,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO;AAAA,IACT;AAAA,EACF;AAAA,EACA,gBAAgB,QAIb;AACD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAQ9B;AACA,WAAO,KAAK,IAAI,OAAO,UAAU,OAAO,WAAW,OAAO,CAAC;AAAA,EAC7D;AAAA,EACA,MAAM,OAAO,OAAe,SAAwB,UAAoB;AACtE,UAAM,EAAE,WAAW,IAAI,MAAM,SAAS,CAAC,KAAK,CAAC;AAC7C,QAAI,CAAC,WAAW,QAAQ;AACtB,aAAO,CAAC;AAAA,IACV;AACA,UAAM,aAAa,aAAa,WAAW,CAAC,CAAC;AAC7C,UAAM,OAAO,QAAQ;AAErB,QAAI,QAAQ,YAAY;AACtB,YAAMA,QAAO,KAAK,kBAAkB;AAAA,QAClC,UAAU,QAAQ;AAAA,QAClB,YAAY,QAAQ;AAAA,QACpB,WAAW;AAAA,QACX,GAAG,QAAQ;AAAA,MACb,CAAC;AACD,aAAOA,MAAK,IAAI,cAAc;AAAA,IAChC;AAEA,UAAM,OAAO,KAAK,gBAAgB;AAAA,MAChC,UAAU,QAAQ;AAAA,MAClB,WAAW;AAAA,MACX,GAAG,QAAQ;AAAA,IACb,CAAC;AACD,WAAO,KAAK,IAAI,cAAc;AAAA,EAChC;AAAA,EACA,cAAc,QAA8B;AAC1C,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA,IAG9B;AACA,WAAO,KAAK,IAAI,OAAO,QAAQ;AAAA,EACjC;AAAA,EACA,wBAAwB,QAAiD;AACvE,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,IAK9B;AACA,WAAO,KAAK,IAAI,OAAO,UAAU,OAAO,SAAS;AAAA,EACnD;AAAA,EACA,UAAU,QAKP;AACD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,OAQ3B;AACH,WAAO,KAAK;AAAA,MACV,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO,WAAW,KAAK,UAAU,OAAO,QAAQ,IAAI;AAAA,IACtD;AAAA,EACF;AAAA,EACA,UAAU,QAAkD;AAC1D,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA,IAG9B;AACA,WAAO,CAAC,UAAiB;AACvB,WAAK;AAAA,QACH,OAAO;AAAA,QACP,OAAO;AAAA,QACP,MAAM;AAAA,QACN,aAAa,MAAM,SAAS;AAAA,MAC9B;AAAA,IACF;AAAA,EACF;AAAA,EACA,OAAO,QAAkD;AACvD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA,IAE9B;AACA,WAAO,KAAK,IAAI,OAAO,UAAU,OAAO,UAAU;AAAA,EACpD;AAAA,EACA,aAAa,UAAkB;AAC7B,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA,IAE9B;AACA,UAAM,MAAM,KAAK,IAAI,QAAQ;AAC7B,WAAO,QAAQ,GAAG;AAAA,EACpB;AAAA,EACA,cAAc,UAAkB;AAC9B,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAM9B;AACA,UAAM,MAAM,KAAK,IAAI,QAAQ;AAC7B,WAAO,QAAQ,GAAG;AAAA,EACpB;AAAA,EACA,gBAAgB,UAAkB,YAAkB;AAClD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA,IAI9B;AACA,WAAO,KAAK,IAAI,WAAW,YAAY,GAAG,QAAQ;AAAA,EACpD;AAAA,EACA,MAAM,MAAM,UAAkB,QAAgB,YAAmB;AAC/D,QAAI,YAAY;AACd,WAAK,wBAAwB;AAAA,QAC3B,WAAW,WAAW,YAAY;AAAA,QAClC;AAAA,MACF,CAAC;AAAA,IACH,OAAO;AACL,WAAK,cAAc,EAAE,SAAS,CAAC;AAAA,IACjC;AAEA,UAAM,OAAO,KAAK,UAAU;AAAA,MAC1B,YAAY,OAAO;AAAA,MACnB;AAAA,MACA,KAAK,OAAO;AAAA,MACZ,UAAU,OAAO;AAAA,IACnB,CAAC;AACD,UAAM,UAAU,KAAK,UAAU;AAE/B,QAAI,CAAC,SAAS;AACZ;AAAA,IACF;AAEA,UAAM,SAAS,KAAK,UAAU,EAAE,UAAU,YAAY,OAAO,GAAG,CAAC;AAEjE,SAAK,aAAa,MAAM;AACtB,WAAK,OAAO,EAAE,UAAU,YAAY,OAAO,GAAG,CAAC;AAAA,IACjD,CAAC;AAED,UAAM,YAAY;AAClB,QAAI,QAAiB,CAAC;AACtB,UAAM,QAAQ,MAAM;AAClB,UAAI,CAAC,MAAM,OAAQ;AACnB,WAAK,aAAa,MAAM;AACtB,iBAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,iBAAO,MAAM,CAAC,CAAC;AAAA,QACjB;AAAA,MACF,CAAC;AACD,cAAQ,CAAC;AAAA,IACX;AAEA,qBAAiB,SAAS,OAAO,QAAQ,GAAG;AAC1C,YAAM,KAAK,KAAK;AAChB,UAAI,MAAM,UAAU,UAAW,OAAM;AAAA,IACvC;AAEA,UAAM;AAAA,EACR;AACF;AAEO,SAAS,aAAa,QAAyC;AACpE,MAAI,kBAAkB,cAAc;AAElC,UAAM,SAAS,IAAI,aAAa,OAAO,MAAM;AAC7C,WAAO,IAAI,MAAM;AACjB,WAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAClC;AACA,QAAM,aAAa,IAAI,aAAa,MAAM;AAC1C,SAAO,OAAO,KAAK,WAAW,MAAM;AACtC;AAEA,SAAS,eAAe,KAA4B;AAClD,MACE,OAAO,QAAQ,YACf,QAAQ,QACR,EAAE,aAAa,QACf,OAAO,IAAI,YAAY,YACvB,EAAE,cAAc,QAChB,OAAO,IAAI,aAAa,YACxB,EAAE,iBAAiB,QACnB,OAAO,IAAI,gBAAgB,YAC3B,EAAE,cAAc,QACf,IAAI,aAAa,QAAQ,OAAO,IAAI,aAAa,UAClD;AACA,UAAM,IAAI,MAAM,8BAA8B;AAAA,EAChD;AACA,SAAO;AAAA,IACL,SAAS,IAAI;AAAA,IACb,UAAU,IAAI;AAAA,IACd,aAAa,IAAI;AAAA,IACjB,UAAU,cAAc,IAAI,QAAQ;AAAA,EACtC;AACF;AAEA,SAAS,cAAc,OAAsD;AAC3E,MAAI,UAAU,KAAM,QAAO;AAC3B,MAAI;AACF,UAAM,SAAkB,KAAK,MAAM,KAAK;AACxC,WAAO,SAAS,MAAM,IAAI,SAAS;AAAA,EACrC,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,SAAS,SAAS,OAAkD;AAClE,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;",
|
|
6
6
|
"names": ["rows"]
|
|
7
7
|
}
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
declare const _default: "-- Embedding store schema\n-- Use <%= DIMENSION %> placeholder replaced at runtime.\n\nPRAGMA page_size = 32768;\nPRAGMA journal_mode = WAL;\nPRAGMA synchronous = NORMAL;\nPRAGMA temp_store = MEMORY;\nPRAGMA foreign_keys = ON;\nPRAGMA cache_size = -131072; -- ~128 MiB page cache for faster repeated reads\nPRAGMA mmap_size = 268435456; -- 256 MiB memory map window to cut syscalls\nPRAGMA wal_autocheckpoint = 1000;\nPRAGMA optimize;\n\nCREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(\n source_id TEXT PARTITION KEY,\n document_id TEXT,\n embedding FLOAT[<%= DIMENSION %>] DISTANCE_METRIC=cosine,\n +content TEXT -- auxiliary payload (not filterable) for reconstruction\n);\n\nCREATE TABLE IF NOT EXISTS sources (\n source_id TEXT PRIMARY KEY,\n updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n expires_at TEXT\n) STRICT;\n\nCREATE TABLE IF NOT EXISTS documents (\n id TEXT PRIMARY KEY,\n source_id TEXT NOT NULL,\n cid TEXT NOT NULL,\n metadata TEXT, -- JSON blob with arbitrary document metadata\n updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n FOREIGN KEY (source_id) REFERENCES sources(source_id) ON DELETE CASCADE\n) STRICT;\n\n-- Indexes to accelerate lookups / maintenance\nCREATE INDEX IF NOT EXISTS idx_documents_source ON documents(source_id);\nCREATE INDEX IF NOT EXISTS idx_documents_source_updated ON documents(source_id, updated_at);\nCREATE INDEX IF NOT EXISTS idx_documents_cid ON documents(cid);\nCREATE INDEX IF NOT EXISTS idx_sources_expires_at ON sources(expires_at) WHERE expires_at IS NOT NULL;\n\n-- emulate cascade for the virtual table\nCREATE TRIGGER IF NOT EXISTS trg_documents_delete_vec\nAFTER DELETE ON documents\nBEGIN\n DELETE FROM vec_chunks\n WHERE source_id = OLD.source_id\n AND document_id = OLD.id;\nEND;\n";
|
|
1
|
+
declare const _default: "-- Embedding store schema\n-- Use <%= DIMENSION %> placeholder replaced at runtime.\n\nPRAGMA busy_timeout = 5000;\nPRAGMA page_size = 32768;\nPRAGMA journal_mode = WAL;\nPRAGMA synchronous = NORMAL;\nPRAGMA temp_store = MEMORY;\nPRAGMA foreign_keys = ON;\nPRAGMA cache_size = -131072; -- ~128 MiB page cache for faster repeated reads\nPRAGMA mmap_size = 268435456; -- 256 MiB memory map window to cut syscalls\nPRAGMA wal_autocheckpoint = 1000;\nPRAGMA optimize;\n\nCREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(\n source_id TEXT PARTITION KEY,\n document_id TEXT,\n embedding FLOAT[<%= DIMENSION %>] DISTANCE_METRIC=cosine,\n +content TEXT -- auxiliary payload (not filterable) for reconstruction\n);\n\nCREATE TABLE IF NOT EXISTS sources (\n source_id TEXT PRIMARY KEY,\n updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n expires_at TEXT\n) STRICT;\n\nCREATE TABLE IF NOT EXISTS documents (\n id TEXT PRIMARY KEY,\n source_id TEXT NOT NULL,\n cid TEXT NOT NULL,\n metadata TEXT, -- JSON blob with arbitrary document metadata\n updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n FOREIGN KEY (source_id) REFERENCES sources(source_id) ON DELETE CASCADE\n) STRICT;\n\n-- Indexes to accelerate lookups / maintenance\nCREATE INDEX IF NOT EXISTS idx_documents_source ON documents(source_id);\nCREATE INDEX IF NOT EXISTS idx_documents_source_updated ON documents(source_id, updated_at);\nCREATE INDEX IF NOT EXISTS idx_documents_cid ON documents(cid);\nCREATE INDEX IF NOT EXISTS idx_sources_expires_at ON sources(expires_at) WHERE expires_at IS NOT NULL;\n\n-- emulate cascade for the virtual table\nCREATE TRIGGER IF NOT EXISTS trg_documents_delete_vec\nAFTER DELETE ON documents\nBEGIN\n DELETE FROM vec_chunks\n WHERE source_id = OLD.source_id\n AND document_id = OLD.id;\nEND;\n";
|
|
2
2
|
export default _default;
|
|
3
3
|
//# sourceMappingURL=sqlite.sql.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sqlite.sql.d.ts","sourceRoot":"","sources":["../../../../src/lib/stores/sqlite/sqlite.sql.ts"],"names":[],"mappings":";AAAA,
|
|
1
|
+
{"version":3,"file":"sqlite.sql.d.ts","sourceRoot":"","sources":["../../../../src/lib/stores/sqlite/sqlite.sql.ts"],"names":[],"mappings":";AAAA,wBAkDE"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/lib/stores/sqlite/sqlite.sql.ts"],
|
|
4
|
-
"sourcesContent": ["export default `-- Embedding store schema\n-- Use <%= DIMENSION %> placeholder replaced at runtime.\n\nPRAGMA page_size = 32768;\nPRAGMA journal_mode = WAL;\nPRAGMA synchronous = NORMAL;\nPRAGMA temp_store = MEMORY;\nPRAGMA foreign_keys = ON;\nPRAGMA cache_size = -131072; -- ~128 MiB page cache for faster repeated reads\nPRAGMA mmap_size = 268435456; -- 256 MiB memory map window to cut syscalls\nPRAGMA wal_autocheckpoint = 1000;\nPRAGMA optimize;\n\nCREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(\n source_id TEXT PARTITION KEY,\n document_id TEXT,\n embedding FLOAT[<%= DIMENSION %>] DISTANCE_METRIC=cosine,\n +content TEXT -- auxiliary payload (not filterable) for reconstruction\n);\n\nCREATE TABLE IF NOT EXISTS sources (\n source_id TEXT PRIMARY KEY,\n updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n expires_at TEXT\n) STRICT;\n\nCREATE TABLE IF NOT EXISTS documents (\n id TEXT PRIMARY KEY,\n source_id TEXT NOT NULL,\n cid TEXT NOT NULL,\n metadata TEXT, -- JSON blob with arbitrary document metadata\n updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n FOREIGN KEY (source_id) REFERENCES sources(source_id) ON DELETE CASCADE\n) STRICT;\n\n-- Indexes to accelerate lookups / maintenance\nCREATE INDEX IF NOT EXISTS idx_documents_source ON documents(source_id);\nCREATE INDEX IF NOT EXISTS idx_documents_source_updated ON documents(source_id, updated_at);\nCREATE INDEX IF NOT EXISTS idx_documents_cid ON documents(cid);\nCREATE INDEX IF NOT EXISTS idx_sources_expires_at ON sources(expires_at) WHERE expires_at IS NOT NULL;\n\n-- emulate cascade for the virtual table\nCREATE TRIGGER IF NOT EXISTS trg_documents_delete_vec\nAFTER DELETE ON documents\nBEGIN\n DELETE FROM vec_chunks\n WHERE source_id = OLD.source_id\n AND document_id = OLD.id;\nEND;\n
|
|
5
|
-
"mappings": "AAAA,IAAO,qBAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;",
|
|
4
|
+
"sourcesContent": ["export default `-- Embedding store schema\n-- Use <%= DIMENSION %> placeholder replaced at runtime.\n\nPRAGMA busy_timeout = 5000;\nPRAGMA page_size = 32768;\nPRAGMA journal_mode = WAL;\nPRAGMA synchronous = NORMAL;\nPRAGMA temp_store = MEMORY;\nPRAGMA foreign_keys = ON;\nPRAGMA cache_size = -131072; -- ~128 MiB page cache for faster repeated reads\nPRAGMA mmap_size = 268435456; -- 256 MiB memory map window to cut syscalls\nPRAGMA wal_autocheckpoint = 1000;\nPRAGMA optimize;\n\nCREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(\n source_id TEXT PARTITION KEY,\n document_id TEXT,\n embedding FLOAT[<%= DIMENSION %>] DISTANCE_METRIC=cosine,\n +content TEXT -- auxiliary payload (not filterable) for reconstruction\n);\n\nCREATE TABLE IF NOT EXISTS sources (\n source_id TEXT PRIMARY KEY,\n updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n expires_at TEXT\n) STRICT;\n\nCREATE TABLE IF NOT EXISTS documents (\n id TEXT PRIMARY KEY,\n source_id TEXT NOT NULL,\n cid TEXT NOT NULL,\n metadata TEXT, -- JSON blob with arbitrary document metadata\n updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n FOREIGN KEY (source_id) REFERENCES sources(source_id) ON DELETE CASCADE\n) STRICT;\n\n-- Indexes to accelerate lookups / maintenance\nCREATE INDEX IF NOT EXISTS idx_documents_source ON documents(source_id);\nCREATE INDEX IF NOT EXISTS idx_documents_source_updated ON documents(source_id, updated_at);\nCREATE INDEX IF NOT EXISTS idx_documents_cid ON documents(cid);\nCREATE INDEX IF NOT EXISTS idx_sources_expires_at ON sources(expires_at) WHERE expires_at IS NOT NULL;\n\n-- emulate cascade for the virtual table\nCREATE TRIGGER IF NOT EXISTS trg_documents_delete_vec\nAFTER DELETE ON documents\nBEGIN\n DELETE FROM vec_chunks\n WHERE source_id = OLD.source_id\n AND document_id = OLD.id;\nEND;\n`;\n"],
|
|
5
|
+
"mappings": "AAAA,IAAO,qBAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@deepagents/retrieval",
|
|
3
|
-
"version": "7.1.
|
|
3
|
+
"version": "7.1.2",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -18,11 +18,13 @@
|
|
|
18
18
|
"exports": {
|
|
19
19
|
"./package.json": "./package.json",
|
|
20
20
|
"./connectors": {
|
|
21
|
+
"@deepagents/source": "./src/lib/connectors/index.ts",
|
|
21
22
|
"types": "./dist/lib/connectors/index.d.ts",
|
|
22
23
|
"import": "./dist/lib/connectors/index.js",
|
|
23
24
|
"default": "./dist/lib/connectors/index.js"
|
|
24
25
|
},
|
|
25
26
|
".": {
|
|
27
|
+
"@deepagents/source": "./src/index.ts",
|
|
26
28
|
"types": "./dist/index.d.ts",
|
|
27
29
|
"import": "./dist/index.js",
|
|
28
30
|
"default": "./dist/index.js"
|