@deepagents/retrieval 7.1.4 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/connectors/rss.d.ts.map +1 -1
- package/dist/lib/connectors/rss.js +17 -25
- package/dist/lib/connectors/rss.js.map +2 -2
- package/dist/lib/ingest.d.ts.map +1 -1
- package/dist/lib/ingest.js +23 -17
- package/dist/lib/ingest.js.map +2 -2
- package/dist/lib/similiarty-search.d.ts.map +1 -1
- package/dist/lib/similiarty-search.js +0 -1
- package/dist/lib/similiarty-search.js.map +2 -2
- package/dist/lib/stores/sqlite/ddl.sqlite.d.ts +4 -0
- package/dist/lib/stores/sqlite/ddl.sqlite.d.ts.map +1 -0
- package/dist/lib/stores/sqlite/{sqlite.sql.js → ddl.sqlite.js} +25 -27
- package/dist/lib/stores/sqlite/ddl.sqlite.js.map +7 -0
- package/dist/lib/stores/sqlite/node-sqlite.d.ts.map +1 -1
- package/dist/lib/stores/sqlite/node-sqlite.js +2 -7
- package/dist/lib/stores/sqlite/node-sqlite.js.map +2 -2
- package/dist/lib/stores/sqlite/open.d.ts +3 -0
- package/dist/lib/stores/sqlite/open.d.ts.map +1 -0
- package/dist/lib/stores/sqlite/open.js +11 -0
- package/dist/lib/stores/sqlite/open.js.map +7 -0
- package/dist/lib/stores/sqlite/sqlite.d.ts +5 -14
- package/dist/lib/stores/sqlite/sqlite.d.ts.map +1 -1
- package/dist/lib/stores/sqlite/sqlite.js +22 -7
- package/dist/lib/stores/sqlite/sqlite.js.map +2 -2
- package/package.json +5 -6
- package/dist/lib/stores/sqlite/bun-sqlite.d.ts +0 -1
- package/dist/lib/stores/sqlite/bun-sqlite.d.ts.map +0 -1
- package/dist/lib/stores/sqlite/bun-sqlite.js +0 -1
- package/dist/lib/stores/sqlite/bun-sqlite.js.map +0 -7
- package/dist/lib/stores/sqlite/sqlite.sql.d.ts +0 -3
- package/dist/lib/stores/sqlite/sqlite.sql.d.ts.map +0 -1
- package/dist/lib/stores/sqlite/sqlite.sql.js.map +0 -7
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rss.d.ts","sourceRoot":"","sources":["../../../src/lib/connectors/rss.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"rss.d.ts","sourceRoot":"","sources":["../../../src/lib/connectors/rss.ts"],"names":[],"mappings":"AAyGA,wBAAgB,GAAG,CACjB,OAAO,EAAE,MAAM,EACf,OAAO,GAAE;IACP,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,OAAO,CAAC;CACxB;;;;;;;EA0DP"}
|
|
@@ -1,5 +1,10 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { experimental_createWebFetchTool } from "@ai-sdk/tool-web-fetch";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
2
3
|
import Parser from "rss-parser";
|
|
4
|
+
const webFetch = experimental_createWebFetchTool({
|
|
5
|
+
// Ingestion stores the complete article in one result.
|
|
6
|
+
maxLength: Number.MAX_SAFE_INTEGER
|
|
7
|
+
});
|
|
3
8
|
const rssParser = new Parser({
|
|
4
9
|
customFields: {
|
|
5
10
|
feed: ["copyright", "language", "managingEditor", "webMaster"],
|
|
@@ -37,37 +42,24 @@ async function parseRSSFeed(feedUrl) {
|
|
|
37
42
|
async function fetchArticleContent(url) {
|
|
38
43
|
try {
|
|
39
44
|
console.log(`Fetching full article content from: ${url}`);
|
|
40
|
-
const
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
},
|
|
48
|
-
signal: AbortSignal.timeout(1e4)
|
|
49
|
-
// 10 second timeout
|
|
50
|
-
});
|
|
51
|
-
if (!response.ok) {
|
|
52
|
-
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
53
|
-
}
|
|
54
|
-
const html = await response.text();
|
|
55
|
-
const { JSDOM } = await import("jsdom");
|
|
56
|
-
const dom = new JSDOM(html, { url });
|
|
57
|
-
const document = dom.window.document;
|
|
58
|
-
const reader = new Readability(document);
|
|
59
|
-
const article = reader.parse();
|
|
60
|
-
if (!article) {
|
|
61
|
-
throw new Error("Readability failed to extract article content");
|
|
45
|
+
const article = await webFetch.execute(
|
|
46
|
+
{ url },
|
|
47
|
+
{ toolCallId: url, messages: [], context: {} }
|
|
48
|
+
);
|
|
49
|
+
assert.ok("type" in article, "Web fetch must return a complete result");
|
|
50
|
+
if (article.type === "error") {
|
|
51
|
+
throw new Error(article.message);
|
|
62
52
|
}
|
|
63
|
-
const fullContent =
|
|
53
|
+
const fullContent = article.title ? `${article.title}
|
|
54
|
+
|
|
55
|
+
${article.content}` : article.content;
|
|
64
56
|
if (fullContent.length < 200) {
|
|
65
57
|
throw new Error(
|
|
66
58
|
`Extracted content too short (${fullContent.length} chars) - likely failed to find main content`
|
|
67
59
|
);
|
|
68
60
|
}
|
|
69
61
|
console.log(
|
|
70
|
-
`\u2713 Successfully extracted article: "${article.title}" (${fullContent.length} chars, ~${Math.round(fullContent.split(" ").length)} words)`
|
|
62
|
+
`\u2713 Successfully extracted article: "${article.title ?? ""}" (${fullContent.length} chars, ~${Math.round(fullContent.split(" ").length)} words)`
|
|
71
63
|
);
|
|
72
64
|
return fullContent;
|
|
73
65
|
} catch (error) {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../src/lib/connectors/rss.ts"],
|
|
4
|
-
"sourcesContent": ["import {
|
|
5
|
-
"mappings": "AAAA,SAAS,
|
|
4
|
+
"sourcesContent": ["import { experimental_createWebFetchTool } from '@ai-sdk/tool-web-fetch';\nimport assert from 'node:assert/strict';\nimport Parser from 'rss-parser';\n\nconst webFetch = experimental_createWebFetchTool({\n // Ingestion stores the complete article in one result.\n maxLength: Number.MAX_SAFE_INTEGER,\n});\n\ntype FeedFields = {\n copyright?: string;\n language?: string;\n managingEditor?: string;\n webMaster?: string;\n lastBuildDate?: string;\n};\n\ntype ItemFields = {\n category?: string;\n creator?: string;\n enclosure?: Parser.Enclosure;\n guid?: string;\n author?: string;\n 'content:encoded'?: string;\n};\n\nconst rssParser = new Parser<FeedFields, ItemFields>({\n customFields: {\n feed: ['copyright', 'language', 'managingEditor', 'webMaster'],\n item: ['category', 'creator', 'enclosure', 'guid'],\n },\n});\n\nasync function parseRSSFeed(feedUrl: string) {\n try {\n console.log(`Fetching RSS feed: ${feedUrl}`);\n const feed = await rssParser.parseURL(feedUrl);\n\n return {\n title: feed.title || '',\n description: feed.description || '',\n link: feed.link || '',\n language: feed.language || 'en',\n lastBuildDate: feed.lastBuildDate || new Date().toISOString(),\n items: feed.items.map((item) => ({\n title: item.title || '',\n description: item.content || item.summary || item.contentSnippet || '',\n link: item.link || '',\n pubDate: item.pubDate || item.isoDate || '',\n author: item.creator || item.author || '',\n categories: Array.isArray(item.categories)\n ? item.categories\n : item.category\n ? [item.category]\n : [],\n guid: item.guid || item.guid || '',\n contentEncoded: item['content:encoded'] || item.content || '',\n })),\n };\n } catch (error) {\n console.error(`Failed to parse RSS feed ${feedUrl}:`, error);\n throw new Error(\n `RSS parsing failed: ${error instanceof Error ? error.message : 'Unknown error'}`,\n );\n }\n}\n\nasync function fetchArticleContent(url: string): Promise<string> {\n try {\n console.log(`Fetching full article content from: ${url}`);\n\n const article = await webFetch.execute(\n { url },\n { toolCallId: url, messages: [], context: {} },\n );\n assert.ok('type' in article, 'Web fetch must return a complete result');\n if (article.type === 'error') {\n throw new Error(article.message);\n }\n\n // Combine title and content for comprehensive text\n const fullContent = article.title\n ? `${article.title}\\n\\n${article.content}`\n : article.content;\n\n // Ensure we have meaningful content (at least 200 characters for full articles)\n if (fullContent.length < 200) {\n throw new Error(\n `Extracted content too short (${fullContent.length} chars) - likely failed to find main content`,\n );\n }\n\n console.log(\n `\u2713 Successfully extracted article: \"${article.title ?? ''}\" (${fullContent.length} chars, ~${Math.round(fullContent.split(' ').length)} words)`,\n );\n\n return fullContent;\n } catch (error) {\n console.warn(\n `Failed to fetch article content from ${url}:`,\n error instanceof Error ? error.message : error,\n );\n return ''; // Return empty string on failure rather than throwing\n }\n}\nexport function rss(\n feedUrl: string,\n options: {\n maxItems?: number;\n fetchFullArticles?: boolean;\n } = {},\n) {\n const sourceId = `rss:${feedUrl}`;\n const { maxItems = 50, fetchFullArticles = false } = options;\n\n return {\n sourceId,\n instructions: `You answer questions about articles and content from the RSS feed: ${feedUrl}.\n Always cite the article title and link when referencing specific content.\n The feed contains recent articles, blog posts, and news items.\n When referencing content, include the publication date and author when available.\n ${fetchFullArticles ? 'Full article content has been extracted from the original links for comprehensive analysis.' : 'Content includes RSS summaries and descriptions.'}`,\n sources: async function* () {\n const feed = await parseRSSFeed(feedUrl);\n // Add feed summary source\n const feedSummary = `RSS Feed: ${feed.title}\nDescription: ${feed.description}\nWebsite: ${feed.link}\nLanguage: ${feed.language}\nLast Updated: ${feed.lastBuildDate}\nTotal Items: ${feed.items.length}\n\nThis feed provides: ${feed.description}`;\n yield {\n id: 'feed-info',\n content: async () => feedSummary,\n };\n\n // Individual article sources (limit to maxItems)\n const itemsToProcess = feed.items.slice(0, maxItems);\n for (const item of itemsToProcess) {\n const documentId =\n item.guid || item.link || `${item.title}-${item.pubDate}`;\n yield {\n id: documentId,\n content: async () => {\n // Try full article fetch if enabled, fallback to RSS content\n let articleContent = item.contentEncoded || item.description;\n if (fetchFullArticles && item.link) {\n const fullContent = await fetchArticleContent(item.link);\n if (fullContent && fullContent.length > articleContent.length) {\n articleContent = fullContent;\n }\n }\n return `Title: ${item.title}\nAuthor: ${item.author}\nPublished: ${item.pubDate}\nCategories: ${item.categories.join(', ')}\nLink: ${item.link}\n${fetchFullArticles ? 'Full Article Content:' : 'Content:'}\n${articleContent}\n\nSummary: ${item.title} - ${item.description}`;\n },\n };\n }\n },\n };\n}\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,uCAAuC;AAChD,OAAO,YAAY;AACnB,OAAO,YAAY;AAEnB,MAAM,WAAW,gCAAgC;AAAA;AAAA,EAE/C,WAAW,OAAO;AACpB,CAAC;AAmBD,MAAM,YAAY,IAAI,OAA+B;AAAA,EACnD,cAAc;AAAA,IACZ,MAAM,CAAC,aAAa,YAAY,kBAAkB,WAAW;AAAA,IAC7D,MAAM,CAAC,YAAY,WAAW,aAAa,MAAM;AAAA,EACnD;AACF,CAAC;AAED,eAAe,aAAa,SAAiB;AAC3C,MAAI;AACF,YAAQ,IAAI,sBAAsB,OAAO,EAAE;AAC3C,UAAM,OAAO,MAAM,UAAU,SAAS,OAAO;AAE7C,WAAO;AAAA,MACL,OAAO,KAAK,SAAS;AAAA,MACrB,aAAa,KAAK,eAAe;AAAA,MACjC,MAAM,KAAK,QAAQ;AAAA,MACnB,UAAU,KAAK,YAAY;AAAA,MAC3B,eAAe,KAAK,kBAAiB,oBAAI,KAAK,GAAE,YAAY;AAAA,MAC5D,OAAO,KAAK,MAAM,IAAI,CAAC,UAAU;AAAA,QAC/B,OAAO,KAAK,SAAS;AAAA,QACrB,aAAa,KAAK,WAAW,KAAK,WAAW,KAAK,kBAAkB;AAAA,QACpE,MAAM,KAAK,QAAQ;AAAA,QACnB,SAAS,KAAK,WAAW,KAAK,WAAW;AAAA,QACzC,QAAQ,KAAK,WAAW,KAAK,UAAU;AAAA,QACvC,YAAY,MAAM,QAAQ,KAAK,UAAU,IACrC,KAAK,aACL,KAAK,WACH,CAAC,KAAK,QAAQ,IACd,CAAC;AAAA,QACP,MAAM,KAAK,QAAQ,KAAK,QAAQ;AAAA,QAChC,gBAAgB,KAAK,iBAAiB,KAAK,KAAK,WAAW;AAAA,MAC7D,EAAE;AAAA,IACJ;AAAA,EACF,SAAS,OAAO;AACd,YAAQ,MAAM,4BAA4B,OAAO,KAAK,KAAK;AAC3D,UAAM,IAAI;AAAA,MACR,uBAAuB,iBAAiB,QAAQ,MAAM,UAAU,eAAe;AAAA,IACjF;AAAA,EACF;AACF;AAEA,eAAe,oBAAoB,KAA8B;AAC/D,MAAI;AACF,YAAQ,IAAI,uCAAuC,GAAG,EAAE;AAExD,UAAM,UAAU,MAAM,SAAS;AAAA,MAC7B,EAAE,IAAI;AAAA,MACN,EAAE,YAAY,KAAK,UAAU,CAAC,GAAG,SAAS,CAAC,EAAE;AAAA,IAC/C;AACA,WAAO,GAAG,UAAU,SAAS,yCAAyC;AACtE,QAAI,QAAQ,SAAS,SAAS;AAC5B,YAAM,IAAI,MAAM,QAAQ,OAAO;AAAA,IACjC;AAGA,UAAM,cAAc,QAAQ,QACxB,GAAG,QAAQ,KAAK;AAAA;AAAA,EAAO,QAAQ,OAAO,KACtC,QAAQ;AAGZ,QAAI,YAAY,SAAS,KAAK;AAC5B,YAAM,IAAI;AAAA,QACR,gCAAgC,YAAY,MAAM;AAAA,MACpD;AAAA,IACF;AAEA,YAAQ;AAAA,MACN,2CAAsC,QAAQ,SAAS,EAAE,MAAM,YAAY,MAAM,YAAY,KAAK,MAAM,YAAY,MAAM,GAAG,EAAE,MAAM,CAAC;AAAA,IACxI;AAEA,WAAO;AAAA,EACT,SAAS,OAAO;AACd,YAAQ;AAAA,MACN,wCAAwC,GAAG;AAAA,MAC3C,iBAAiB,QAAQ,MAAM,UAAU;AAAA,IAC3C;AACA,WAAO;AAAA,EACT;AACF;AACO,SAAS,IACd,SACA,UAGI,CAAC,GACL;AACA,QAAM,WAAW,OAAO,OAAO;AAC/B,QAAM,EAAE,WAAW,IAAI,oBAAoB,MAAM,IAAI;AAErD,SAAO;AAAA,IACL;AAAA,IACA,cAAc,sEAAsE,OAAO;AAAA;AAAA;AAAA;AAAA,QAIvF,oBAAoB,gGAAgG,kDAAkD;AAAA,IAC1K,SAAS,mBAAmB;AAC1B,YAAM,OAAO,MAAM,aAAa,OAAO;AAEvC,YAAM,cAAc,aAAa,KAAK,KAAK;AAAA,eAClC,KAAK,WAAW;AAAA,WACpB,KAAK,IAAI;AAAA,YACR,KAAK,QAAQ;AAAA,gBACT,KAAK,aAAa;AAAA,eACnB,KAAK,MAAM,MAAM;AAAA;AAAA,sBAEV,KAAK,WAAW;AAChC,YAAM;AAAA,QACJ,IAAI;AAAA,QACJ,SAAS,YAAY;AAAA,MACvB;AAGA,YAAM,iBAAiB,KAAK,MAAM,MAAM,GAAG,QAAQ;AACnD,iBAAW,QAAQ,gBAAgB;AACjC,cAAM,aACJ,KAAK,QAAQ,KAAK,QAAQ,GAAG,KAAK,KAAK,IAAI,KAAK,OAAO;AACzD,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,SAAS,YAAY;AAEnB,gBAAI,iBAAiB,KAAK,kBAAkB,KAAK;AACjD,gBAAI,qBAAqB,KAAK,MAAM;AAClC,oBAAM,cAAc,MAAM,oBAAoB,KAAK,IAAI;AACvD,kBAAI,eAAe,YAAY,SAAS,eAAe,QAAQ;AAC7D,iCAAiB;AAAA,cACnB;AAAA,YACF;AACA,mBAAO,UAAU,KAAK,KAAK;AAAA,UAC7B,KAAK,MAAM;AAAA,aACR,KAAK,OAAO;AAAA,cACX,KAAK,WAAW,KAAK,IAAI,CAAC;AAAA,QAChC,KAAK,IAAI;AAAA,EACf,oBAAoB,0BAA0B,UAAU;AAAA,EACxD,cAAc;AAAA;AAAA,WAEL,KAAK,KAAK,MAAM,KAAK,WAAW;AAAA,UACjC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
package/dist/lib/ingest.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../../src/lib/ingest.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAE9C,OAAO,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAEzD,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,SAAS,CAAC;IACrB,KAAK,EAAE,KAAK,CAAC;IACb,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;CACpB;AAED,wBAAsB,MAAM,CAC1B,MAAM,EAAE,eAAe,EACvB,QAAQ,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,IAAI,
|
|
1
|
+
{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../../src/lib/ingest.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAE9C,OAAO,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAEzD,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,SAAS,CAAC;IACrB,KAAK,EAAE,KAAK,CAAC;IACb,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;CACpB;AAED,wBAAsB,MAAM,CAC1B,MAAM,EAAE,eAAe,EACvB,QAAQ,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,IAAI,iBA2ChC;AAOD,MAAM,MAAM,aAAa,GAAG;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;IAC/B,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AA+EF,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,sBAAsB,EAAE,CAAC,CAQnC;AAED,wBAAsB,eAAe,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,qBAGhE"}
|
package/dist/lib/ingest.js
CHANGED
|
@@ -7,31 +7,37 @@ async function ingest(config, callback) {
|
|
|
7
7
|
const splitter = config.splitter ?? split;
|
|
8
8
|
const embedder = config.embedder;
|
|
9
9
|
const corpuses = config.connector.sources();
|
|
10
|
+
const { expiresAfter } = config.connector;
|
|
11
|
+
const expiryDate = expiresAfter === void 0 ? void 0 : new Date(Date.now() + expiresAfter);
|
|
10
12
|
for await (const it of corpuses) {
|
|
11
13
|
callback?.(it.id);
|
|
12
14
|
const content = await it.content();
|
|
13
15
|
if (!content.trim()) {
|
|
14
16
|
continue;
|
|
15
17
|
}
|
|
16
|
-
await config.store.index(
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
const
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
18
|
+
await config.store.index(
|
|
19
|
+
config.connector.sourceId,
|
|
20
|
+
{
|
|
21
|
+
id: it.id,
|
|
22
|
+
cid: cid(content),
|
|
23
|
+
metadata: it.metadata,
|
|
24
|
+
chunker: async function* () {
|
|
25
|
+
const values = await splitter(it.id, content);
|
|
26
|
+
const batchSize = 40;
|
|
27
|
+
for (let i = 0; i < values.length; i += batchSize) {
|
|
28
|
+
const batch = values.slice(i, i + batchSize);
|
|
29
|
+
const { embeddings } = await embedder(batch);
|
|
30
|
+
for (let j = 0; j < embeddings.length; j++) {
|
|
31
|
+
yield {
|
|
32
|
+
content: batch[j],
|
|
33
|
+
embedding: embeddings[j]
|
|
34
|
+
};
|
|
35
|
+
}
|
|
31
36
|
}
|
|
32
37
|
}
|
|
33
|
-
}
|
|
34
|
-
|
|
38
|
+
},
|
|
39
|
+
expiryDate
|
|
40
|
+
);
|
|
35
41
|
}
|
|
36
42
|
}
|
|
37
43
|
function split(id, content) {
|
package/dist/lib/ingest.js.map
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/lib/ingest.ts"],
|
|
4
|
-
"sourcesContent": ["import {\n MarkdownTextSplitter,\n RecursiveCharacterTextSplitter,\n} from '@langchain/textsplitters';\n\nimport type { Connector } from './connectors/connector.js';\nimport type { Splitter } from './pipeline.js';\nimport { cid } from './stores/cid.js';\nimport type { Embedder, Store } from './stores/store.js';\n\nexport interface IngestionConfig {\n connector: Connector;\n store: Store;\n splitter?: Splitter;\n embedder: Embedder;\n}\n\nexport async function ingest(\n config: IngestionConfig,\n callback?: (it: string) => void,\n) {\n const splitter = config.splitter ?? split;\n const embedder = config.embedder;\n const corpuses = config.connector.sources();\n\n for await (const it of corpuses) {\n callback?.(it.id);\n const content = await it.content();\n if (!content.trim()) {\n // skip empty files\n continue;\n }\n await config.store.index(config.connector.sourceId
|
|
5
|
-
"mappings": "AAAA;AAAA,EACE;AAAA,EACA;AAAA,OACK;AAIP,SAAS,WAAW;AAUpB,eAAsB,OACpB,QACA,UACA;AACA,QAAM,WAAW,OAAO,YAAY;AACpC,QAAM,WAAW,OAAO;AACxB,QAAM,WAAW,OAAO,UAAU,QAAQ;
|
|
4
|
+
"sourcesContent": ["import {\n MarkdownTextSplitter,\n RecursiveCharacterTextSplitter,\n} from '@langchain/textsplitters';\n\nimport type { Connector } from './connectors/connector.js';\nimport type { Splitter } from './pipeline.js';\nimport { cid } from './stores/cid.js';\nimport type { Embedder, Store } from './stores/store.js';\n\nexport interface IngestionConfig {\n connector: Connector;\n store: Store;\n splitter?: Splitter;\n embedder: Embedder;\n}\n\nexport async function ingest(\n config: IngestionConfig,\n callback?: (it: string) => void,\n) {\n const splitter = config.splitter ?? split;\n const embedder = config.embedder;\n const corpuses = config.connector.sources();\n const { expiresAfter } = config.connector;\n const expiryDate =\n expiresAfter === undefined\n ? undefined\n : new Date(Date.now() + expiresAfter);\n\n for await (const it of corpuses) {\n callback?.(it.id);\n const content = await it.content();\n if (!content.trim()) {\n // skip empty files\n continue;\n }\n await config.store.index(\n config.connector.sourceId,\n {\n id: it.id,\n cid: cid(content),\n metadata: it.metadata,\n chunker: async function* () {\n // Embed in small batches to control memory usage\n const values = await splitter(it.id, content);\n const batchSize = 40;\n for (let i = 0; i < values.length; i += batchSize) {\n const batch = values.slice(i, i + batchSize);\n const { embeddings } = await embedder(batch);\n for (let j = 0; j < embeddings.length; j++) {\n yield {\n content: batch[j],\n embedding: embeddings[j],\n };\n }\n }\n },\n },\n expiryDate,\n );\n }\n}\n\nfunction split(id: string, content: string) {\n const splitter = new MarkdownTextSplitter();\n return splitter.splitText(content);\n}\n\nexport type ChunkPosition = {\n startLine: number;\n startColumn: number;\n endLine: number;\n endColumn: number;\n};\n\nexport type SplitChunkWithPosition = {\n content: string;\n position: ChunkPosition | null;\n index: number;\n};\n\nfunction normalizeNewlines(value: string) {\n return value.replace(/\\r\\n/g, '\\n');\n}\n\nfunction computePositions(\n originalContent: string,\n chunks: string[],\n): Array<ChunkPosition | null> {\n if (!chunks.length) {\n return [];\n }\n\n const normalizedContent = normalizeNewlines(originalContent);\n const positions: Array<ChunkPosition | null> = [];\n let searchOffset = 0;\n\n for (const chunk of chunks) {\n const normalizedChunk = normalizeNewlines(chunk);\n const trimmedChunk = normalizedChunk.trim();\n\n const seek = (needle: string, fromIndex: number) =>\n needle ? normalizedContent.indexOf(needle, fromIndex) : -1;\n\n let matchIndex = seek(normalizedChunk, searchOffset);\n let matchValue = normalizedChunk;\n\n if (matchIndex === -1 && trimmedChunk) {\n matchIndex = seek(trimmedChunk, searchOffset);\n matchValue = trimmedChunk;\n }\n\n if (matchIndex === -1) {\n matchIndex = seek(normalizedChunk, 0);\n matchValue = normalizedChunk;\n }\n\n if (matchIndex === -1 && trimmedChunk) {\n matchIndex = seek(trimmedChunk, 0);\n matchValue = trimmedChunk;\n }\n\n if (matchIndex === -1) {\n positions.push(null);\n continue;\n }\n\n const before = normalizedContent.slice(0, matchIndex);\n const beforeLines = before.split('\\n');\n const startLine = beforeLines.length;\n const startColumn = beforeLines[beforeLines.length - 1].length + 1;\n\n const lines = matchValue.split('\\n');\n const endLine = startLine + lines.length - 1;\n const endColumn =\n lines.length === 1\n ? startColumn + lines[0].length\n : lines[lines.length - 1].length + 1;\n\n positions.push({ startLine, startColumn, endLine, endColumn });\n searchOffset = matchIndex + matchValue.length;\n }\n\n return positions;\n}\n\nfunction buildChunksWithPositions(\n originalContent: string,\n chunks: string[],\n): SplitChunkWithPosition[] {\n const positions = computePositions(originalContent, chunks);\n return chunks.map((content, index) => ({\n content,\n index,\n position: positions[index] ?? null,\n }));\n}\n\nexport async function splitTypeScriptWithPositions(\n id: string,\n content: string,\n): Promise<SplitChunkWithPosition[]> {\n const splitter = RecursiveCharacterTextSplitter.fromLanguage('js', {\n chunkSize: 512,\n chunkOverlap: 100,\n });\n const docs = await splitter.createDocuments([content]);\n const chunks = docs.map((d) => d.pageContent);\n return buildChunksWithPositions(content, chunks);\n}\n\nexport async function splitTypeScript(id: string, content: string) {\n const chunks = await splitTypeScriptWithPositions(id, content);\n return chunks.map((chunk) => chunk.content);\n}\n"],
|
|
5
|
+
"mappings": "AAAA;AAAA,EACE;AAAA,EACA;AAAA,OACK;AAIP,SAAS,WAAW;AAUpB,eAAsB,OACpB,QACA,UACA;AACA,QAAM,WAAW,OAAO,YAAY;AACpC,QAAM,WAAW,OAAO;AACxB,QAAM,WAAW,OAAO,UAAU,QAAQ;AAC1C,QAAM,EAAE,aAAa,IAAI,OAAO;AAChC,QAAM,aACJ,iBAAiB,SACb,SACA,IAAI,KAAK,KAAK,IAAI,IAAI,YAAY;AAExC,mBAAiB,MAAM,UAAU;AAC/B,eAAW,GAAG,EAAE;AAChB,UAAM,UAAU,MAAM,GAAG,QAAQ;AACjC,QAAI,CAAC,QAAQ,KAAK,GAAG;AAEnB;AAAA,IACF;AACA,UAAM,OAAO,MAAM;AAAA,MACjB,OAAO,UAAU;AAAA,MACjB;AAAA,QACE,IAAI,GAAG;AAAA,QACP,KAAK,IAAI,OAAO;AAAA,QAChB,UAAU,GAAG;AAAA,QACb,SAAS,mBAAmB;AAE1B,gBAAM,SAAS,MAAM,SAAS,GAAG,IAAI,OAAO;AAC5C,gBAAM,YAAY;AAClB,mBAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,WAAW;AACjD,kBAAM,QAAQ,OAAO,MAAM,GAAG,IAAI,SAAS;AAC3C,kBAAM,EAAE,WAAW,IAAI,MAAM,SAAS,KAAK;AAC3C,qBAAS,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK;AAC1C,oBAAM;AAAA,gBACJ,SAAS,MAAM,CAAC;AAAA,gBAChB,WAAW,WAAW,CAAC;AAAA,cACzB;AAAA,YACF;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,MAAM,IAAY,SAAiB;AAC1C,QAAM,WAAW,IAAI,qBAAqB;AAC1C,SAAO,SAAS,UAAU,OAAO;AACnC;AAeA,SAAS,kBAAkB,OAAe;AACxC,SAAO,MAAM,QAAQ,SAAS,IAAI;AACpC;AAEA,SAAS,iBACP,iBACA,QAC6B;AAC7B,MAAI,CAAC,OAAO,QAAQ;AAClB,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,oBAAoB,kBAAkB,eAAe;AAC3D,QAAM,YAAyC,CAAC;AAChD,MAAI,eAAe;AAEnB,aAAW,SAAS,QAAQ;AAC1B,UAAM,kBAAkB,kBAAkB,KAAK;AAC/C,UAAM,eAAe,gBAAgB,KAAK;AAE1C,UAAM,OAAO,CAAC,QAAgB,cAC5B,SAAS,kBAAkB,QAAQ,QAAQ,SAAS,IAAI;AAE1D,QAAI,aAAa,KAAK,iBAAiB,YAAY;AACnD,QAAI,aAAa;AAEjB,QAAI,eAAe,MAAM,cAAc;AACrC,mBAAa,KAAK,cAAc,YAAY;AAC5C,mBAAa;AAAA,IACf;AAEA,QAAI,eAAe,IAAI;AACrB,mBAAa,KAAK,iBAAiB,CAAC;AACpC,mBAAa;AAAA,IACf;AAEA,QAAI,eAAe,MAAM,cAAc;AACrC,mBAAa,KAAK,cAAc,CAAC;AACjC,mBAAa;AAAA,IACf;AAEA,QAAI,eAAe,IAAI;AACrB,gBAAU,KAAK,IAAI;AACnB;AAAA,IACF;AAEA,UAAM,SAAS,kBAAkB,MAAM,GAAG,UAAU;AACpD,UAAM,cAAc,OAAO,MAAM,IAAI;AACrC,UAAM,YAAY,YAAY;AAC9B,UAAM,cAAc,YAAY,YAAY,SAAS,CAAC,EAAE,SAAS;AAEjE,UAAM,QAAQ,WAAW,MAAM,IAAI;AACnC,UAAM,UAAU,YAAY,MAAM,SAAS;AAC3C,UAAM,YACJ,MAAM,WAAW,IACb,cAAc,MAAM,CAAC,EAAE,SACvB,MAAM,MAAM,SAAS,CAAC,EAAE,SAAS;AAEvC,cAAU,KAAK,EAAE,WAAW,aAAa,SAAS,UAAU,CAAC;AAC7D,mBAAe,aAAa,WAAW;AAAA,EACzC;AAEA,SAAO;AACT;AAEA,SAAS,yBACP,iBACA,QAC0B;AAC1B,QAAM,YAAY,iBAAiB,iBAAiB,MAAM;AAC1D,SAAO,OAAO,IAAI,CAAC,SAAS,WAAW;AAAA,IACrC;AAAA,IACA;AAAA,IACA,UAAU,UAAU,KAAK,KAAK;AAAA,EAChC,EAAE;AACJ;AAEA,eAAsB,6BACpB,IACA,SACmC;AACnC,QAAM,WAAW,+BAA+B,aAAa,MAAM;AAAA,IACjE,WAAW;AAAA,IACX,cAAc;AAAA,EAChB,CAAC;AACD,QAAM,OAAO,MAAM,SAAS,gBAAgB,CAAC,OAAO,CAAC;AACrD,QAAM,SAAS,KAAK,IAAI,CAAC,MAAM,EAAE,WAAW;AAC5C,SAAO,yBAAyB,SAAS,MAAM;AACjD;AAEA,eAAsB,gBAAgB,IAAY,SAAiB;AACjE,QAAM,SAAS,MAAM,6BAA6B,IAAI,OAAO;AAC7D,SAAO,OAAO,IAAI,CAAC,UAAU,MAAM,OAAO;AAC5C;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"similiarty-search.d.ts","sourceRoot":"","sources":["../../src/lib/similiarty-search.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,eAAe,EAAU,MAAM,aAAa,CAAC;AAE3D,wBAAsB,gBAAgB,CACpC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,IAAI,CAAC,eAAe,EAAE,UAAU,CAAC;;;;;;
|
|
1
|
+
{"version":3,"file":"similiarty-search.d.ts","sourceRoot":"","sources":["../../src/lib/similiarty-search.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,eAAe,EAAU,MAAM,aAAa,CAAC;AAE3D,wBAAsB,gBAAgB,CACpC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,IAAI,CAAC,eAAe,EAAE,UAAU,CAAC;;;;;;KA4C1C"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/lib/similiarty-search.ts"],
|
|
4
|
-
"sourcesContent": ["import { embedMany } from 'ai';\n\nimport { type IngestionConfig, ingest } from './ingest.js';\n\nexport async function similaritySearch(\n query: string,\n config: Omit<IngestionConfig, 'splitter'>,\n) {\n // Ingest if needed, then perform vector similarity search via the configured store\n const mode = config.connector.ingestWhen ?? 'contentChanged';\n let shouldIngest = true;\n\n if (mode === 'never') {\n if (await config.store.sourceExists(config.connector.sourceId)) {\n console.log(\n `Skipping ingestion for source ${config.connector.sourceId} (ingestWhen=never and source exists)`,\n );\n shouldIngest = false;\n }\n } else if (mode === 'expired') {\n const sourceExists = await config.store.sourceExists(\n config.connector.sourceId,\n );\n if (\n sourceExists &&\n !(await config.store.sourceExpired(config.connector.sourceId))\n ) {\n console.log(\n `Skipping ingestion for source ${config.connector.sourceId} (ingestWhen=expired and source not expired)`,\n );\n shouldIngest = false;\n }\n }\n\n
|
|
5
|
-
"mappings": "AAAA,SAAS,iBAAiB;AAE1B,SAA+B,cAAc;AAE7C,eAAsB,iBACpB,OACA,QACA;AAEA,QAAM,OAAO,OAAO,UAAU,cAAc;AAC5C,MAAI,eAAe;AAEnB,MAAI,SAAS,SAAS;AACpB,QAAI,MAAM,OAAO,MAAM,aAAa,OAAO,UAAU,QAAQ,GAAG;AAC9D,cAAQ;AAAA,QACN,iCAAiC,OAAO,UAAU,QAAQ;AAAA,MAC5D;AACA,qBAAe;AAAA,IACjB;AAAA,EACF,WAAW,SAAS,WAAW;AAC7B,UAAM,eAAe,MAAM,OAAO,MAAM;AAAA,MACtC,OAAO,UAAU;AAAA,IACnB;AACA,QACE,gBACA,CAAE,MAAM,OAAO,MAAM,cAAc,OAAO,UAAU,QAAQ,GAC5D;AACA,cAAQ;AAAA,QACN,iCAAiC,OAAO,UAAU,QAAQ;AAAA,MAC5D;AACA,qBAAe;AAAA,IACjB;AAAA,EACF;
|
|
4
|
+
"sourcesContent": ["import { embedMany } from 'ai';\n\nimport { type IngestionConfig, ingest } from './ingest.js';\n\nexport async function similaritySearch(\n query: string,\n config: Omit<IngestionConfig, 'splitter'>,\n) {\n // Ingest if needed, then perform vector similarity search via the configured store\n const mode = config.connector.ingestWhen ?? 'contentChanged';\n let shouldIngest = true;\n\n if (mode === 'never') {\n if (await config.store.sourceExists(config.connector.sourceId)) {\n console.log(\n `Skipping ingestion for source ${config.connector.sourceId} (ingestWhen=never and source exists)`,\n );\n shouldIngest = false;\n }\n } else if (mode === 'expired') {\n const sourceExists = await config.store.sourceExists(\n config.connector.sourceId,\n );\n if (\n sourceExists &&\n !(await config.store.sourceExpired(config.connector.sourceId))\n ) {\n console.log(\n `Skipping ingestion for source ${config.connector.sourceId} (ingestWhen=expired and source not expired)`,\n );\n shouldIngest = false;\n }\n }\n\n if (shouldIngest) {\n await ingest(config);\n }\n return config.store\n .search(\n query,\n { sourceId: config.connector.sourceId, topN: 50 },\n config.embedder,\n )\n .then((results) =>\n results.map((it) => ({\n ...it,\n similarity: 1 - it.distance,\n distance: it.distance,\n })),\n );\n}\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,iBAAiB;AAE1B,SAA+B,cAAc;AAE7C,eAAsB,iBACpB,OACA,QACA;AAEA,QAAM,OAAO,OAAO,UAAU,cAAc;AAC5C,MAAI,eAAe;AAEnB,MAAI,SAAS,SAAS;AACpB,QAAI,MAAM,OAAO,MAAM,aAAa,OAAO,UAAU,QAAQ,GAAG;AAC9D,cAAQ;AAAA,QACN,iCAAiC,OAAO,UAAU,QAAQ;AAAA,MAC5D;AACA,qBAAe;AAAA,IACjB;AAAA,EACF,WAAW,SAAS,WAAW;AAC7B,UAAM,eAAe,MAAM,OAAO,MAAM;AAAA,MACtC,OAAO,UAAU;AAAA,IACnB;AACA,QACE,gBACA,CAAE,MAAM,OAAO,MAAM,cAAc,OAAO,UAAU,QAAQ,GAC5D;AACA,cAAQ;AAAA,QACN,iCAAiC,OAAO,UAAU,QAAQ;AAAA,MAC5D;AACA,qBAAe;AAAA,IACjB;AAAA,EACF;AAEA,MAAI,cAAc;AAChB,UAAM,OAAO,MAAM;AAAA,EACrB;AACA,SAAO,OAAO,MACX;AAAA,IACC;AAAA,IACA,EAAE,UAAU,OAAO,UAAU,UAAU,MAAM,GAAG;AAAA,IAChD,OAAO;AAAA,EACT,EACC;AAAA,IAAK,CAAC,YACL,QAAQ,IAAI,CAAC,QAAQ;AAAA,MACnB,GAAG;AAAA,MACH,YAAY,IAAI,GAAG;AAAA,MACnB,UAAU,GAAG;AAAA,IACf,EAAE;AAAA,EACJ;AACJ;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export declare const SETUP = "\nPRAGMA busy_timeout = 5000;\nPRAGMA page_size = 32768;\nPRAGMA journal_mode = WAL;\nPRAGMA synchronous = NORMAL;\nPRAGMA temp_store = MEMORY;\nPRAGMA cache_size = -131072; -- ~128 MiB page cache for faster repeated reads\nPRAGMA mmap_size = 268435456; -- 256 MiB memory map window to cut syscalls\nPRAGMA wal_autocheckpoint = 1000;\nPRAGMA optimize;\n";
|
|
2
|
+
export declare function vectors(dimension: number): string;
|
|
3
|
+
export declare const statements: string[];
|
|
4
|
+
//# sourceMappingURL=ddl.sqlite.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ddl.sqlite.d.ts","sourceRoot":"","sources":["../../../../src/lib/stores/sqlite/ddl.sqlite.ts"],"names":[],"mappings":"AAKA,eAAO,MAAM,KAAK,sWAUjB,CAAC;AAEF,wBAAgB,OAAO,CAAC,SAAS,EAAE,MAAM,UAOxC;AAED,eAAO,MAAM,UAAU,UA2BtB,CAAC"}
|
|
@@ -1,55 +1,53 @@
|
|
|
1
|
-
|
|
2
|
-
-- Use <%= DIMENSION %> placeholder replaced at runtime.
|
|
3
|
-
|
|
1
|
+
const SETUP = `
|
|
4
2
|
PRAGMA busy_timeout = 5000;
|
|
5
3
|
PRAGMA page_size = 32768;
|
|
6
4
|
PRAGMA journal_mode = WAL;
|
|
7
5
|
PRAGMA synchronous = NORMAL;
|
|
8
6
|
PRAGMA temp_store = MEMORY;
|
|
9
|
-
PRAGMA foreign_keys = ON;
|
|
10
7
|
PRAGMA cache_size = -131072; -- ~128 MiB page cache for faster repeated reads
|
|
11
8
|
PRAGMA mmap_size = 268435456; -- 256 MiB memory map window to cut syscalls
|
|
12
9
|
PRAGMA wal_autocheckpoint = 1000;
|
|
13
10
|
PRAGMA optimize;
|
|
14
|
-
|
|
15
|
-
|
|
11
|
+
`;
|
|
12
|
+
function vectors(dimension) {
|
|
13
|
+
return `CREATE VIRTUAL TABLE vec_chunks USING vec0(
|
|
16
14
|
source_id TEXT PARTITION KEY,
|
|
17
15
|
document_id TEXT,
|
|
18
|
-
embedding FLOAT[
|
|
16
|
+
embedding FLOAT[${dimension}] DISTANCE_METRIC=cosine,
|
|
19
17
|
+content TEXT -- auxiliary payload (not filterable) for reconstruction
|
|
20
|
-
)
|
|
21
|
-
|
|
22
|
-
|
|
18
|
+
)`;
|
|
19
|
+
}
|
|
20
|
+
const statements = [
|
|
21
|
+
`CREATE TABLE sources (
|
|
23
22
|
source_id TEXT PRIMARY KEY,
|
|
24
23
|
updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
|
|
25
24
|
expires_at TEXT
|
|
26
|
-
) STRICT
|
|
27
|
-
|
|
28
|
-
CREATE TABLE IF NOT EXISTS documents (
|
|
25
|
+
) STRICT`,
|
|
26
|
+
`CREATE TABLE documents (
|
|
29
27
|
id TEXT PRIMARY KEY,
|
|
30
28
|
source_id TEXT NOT NULL,
|
|
31
29
|
cid TEXT NOT NULL,
|
|
32
30
|
metadata TEXT, -- JSON blob with arbitrary document metadata
|
|
33
31
|
updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),
|
|
34
32
|
FOREIGN KEY (source_id) REFERENCES sources(source_id) ON DELETE CASCADE
|
|
35
|
-
) STRICT
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
CREATE INDEX
|
|
39
|
-
CREATE INDEX
|
|
40
|
-
CREATE INDEX
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
-- emulate cascade for the virtual table
|
|
44
|
-
CREATE TRIGGER IF NOT EXISTS trg_documents_delete_vec
|
|
33
|
+
) STRICT`,
|
|
34
|
+
// Indexes to accelerate lookups / maintenance
|
|
35
|
+
`CREATE INDEX idx_documents_source ON documents(source_id)`,
|
|
36
|
+
`CREATE INDEX idx_documents_source_updated ON documents(source_id, updated_at)`,
|
|
37
|
+
`CREATE INDEX idx_documents_cid ON documents(cid)`,
|
|
38
|
+
`CREATE INDEX idx_sources_expires_at ON sources(expires_at) WHERE expires_at IS NOT NULL`,
|
|
39
|
+
// emulate cascade for the virtual table
|
|
40
|
+
`CREATE TRIGGER trg_documents_delete_vec
|
|
45
41
|
AFTER DELETE ON documents
|
|
46
42
|
BEGIN
|
|
47
43
|
DELETE FROM vec_chunks
|
|
48
44
|
WHERE source_id = OLD.source_id
|
|
49
45
|
AND document_id = OLD.id;
|
|
50
|
-
END
|
|
51
|
-
|
|
46
|
+
END`
|
|
47
|
+
];
|
|
52
48
|
export {
|
|
53
|
-
|
|
49
|
+
SETUP,
|
|
50
|
+
statements,
|
|
51
|
+
vectors
|
|
54
52
|
};
|
|
55
|
-
//# sourceMappingURL=sqlite.
|
|
53
|
+
//# sourceMappingURL=ddl.sqlite.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../../src/lib/stores/sqlite/ddl.sqlite.ts"],
|
|
4
|
+
"sourcesContent": ["// Embedding store schema for SQLite. SQLite stores the text of each CREATE and\n// @zukhruf/sqlite-drift compares it, so the text after each object name must\n// stay byte-identical to what earlier releases wrote, inline comments included.\n\n// Session-level PRAGMAs, run on each connection before the schema is applied.\nexport const SETUP = `\nPRAGMA busy_timeout = 5000;\nPRAGMA page_size = 32768;\nPRAGMA journal_mode = WAL;\nPRAGMA synchronous = NORMAL;\nPRAGMA temp_store = MEMORY;\nPRAGMA cache_size = -131072; -- ~128 MiB page cache for faster repeated reads\nPRAGMA mmap_size = 268435456; -- 256 MiB memory map window to cut syscalls\nPRAGMA wal_autocheckpoint = 1000;\nPRAGMA optimize;\n`;\n\nexport function vectors(dimension: number) {\n return `CREATE VIRTUAL TABLE vec_chunks USING vec0(\n source_id TEXT PARTITION KEY,\n document_id TEXT,\n embedding FLOAT[${dimension}] DISTANCE_METRIC=cosine,\n +content TEXT -- auxiliary payload (not filterable) for reconstruction\n)`;\n}\n\nexport const statements = [\n `CREATE TABLE sources (\n source_id TEXT PRIMARY KEY,\n updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n expires_at TEXT\n) STRICT`,\n `CREATE TABLE documents (\n id TEXT PRIMARY KEY,\n source_id TEXT NOT NULL,\n cid TEXT NOT NULL,\n metadata TEXT, -- JSON blob with arbitrary document metadata\n updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n FOREIGN KEY (source_id) REFERENCES sources(source_id) ON DELETE CASCADE\n) STRICT`,\n // Indexes to accelerate lookups / maintenance\n `CREATE INDEX idx_documents_source ON documents(source_id)`,\n `CREATE INDEX idx_documents_source_updated ON documents(source_id, updated_at)`,\n `CREATE INDEX idx_documents_cid ON documents(cid)`,\n `CREATE INDEX idx_sources_expires_at ON sources(expires_at) WHERE expires_at IS NOT NULL`,\n // emulate cascade for the virtual table\n `CREATE TRIGGER trg_documents_delete_vec\nAFTER DELETE ON documents\nBEGIN\n DELETE FROM vec_chunks\n WHERE source_id = OLD.source_id\n AND document_id = OLD.id;\nEND`,\n];\n"],
|
|
5
|
+
"mappings": "AAKO,MAAM,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAYd,SAAS,QAAQ,WAAmB;AACzC,SAAO;AAAA;AAAA;AAAA,oBAGW,SAAS;AAAA;AAAA;AAG7B;AAEO,MAAM,aAAa;AAAA,EACxB;AAAA;AAAA;AAAA;AAAA;AAAA,EAKA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA;AAAA,EAEA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAOF;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"node-sqlite.d.ts","sourceRoot":"","sources":["../../../../src/lib/stores/sqlite/node-sqlite.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"node-sqlite.d.ts","sourceRoot":"","sources":["../../../../src/lib/stores/sqlite/node-sqlite.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1C,wBAAgB,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,eAE3D"}
|
|
@@ -1,12 +1,7 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import * as sqliteVec from "sqlite-vec";
|
|
1
|
+
import { open } from "./open.js";
|
|
3
2
|
import { SQLiteStore } from "./sqlite.js";
|
|
4
3
|
function nodeSQLite(dbName, dimension) {
|
|
5
|
-
|
|
6
|
-
allowExtension: true
|
|
7
|
-
});
|
|
8
|
-
db.loadExtension(sqliteVec.getLoadablePath());
|
|
9
|
-
return new SQLiteStore(db, dimension);
|
|
4
|
+
return new SQLiteStore(open(dbName), dimension);
|
|
10
5
|
}
|
|
11
6
|
export {
|
|
12
7
|
nodeSQLite
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/lib/stores/sqlite/node-sqlite.ts"],
|
|
4
|
-
"sourcesContent": ["import {
|
|
5
|
-
"mappings": "AAAA,SAAS,
|
|
4
|
+
"sourcesContent": ["import { open } from './open.js';\nimport { SQLiteStore } from './sqlite.js';\n\nexport function nodeSQLite(dbName: string, dimension: number) {\n return new SQLiteStore(open(dbName), dimension);\n}\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,YAAY;AACrB,SAAS,mBAAmB;AAErB,SAAS,WAAW,QAAgB,WAAmB;AAC5D,SAAO,IAAI,YAAY,KAAK,MAAM,GAAG,SAAS;AAChD;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"open.d.ts","sourceRoot":"","sources":["../../../../src/lib/stores/sqlite/open.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAK3C,wBAAgB,IAAI,CAAC,IAAI,EAAE,MAAM,gBAIhC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { DatabaseSync } from "node:sqlite";
|
|
2
|
+
import * as sqliteVec from "sqlite-vec";
|
|
3
|
+
function open(path) {
|
|
4
|
+
const db = new DatabaseSync(path, { allowExtension: true });
|
|
5
|
+
db.loadExtension(sqliteVec.getLoadablePath());
|
|
6
|
+
return db;
|
|
7
|
+
}
|
|
8
|
+
export {
|
|
9
|
+
open
|
|
10
|
+
};
|
|
11
|
+
//# sourceMappingURL=open.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../../src/lib/stores/sqlite/open.ts"],
|
|
4
|
+
"sourcesContent": ["import { DatabaseSync } from 'node:sqlite';\nimport * as sqliteVec from 'sqlite-vec';\n\n// The store's statements use the vec0 module, so every connection that runs\n// them, the dev database of the declaration included, loads sqlite-vec.\nexport function open(path: string) {\n const db = new DatabaseSync(path, { allowExtension: true });\n db.loadExtension(sqliteVec.getLoadablePath());\n return db;\n}\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,oBAAoB;AAC7B,YAAY,eAAe;AAIpB,SAAS,KAAK,MAAc;AACjC,QAAM,KAAK,IAAI,aAAa,MAAM,EAAE,gBAAgB,KAAK,CAAC;AAC1D,KAAG,cAAc,UAAU,gBAAgB,CAAC;AAC5C,SAAO;AACT;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
|
@@ -1,23 +1,15 @@
|
|
|
1
|
+
import type { DatabaseSync } from 'node:sqlite';
|
|
1
2
|
import type { Chunk, Corpus, Embedder, SearchOptions, SearchResult, Store } from '../store.js';
|
|
2
|
-
interface DB {
|
|
3
|
-
prepare: (sql: string) => {
|
|
4
|
-
run: (...args: any[]) => any;
|
|
5
|
-
all: (...args: any[]) => any[];
|
|
6
|
-
get: (...args: any[]) => any;
|
|
7
|
-
};
|
|
8
|
-
exec: (sql: string) => void;
|
|
9
|
-
isTransaction: boolean;
|
|
10
|
-
}
|
|
11
3
|
export declare class SQLiteStore implements Store {
|
|
12
4
|
#private;
|
|
13
|
-
constructor(db:
|
|
5
|
+
constructor(db: DatabaseSync, dimension: number);
|
|
14
6
|
search(query: string, options: SearchOptions, embedder: Embedder): Promise<SearchResult[]>;
|
|
15
7
|
upsertDoc(inputs: {
|
|
16
8
|
documentId: string;
|
|
17
9
|
sourceId: string;
|
|
18
10
|
cid: string;
|
|
19
11
|
metadata?: Record<string, any>;
|
|
20
|
-
}):
|
|
12
|
+
}): import("node:sqlite").StatementResultingChanges;
|
|
21
13
|
insertDoc(inputs: {
|
|
22
14
|
sourceId: string;
|
|
23
15
|
documentId: string;
|
|
@@ -25,12 +17,11 @@ export declare class SQLiteStore implements Store {
|
|
|
25
17
|
delete(inputs: {
|
|
26
18
|
sourceId: string;
|
|
27
19
|
documentId: string;
|
|
28
|
-
}):
|
|
20
|
+
}): import("node:sqlite").StatementResultingChanges;
|
|
29
21
|
sourceExists(sourceId: string): boolean;
|
|
30
22
|
sourceExpired(sourceId: string): boolean;
|
|
31
|
-
setSourceExpiry(sourceId: string, expiryDate: Date):
|
|
23
|
+
setSourceExpiry(sourceId: string, expiryDate: Date): void;
|
|
32
24
|
index(sourceId: string, corpus: Corpus, expiryDate?: Date): Promise<void>;
|
|
33
25
|
}
|
|
34
26
|
export declare function vectorToBlob(vector: number[] | Float32Array): Buffer;
|
|
35
|
-
export {};
|
|
36
27
|
//# sourceMappingURL=sqlite.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sqlite.d.ts","sourceRoot":"","sources":["../../../../src/lib/stores/sqlite/sqlite.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"sqlite.d.ts","sourceRoot":"","sources":["../../../../src/lib/stores/sqlite/sqlite.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,KAAK,EACV,KAAK,EACL,MAAM,EACN,QAAQ,EACR,aAAa,EACb,YAAY,EACZ,KAAK,EACN,MAAM,aAAa,CAAC;AAQrB,qBAAa,WAAY,YAAW,KAAK;;gBAE3B,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM;IA8DzC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ;IAyCtE,SAAS,CAAC,MAAM,EAAE;QAChB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,GAAG,EAAE,MAAM,CAAC;QACZ,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KAChC;IAiBD,SAAS,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,IAKhD,OAAO,KAAK;IAStB,MAAM,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE;IAMvD,YAAY,CAAC,QAAQ,EAAE,MAAM;IAO7B,aAAa,CAAC,QAAQ,EAAE,MAAM;IAW9B,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI;IAQ5C,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,IAAI;CAuDhE;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,YAAY,GAAG,MAAM,CASpE"}
|
|
@@ -1,14 +1,29 @@
|
|
|
1
|
+
import { Declaration } from "@zukhruf/sqlite-drift";
|
|
1
2
|
import { transaction } from "@zukhruf/sqlite-transaction";
|
|
2
3
|
import dedent from "dedent";
|
|
3
|
-
import {
|
|
4
|
-
import
|
|
4
|
+
import { SETUP, statements, vectors } from "./ddl.sqlite.js";
|
|
5
|
+
import { open } from "./open.js";
|
|
5
6
|
const DEFAULT_TOP_N = 10;
|
|
7
|
+
const tables = new Declaration(statements, { rows: "keep" });
|
|
6
8
|
class SQLiteStore {
|
|
7
9
|
#db;
|
|
8
10
|
constructor(db, dimension) {
|
|
9
11
|
this.#db = db;
|
|
10
|
-
|
|
11
|
-
|
|
12
|
+
this.#db.exec(SETUP);
|
|
13
|
+
const chunks = new Declaration([vectors(dimension)], {
|
|
14
|
+
rows: "replace",
|
|
15
|
+
devDatabase: () => open(":memory:")
|
|
16
|
+
});
|
|
17
|
+
transaction(
|
|
18
|
+
this.#db,
|
|
19
|
+
() => {
|
|
20
|
+
const stale = chunks.drift(this.#db) === "stale";
|
|
21
|
+
chunks.apply(this.#db);
|
|
22
|
+
if (stale) this.#db.exec("DELETE FROM sources");
|
|
23
|
+
},
|
|
24
|
+
{ mode: "immediate" }
|
|
25
|
+
);
|
|
26
|
+
tables.apply(this.#db);
|
|
12
27
|
}
|
|
13
28
|
#searchByDocument(inputs) {
|
|
14
29
|
const stmt = this.#db.prepare(dedent`
|
|
@@ -74,7 +89,7 @@ class SQLiteStore {
|
|
|
74
89
|
const stmt = this.#db.prepare(dedent`
|
|
75
90
|
INSERT INTO sources (source_id, expires_at) VALUES (?, ?)
|
|
76
91
|
ON CONFLICT(source_id) DO UPDATE SET
|
|
77
|
-
updated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now',
|
|
92
|
+
updated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now'),
|
|
78
93
|
expires_at=excluded.expires_at
|
|
79
94
|
`);
|
|
80
95
|
return stmt.run(inputs.sourceId, inputs.expiresAt);
|
|
@@ -131,7 +146,7 @@ class SQLiteStore {
|
|
|
131
146
|
AND expires_at <= strftime('%Y-%m-%dT%H:%M:%fZ','now')
|
|
132
147
|
LIMIT 1
|
|
133
148
|
`);
|
|
134
|
-
const row = stmt.
|
|
149
|
+
const row = stmt.get(sourceId);
|
|
135
150
|
return Boolean(row);
|
|
136
151
|
}
|
|
137
152
|
setSourceExpiry(sourceId, expiryDate) {
|
|
@@ -140,7 +155,7 @@ class SQLiteStore {
|
|
|
140
155
|
SET expires_at = ?, updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')
|
|
141
156
|
WHERE source_id = ?
|
|
142
157
|
`);
|
|
143
|
-
|
|
158
|
+
stmt.run(expiryDate.toISOString(), sourceId);
|
|
144
159
|
}
|
|
145
160
|
async index(sourceId, corpus, expiryDate) {
|
|
146
161
|
if (expiryDate) {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/lib/stores/sqlite/sqlite.ts"],
|
|
4
|
-
"sourcesContent": ["import { transaction } from '@zukhruf/sqlite-transaction';\nimport dedent from 'dedent';\nimport {
|
|
5
|
-
"mappings": "AAAA,SAAS,mBAAmB;AAC5B,OAAO,YAAY;
|
|
4
|
+
"sourcesContent": ["import { Declaration } from '@zukhruf/sqlite-drift';\nimport { transaction } from '@zukhruf/sqlite-transaction';\nimport dedent from 'dedent';\nimport type { DatabaseSync } from 'node:sqlite';\n\nimport type {\n Chunk,\n Corpus,\n Embedder,\n SearchOptions,\n SearchResult,\n Store,\n} from '../store.js';\nimport { SETUP, statements, vectors } from './ddl.sqlite.js';\nimport { open } from './open.js';\n\nconst DEFAULT_TOP_N = 10;\n\nconst tables = new Declaration(statements, { rows: 'keep' });\n\nexport class SQLiteStore implements Store {\n readonly #db: DatabaseSync;\n constructor(db: DatabaseSync, dimension: number) {\n this.#db = db;\n this.#db.exec(SETUP);\n const chunks = new Declaration([vectors(dimension)], {\n rows: 'replace',\n devDatabase: () => open(':memory:'),\n });\n transaction(\n this.#db,\n () => {\n const stale = chunks.drift(this.#db) === 'stale';\n chunks.apply(this.#db);\n // A replaced vec_chunks is empty, yet upsertDoc skips a document whose\n // cid is unchanged and similaritySearch skips a source that exists.\n // Deleting the sources cascades to their documents, so the next\n // ingest embeds every document again.\n if (stale) this.#db.exec('DELETE FROM sources');\n },\n { mode: 'immediate' },\n );\n tables.apply(this.#db);\n }\n\n #searchByDocument(inputs: {\n sourceId: string;\n documentId: string;\n embedding: Buffer<ArrayBufferLike>;\n k: number;\n }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT v.content, v.distance, v.document_id, d.metadata\n\t\t\t\tFROM vec_chunks v\n\t\t\t\tJOIN documents d ON d.id = v.document_id\n\t\t\t\tWHERE v.source_id = ?\n\t\t\t\t\tAND v.document_id = ?\n\t\t\t\t\tAND v.embedding MATCH vec_normalize(vec_f32(?))\n\t\t\t\t\tAND v.k = ?\n\t\t\t\tORDER BY v.distance ASC\n\t\t\t`);\n return stmt.all(\n inputs.sourceId,\n inputs.documentId,\n inputs.embedding,\n inputs.k,\n );\n }\n #searchBySource(inputs: {\n sourceId: string;\n embedding: Buffer<ArrayBufferLike>;\n k: number;\n }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT v.content, v.distance, v.document_id, d.metadata\n\t\t\t\tFROM vec_chunks v\n\t\t\t\tJOIN documents d ON d.id = v.document_id\n\t\t\t\tWHERE v.source_id = ?\n\t\t\t\t\tAND v.embedding MATCH vec_normalize(vec_f32(?))\n\t\t\t\t\tAND v.k = ?\n\t\t\t\tORDER BY v.distance ASC\n\t\t\t`);\n return stmt.all(inputs.sourceId, inputs.embedding, inputs.k);\n }\n async search(query: string, options: SearchOptions, embedder: Embedder) {\n const { embeddings } = await embedder([query]);\n if (!embeddings.length) {\n return [];\n }\n const vectorBlob = vectorToBlob(embeddings[0]);\n const topN = options.topN;\n\n if (options.documentId) {\n const rows = this.#searchByDocument({\n sourceId: options.sourceId,\n documentId: options.documentId,\n embedding: vectorBlob,\n k: topN ?? DEFAULT_TOP_N,\n });\n return rows.map(toSearchResult);\n }\n\n const rows = this.#searchBySource({\n sourceId: options.sourceId,\n embedding: vectorBlob,\n k: topN ?? DEFAULT_TOP_N,\n });\n return rows.map(toSearchResult);\n }\n #upsertSource(inputs: { sourceId: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tINSERT INTO sources (source_id) VALUES (?)\n\t\t\t\tON CONFLICT(source_id) DO UPDATE SET updated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now')\n\t\t\t`);\n return stmt.run(inputs.sourceId);\n }\n #upsertSourceWithExpiry(inputs: { sourceId: string; expiresAt: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tINSERT INTO sources (source_id, expires_at) VALUES (?, ?)\n\t\t\t\tON CONFLICT(source_id) DO UPDATE SET\n\t\t\t\t\tupdated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now'),\n\t\t\t\t\texpires_at=excluded.expires_at\n\t\t\t`);\n return stmt.run(inputs.sourceId, inputs.expiresAt);\n }\n upsertDoc(inputs: {\n documentId: string;\n sourceId: string;\n cid: string;\n metadata?: Record<string, any>;\n }) {\n const stmt = this.#db.prepare(dedent`\n INSERT INTO documents (id, source_id, cid, metadata)\n VALUES (?, ?, ?, json(?))\n ON CONFLICT(id) DO UPDATE SET\n cid=excluded.cid,\n metadata=COALESCE(excluded.metadata, documents.metadata),\n updated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now')\n WHERE documents.cid != excluded.cid;\n `);\n return stmt.run(\n inputs.documentId,\n inputs.sourceId,\n inputs.cid,\n inputs.metadata ? JSON.stringify(inputs.metadata) : null,\n );\n }\n insertDoc(inputs: { sourceId: string; documentId: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tINSERT INTO vec_chunks (source_id, document_id, content, embedding)\n\t\t\t\tVALUES (?, ?, ?, vec_normalize(vec_f32(?)))\n\t\t\t`);\n return (chunk: Chunk) => {\n stmt.run(\n inputs.sourceId,\n inputs.documentId,\n chunk.content,\n vectorToBlob(chunk.embedding),\n );\n };\n }\n delete(inputs: { sourceId: string; documentId: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tDELETE FROM vec_chunks WHERE source_id = ? AND document_id = ?\n\t\t\t`);\n return stmt.run(inputs.sourceId, inputs.documentId);\n }\n sourceExists(sourceId: string) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT 1 FROM sources WHERE source_id = ? LIMIT 1\n\t\t\t`);\n const row = stmt.get(sourceId);\n return Boolean(row);\n }\n sourceExpired(sourceId: string) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT 1 FROM sources\n\t\t\t\tWHERE source_id = ?\n\t\t\t\t\tAND expires_at IS NOT NULL\n\t\t\t\t\tAND expires_at <= strftime('%Y-%m-%dT%H:%M:%fZ','now')\n\t\t\t\tLIMIT 1\n\t\t\t`);\n const row = stmt.get(sourceId);\n return Boolean(row);\n }\n setSourceExpiry(sourceId: string, expiryDate: Date) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tUPDATE sources\n\t\t\t\tSET expires_at = ?, updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')\n\t\t\t\tWHERE source_id = ?\n\t\t\t`);\n stmt.run(expiryDate.toISOString(), sourceId);\n }\n async index(sourceId: string, corpus: Corpus, expiryDate?: Date) {\n if (expiryDate) {\n this.#upsertSourceWithExpiry({\n expiresAt: expiryDate.toISOString(),\n sourceId,\n });\n } else {\n this.#upsertSource({ sourceId });\n }\n\n const info = this.upsertDoc({\n documentId: corpus.id,\n sourceId,\n cid: corpus.cid,\n metadata: corpus.metadata,\n });\n const changed = info.changes > 0;\n\n if (!changed) {\n return;\n }\n\n const insert = this.insertDoc({ sourceId, documentId: corpus.id });\n // Delete previous rows once before inserting\n transaction(\n this.#db,\n () => {\n this.delete({ sourceId, documentId: corpus.id });\n },\n { mode: 'immediate' },\n );\n\n const batchSize = 32;\n let batch: Chunk[] = [];\n const flush = () => {\n if (!batch.length) return;\n transaction(\n this.#db,\n () => {\n for (let i = 0; i < batch.length; i++) {\n insert(batch[i]);\n }\n },\n { mode: 'immediate' },\n );\n batch = [];\n };\n\n for await (const chunk of corpus.chunker()) {\n batch.push(chunk);\n if (batch.length >= batchSize) flush();\n }\n // flush any remaining\n flush();\n }\n}\n\nexport function vectorToBlob(vector: number[] | Float32Array): Buffer {\n if (vector instanceof Float32Array) {\n // Copy into a fresh Buffer to avoid retaining references to a larger batch tensor buffer\n const copied = new Float32Array(vector.length);\n copied.set(vector);\n return Buffer.from(copied.buffer);\n }\n const floatArray = new Float32Array(vector);\n return Buffer.from(floatArray.buffer);\n}\n\nfunction toSearchResult(row: unknown): SearchResult {\n if (\n typeof row !== 'object' ||\n row === null ||\n !('content' in row) ||\n typeof row.content !== 'string' ||\n !('distance' in row) ||\n typeof row.distance !== 'number' ||\n !('document_id' in row) ||\n typeof row.document_id !== 'string' ||\n !('metadata' in row) ||\n (row.metadata !== null && typeof row.metadata !== 'string')\n ) {\n throw new Error('Unexpected vector search row');\n }\n return {\n content: row.content,\n distance: row.distance,\n document_id: row.document_id,\n metadata: parseMetadata(row.metadata),\n };\n}\n\nfunction parseMetadata(value: string | null): Record<string, unknown> | null {\n if (value === null) return null;\n try {\n const parsed: unknown = JSON.parse(value);\n return isRecord(parsed) ? parsed : null;\n } catch {\n return null;\n }\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n return typeof value === 'object' && value !== null && !Array.isArray(value);\n}\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,mBAAmB;AAC5B,SAAS,mBAAmB;AAC5B,OAAO,YAAY;AAWnB,SAAS,OAAO,YAAY,eAAe;AAC3C,SAAS,YAAY;AAErB,MAAM,gBAAgB;AAEtB,MAAM,SAAS,IAAI,YAAY,YAAY,EAAE,MAAM,OAAO,CAAC;AAEpD,MAAM,YAA6B;AAAA,EAC/B;AAAA,EACT,YAAY,IAAkB,WAAmB;AAC/C,SAAK,MAAM;AACX,SAAK,IAAI,KAAK,KAAK;AACnB,UAAM,SAAS,IAAI,YAAY,CAAC,QAAQ,SAAS,CAAC,GAAG;AAAA,MACnD,MAAM;AAAA,MACN,aAAa,MAAM,KAAK,UAAU;AAAA,IACpC,CAAC;AACD;AAAA,MACE,KAAK;AAAA,MACL,MAAM;AACJ,cAAM,QAAQ,OAAO,MAAM,KAAK,GAAG,MAAM;AACzC,eAAO,MAAM,KAAK,GAAG;AAKrB,YAAI,MAAO,MAAK,IAAI,KAAK,qBAAqB;AAAA,MAChD;AAAA,MACA,EAAE,MAAM,YAAY;AAAA,IACtB;AACA,WAAO,MAAM,KAAK,GAAG;AAAA,EACvB;AAAA,EAEA,kBAAkB,QAKf;AACD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAS9B;AACA,WAAO,KAAK;AAAA,MACV,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO;AAAA,IACT;AAAA,EACF;AAAA,EACA,gBAAgB,QAIb;AACD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAQ9B;AACA,WAAO,KAAK,IAAI,OAAO,UAAU,OAAO,WAAW,OAAO,CAAC;AAAA,EAC7D;AAAA,EACA,MAAM,OAAO,OAAe,SAAwB,UAAoB;AACtE,UAAM,EAAE,WAAW,IAAI,MAAM,SAAS,CAAC,KAAK,CAAC;AAC7C,QAAI,CAAC,WAAW,QAAQ;AACtB,aAAO,CAAC;AAAA,IACV;AACA,UAAM,aAAa,aAAa,WAAW,CAAC,CAAC;AAC7C,UAAM,OAAO,QAAQ;AAErB,QAAI,QAAQ,YAAY;AACtB,YAAMA,QAAO,KAAK,kBAAkB;AAAA,QAClC,UAAU,QAAQ;AAAA,QAClB,YAAY,QAAQ;AAAA,QACpB,WAAW;AAAA,QACX,GAAG,QAAQ;AAAA,MACb,CAAC;AACD,aAAOA,MAAK,IAAI,cAAc;AAAA,IAChC;AAEA,UAAM,OAAO,KAAK,gBAAgB;AAAA,MAChC,UAAU,QAAQ;AAAA,MAClB,WAAW;AAAA,MACX,GAAG,QAAQ;AAAA,IACb,CAAC;AACD,WAAO,KAAK,IAAI,cAAc;AAAA,EAChC;AAAA,EACA,cAAc,QAA8B;AAC1C,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA,IAG9B;AACA,WAAO,KAAK,IAAI,OAAO,QAAQ;AAAA,EACjC;AAAA,EACA,wBAAwB,QAAiD;AACvE,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,IAK9B;AACA,WAAO,KAAK,IAAI,OAAO,UAAU,OAAO,SAAS;AAAA,EACnD;AAAA,EACA,UAAU,QAKP;AACD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,OAQ3B;AACH,WAAO,KAAK;AAAA,MACV,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO,WAAW,KAAK,UAAU,OAAO,QAAQ,IAAI;AAAA,IACtD;AAAA,EACF;AAAA,EACA,UAAU,QAAkD;AAC1D,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA,IAG9B;AACA,WAAO,CAAC,UAAiB;AACvB,WAAK;AAAA,QACH,OAAO;AAAA,QACP,OAAO;AAAA,QACP,MAAM;AAAA,QACN,aAAa,MAAM,SAAS;AAAA,MAC9B;AAAA,IACF;AAAA,EACF;AAAA,EACA,OAAO,QAAkD;AACvD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA,IAE9B;AACA,WAAO,KAAK,IAAI,OAAO,UAAU,OAAO,UAAU;AAAA,EACpD;AAAA,EACA,aAAa,UAAkB;AAC7B,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA,IAE9B;AACA,UAAM,MAAM,KAAK,IAAI,QAAQ;AAC7B,WAAO,QAAQ,GAAG;AAAA,EACpB;AAAA,EACA,cAAc,UAAkB;AAC9B,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAM9B;AACA,UAAM,MAAM,KAAK,IAAI,QAAQ;AAC7B,WAAO,QAAQ,GAAG;AAAA,EACpB;AAAA,EACA,gBAAgB,UAAkB,YAAkB;AAClD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA,IAI9B;AACA,SAAK,IAAI,WAAW,YAAY,GAAG,QAAQ;AAAA,EAC7C;AAAA,EACA,MAAM,MAAM,UAAkB,QAAgB,YAAmB;AAC/D,QAAI,YAAY;AACd,WAAK,wBAAwB;AAAA,QAC3B,WAAW,WAAW,YAAY;AAAA,QAClC;AAAA,MACF,CAAC;AAAA,IACH,OAAO;AACL,WAAK,cAAc,EAAE,SAAS,CAAC;AAAA,IACjC;AAEA,UAAM,OAAO,KAAK,UAAU;AAAA,MAC1B,YAAY,OAAO;AAAA,MACnB;AAAA,MACA,KAAK,OAAO;AAAA,MACZ,UAAU,OAAO;AAAA,IACnB,CAAC;AACD,UAAM,UAAU,KAAK,UAAU;AAE/B,QAAI,CAAC,SAAS;AACZ;AAAA,IACF;AAEA,UAAM,SAAS,KAAK,UAAU,EAAE,UAAU,YAAY,OAAO,GAAG,CAAC;AAEjE;AAAA,MACE,KAAK;AAAA,MACL,MAAM;AACJ,aAAK,OAAO,EAAE,UAAU,YAAY,OAAO,GAAG,CAAC;AAAA,MACjD;AAAA,MACA,EAAE,MAAM,YAAY;AAAA,IACtB;AAEA,UAAM,YAAY;AAClB,QAAI,QAAiB,CAAC;AACtB,UAAM,QAAQ,MAAM;AAClB,UAAI,CAAC,MAAM,OAAQ;AACnB;AAAA,QACE,KAAK;AAAA,QACL,MAAM;AACJ,mBAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,mBAAO,MAAM,CAAC,CAAC;AAAA,UACjB;AAAA,QACF;AAAA,QACA,EAAE,MAAM,YAAY;AAAA,MACtB;AACA,cAAQ,CAAC;AAAA,IACX;AAEA,qBAAiB,SAAS,OAAO,QAAQ,GAAG;AAC1C,YAAM,KAAK,KAAK;AAChB,UAAI,MAAM,UAAU,UAAW,OAAM;AAAA,IACvC;AAEA,UAAM;AAAA,EACR;AACF;AAEO,SAAS,aAAa,QAAyC;AACpE,MAAI,kBAAkB,cAAc;AAElC,UAAM,SAAS,IAAI,aAAa,OAAO,MAAM;AAC7C,WAAO,IAAI,MAAM;AACjB,WAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAClC;AACA,QAAM,aAAa,IAAI,aAAa,MAAM;AAC1C,SAAO,OAAO,KAAK,WAAW,MAAM;AACtC;AAEA,SAAS,eAAe,KAA4B;AAClD,MACE,OAAO,QAAQ,YACf,QAAQ,QACR,EAAE,aAAa,QACf,OAAO,IAAI,YAAY,YACvB,EAAE,cAAc,QAChB,OAAO,IAAI,aAAa,YACxB,EAAE,iBAAiB,QACnB,OAAO,IAAI,gBAAgB,YAC3B,EAAE,cAAc,QACf,IAAI,aAAa,QAAQ,OAAO,IAAI,aAAa,UAClD;AACA,UAAM,IAAI,MAAM,8BAA8B;AAAA,EAChD;AACA,SAAO;AAAA,IACL,SAAS,IAAI;AAAA,IACb,UAAU,IAAI;AAAA,IACd,aAAa,IAAI;AAAA,IACjB,UAAU,cAAc,IAAI,QAAQ;AAAA,EACtC;AACF;AAEA,SAAS,cAAc,OAAsD;AAC3E,MAAI,UAAU,KAAM,QAAO;AAC3B,MAAI;AACF,UAAM,SAAkB,KAAK,MAAM,KAAK;AACxC,WAAO,SAAS,MAAM,IAAI,SAAS;AAAA,EACrC,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,SAAS,SAAS,OAAkD;AAClE,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;",
|
|
6
6
|
"names": ["rows"]
|
|
7
7
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@deepagents/retrieval",
|
|
3
|
-
"version": "7.
|
|
3
|
+
"version": "7.2.0",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -40,22 +40,21 @@
|
|
|
40
40
|
"access": "public"
|
|
41
41
|
},
|
|
42
42
|
"dependencies": {
|
|
43
|
+
"@ai-sdk/tool-web-fetch": "^1.0.3",
|
|
43
44
|
"@langchain/textsplitters": "^1.0.1",
|
|
44
|
-
"@zukhruf/sqlite-
|
|
45
|
+
"@zukhruf/sqlite-drift": "^0.3.45",
|
|
46
|
+
"@zukhruf/sqlite-transaction": "^0.3.45",
|
|
45
47
|
"execa": "^10.0.0",
|
|
46
48
|
"@linear/sdk": "^88.2.0",
|
|
47
49
|
"fast-glob": "^3.3.3",
|
|
48
50
|
"ignore": "^7.0.6",
|
|
49
51
|
"unpdf": "^1.6.2",
|
|
50
|
-
"@mozilla/readability": "^0.6.0",
|
|
51
52
|
"rss-parser": "^3.13.0",
|
|
52
|
-
"jsdom": "^29.1.1",
|
|
53
53
|
"@huggingface/transformers": "^4.2.0",
|
|
54
54
|
"langchain": "^1.5.3",
|
|
55
|
-
"ai": "^7.0.
|
|
55
|
+
"ai": "^7.0.153",
|
|
56
56
|
"sqlite-vec": "^0.1.9",
|
|
57
57
|
"dedent": "1.7.2",
|
|
58
|
-
"lodash-es": "^4.18.1",
|
|
59
58
|
"@langchain/core": "^1.2.3"
|
|
60
59
|
}
|
|
61
60
|
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
//# sourceMappingURL=bun-sqlite.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"bun-sqlite.d.ts","sourceRoot":"","sources":["../../../../src/lib/stores/sqlite/bun-sqlite.ts"],"names":[],"mappings":""}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
//# sourceMappingURL=bun-sqlite.js.map
|
|
@@ -1,3 +0,0 @@
|
|
|
1
|
-
declare const _default: "-- Embedding store schema\n-- Use <%= DIMENSION %> placeholder replaced at runtime.\n\nPRAGMA busy_timeout = 5000;\nPRAGMA page_size = 32768;\nPRAGMA journal_mode = WAL;\nPRAGMA synchronous = NORMAL;\nPRAGMA temp_store = MEMORY;\nPRAGMA foreign_keys = ON;\nPRAGMA cache_size = -131072; -- ~128 MiB page cache for faster repeated reads\nPRAGMA mmap_size = 268435456; -- 256 MiB memory map window to cut syscalls\nPRAGMA wal_autocheckpoint = 1000;\nPRAGMA optimize;\n\nCREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(\n source_id TEXT PARTITION KEY,\n document_id TEXT,\n embedding FLOAT[<%= DIMENSION %>] DISTANCE_METRIC=cosine,\n +content TEXT -- auxiliary payload (not filterable) for reconstruction\n);\n\nCREATE TABLE IF NOT EXISTS sources (\n source_id TEXT PRIMARY KEY,\n updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n expires_at TEXT\n) STRICT;\n\nCREATE TABLE IF NOT EXISTS documents (\n id TEXT PRIMARY KEY,\n source_id TEXT NOT NULL,\n cid TEXT NOT NULL,\n metadata TEXT, -- JSON blob with arbitrary document metadata\n updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n FOREIGN KEY (source_id) REFERENCES sources(source_id) ON DELETE CASCADE\n) STRICT;\n\n-- Indexes to accelerate lookups / maintenance\nCREATE INDEX IF NOT EXISTS idx_documents_source ON documents(source_id);\nCREATE INDEX IF NOT EXISTS idx_documents_source_updated ON documents(source_id, updated_at);\nCREATE INDEX IF NOT EXISTS idx_documents_cid ON documents(cid);\nCREATE INDEX IF NOT EXISTS idx_sources_expires_at ON sources(expires_at) WHERE expires_at IS NOT NULL;\n\n-- emulate cascade for the virtual table\nCREATE TRIGGER IF NOT EXISTS trg_documents_delete_vec\nAFTER DELETE ON documents\nBEGIN\n DELETE FROM vec_chunks\n WHERE source_id = OLD.source_id\n AND document_id = OLD.id;\nEND;\n";
|
|
2
|
-
export default _default;
|
|
3
|
-
//# sourceMappingURL=sqlite.sql.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"sqlite.sql.d.ts","sourceRoot":"","sources":["../../../../src/lib/stores/sqlite/sqlite.sql.ts"],"names":[],"mappings":";AAAA,wBAkDE"}
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": 3,
|
|
3
|
-
"sources": ["../../../../src/lib/stores/sqlite/sqlite.sql.ts"],
|
|
4
|
-
"sourcesContent": ["export default `-- Embedding store schema\n-- Use <%= DIMENSION %> placeholder replaced at runtime.\n\nPRAGMA busy_timeout = 5000;\nPRAGMA page_size = 32768;\nPRAGMA journal_mode = WAL;\nPRAGMA synchronous = NORMAL;\nPRAGMA temp_store = MEMORY;\nPRAGMA foreign_keys = ON;\nPRAGMA cache_size = -131072; -- ~128 MiB page cache for faster repeated reads\nPRAGMA mmap_size = 268435456; -- 256 MiB memory map window to cut syscalls\nPRAGMA wal_autocheckpoint = 1000;\nPRAGMA optimize;\n\nCREATE VIRTUAL TABLE IF NOT EXISTS vec_chunks USING vec0(\n source_id TEXT PARTITION KEY,\n document_id TEXT,\n embedding FLOAT[<%= DIMENSION %>] DISTANCE_METRIC=cosine,\n +content TEXT -- auxiliary payload (not filterable) for reconstruction\n);\n\nCREATE TABLE IF NOT EXISTS sources (\n source_id TEXT PRIMARY KEY,\n updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n expires_at TEXT\n) STRICT;\n\nCREATE TABLE IF NOT EXISTS documents (\n id TEXT PRIMARY KEY,\n source_id TEXT NOT NULL,\n cid TEXT NOT NULL,\n metadata TEXT, -- JSON blob with arbitrary document metadata\n updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')),\n FOREIGN KEY (source_id) REFERENCES sources(source_id) ON DELETE CASCADE\n) STRICT;\n\n-- Indexes to accelerate lookups / maintenance\nCREATE INDEX IF NOT EXISTS idx_documents_source ON documents(source_id);\nCREATE INDEX IF NOT EXISTS idx_documents_source_updated ON documents(source_id, updated_at);\nCREATE INDEX IF NOT EXISTS idx_documents_cid ON documents(cid);\nCREATE INDEX IF NOT EXISTS idx_sources_expires_at ON sources(expires_at) WHERE expires_at IS NOT NULL;\n\n-- emulate cascade for the virtual table\nCREATE TRIGGER IF NOT EXISTS trg_documents_delete_vec\nAFTER DELETE ON documents\nBEGIN\n DELETE FROM vec_chunks\n WHERE source_id = OLD.source_id\n AND document_id = OLD.id;\nEND;\n`;\n"],
|
|
5
|
-
"mappings": "AAAA,IAAO,qBAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;",
|
|
6
|
-
"names": []
|
|
7
|
-
}
|