@deepagents/retrieval 7.0.2 → 7.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/connectors/github.d.ts.map +1 -1
- package/dist/lib/connectors/github.js +9 -0
- package/dist/lib/connectors/github.js.map +2 -2
- package/dist/lib/connectors/repo.d.ts.map +1 -1
- package/dist/lib/connectors/repo.js +9 -1
- package/dist/lib/connectors/repo.js.map +2 -2
- package/dist/lib/connectors/rss.d.ts +1 -1
- package/dist/lib/connectors/rss.d.ts.map +1 -1
- package/dist/lib/connectors/rss.js.map +2 -2
- package/dist/lib/embedders/fastembed.d.ts +10 -9
- package/dist/lib/embedders/fastembed.d.ts.map +1 -1
- package/dist/lib/embedders/fastembed.js +18 -19
- package/dist/lib/embedders/fastembed.js.map +3 -3
- package/dist/lib/embedders/huggingface.d.ts.map +1 -1
- package/dist/lib/embedders/huggingface.js +5 -3
- package/dist/lib/embedders/huggingface.js.map +2 -2
- package/dist/lib/similiarty-search.d.ts +7 -1
- package/dist/lib/similiarty-search.d.ts.map +1 -1
- package/dist/lib/similiarty-search.js.map +2 -2
- package/dist/lib/stores/sqlite/sqlite.d.ts +2 -2
- package/dist/lib/stores/sqlite/sqlite.d.ts.map +1 -1
- package/dist/lib/stores/sqlite/sqlite.js +20 -12
- package/dist/lib/stores/sqlite/sqlite.js.map +2 -2
- package/dist/lib/stores/store.d.ts +7 -1
- package/dist/lib/stores/store.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"github.d.ts","sourceRoot":"","sources":["../../../src/lib/connectors/github.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhD,UAAU,mBAAmB;IAC3B,uEAAuE;IACvE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,4DAA4D;IAC5D,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,8CAA8C;IAC9C,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,2CAA2C;IAC3C,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;
|
|
1
|
+
{"version":3,"file":"github.d.ts","sourceRoot":"","sources":["../../../src/lib/connectors/github.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhD,UAAU,mBAAmB;IAC3B,uEAAuE;IACvE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,4DAA4D;IAC5D,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,8CAA8C;IAC9C,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,2CAA2C;IAC3C,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B;AA0FD,eAAO,MAAM,MAAM;mBACF,MAAM,GAAG,SAAS;IAcjC;;;;OAIG;oBACa,MAAM,YAAW,mBAAmB,KAAQ,SAAS;IAmBrE;;;OAGG;oBAEQ,MAAM,QACT;QACJ,QAAQ,EAAE,MAAM,EAAE,CAAC;QACnB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;QACpB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,UAAU,CAAC,EAAE,OAAO,GAAG,gBAAgB,CAAC;KACzC,KACA,SAAS;CA8Db,CAAC"}
|
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
import { execa } from "execa";
|
|
2
|
+
function isRelease(value) {
|
|
3
|
+
return typeof value === "object" && value !== null && "tag_name" in value && typeof value.tag_name === "string" && "name" in value && (value.name === null || typeof value.name === "string") && "body" in value && (value.body === null || typeof value.body === "string") && "updated_at" in value && typeof value.updated_at === "string" && "published_at" in value && typeof value.published_at === "string" && "html_url" in value && typeof value.html_url === "string" && "draft" in value && typeof value.draft === "boolean" && "prerelease" in value && typeof value.prerelease === "boolean";
|
|
4
|
+
}
|
|
2
5
|
function fs(path) {
|
|
3
6
|
const [owner, repo, ...filePath] = path.split("/");
|
|
4
7
|
return {
|
|
@@ -6,6 +9,9 @@ function fs(path) {
|
|
|
6
9
|
const url = `https://api.github.com/repos/${owner}/${repo}/contents/${filePath.join("/")}`;
|
|
7
10
|
const res = await fetch(url);
|
|
8
11
|
const data = await res.json();
|
|
12
|
+
if (typeof data !== "object" || data === null || !("content" in data) || typeof data.content !== "string") {
|
|
13
|
+
throw new Error(`GitHub returned no file content for ${path}`);
|
|
14
|
+
}
|
|
9
15
|
return atob(data.content);
|
|
10
16
|
},
|
|
11
17
|
release: async (repo2, opts = {}) => {
|
|
@@ -21,6 +27,9 @@ function fs(path) {
|
|
|
21
27
|
});
|
|
22
28
|
if (!res.ok) break;
|
|
23
29
|
const pageData = await res.json();
|
|
30
|
+
if (!Array.isArray(pageData) || !pageData.every(isRelease)) {
|
|
31
|
+
throw new Error(`Unexpected GitHub releases response for ${repo2}`);
|
|
32
|
+
}
|
|
24
33
|
if (!pageData.length) break;
|
|
25
34
|
for (const rel of pageData) {
|
|
26
35
|
if (!opts.includeDrafts && rel.draft) continue;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../src/lib/connectors/github.ts"],
|
|
4
|
-
"sourcesContent": ["import { execa } from 'execa';\n\nimport type { Connector } from './connector.ts';\n\ninterface ReleaseFetchOptions {\n /** Stop fetching when this tag is encountered (latest-first order). */\n untilTag?: string;\n /** Include the matching untilTag release (default true). */\n inclusive?: boolean;\n /** Include draft releases (default false). */\n includeDrafts?: boolean;\n /** Include prereleases (default false). */\n includePrerelease?: boolean;\n}\n\nfunction fs(path: string) {\n const [owner, repo, ...filePath] = path.split('/');\n return {\n readFile: async () => {\n const url = `https://api.github.com/repos/${owner}/${repo}/contents/${filePath.join('/')}`;\n const res = await fetch(url);\n const data =
|
|
5
|
-
"mappings": "AAAA,SAAS,aAAa;
|
|
4
|
+
"sourcesContent": ["import { execa } from 'execa';\n\nimport type { Connector } from './connector.ts';\n\ninterface ReleaseFetchOptions {\n /** Stop fetching when this tag is encountered (latest-first order). */\n untilTag?: string;\n /** Include the matching untilTag release (default true). */\n inclusive?: boolean;\n /** Include draft releases (default false). */\n includeDrafts?: boolean;\n /** Include prereleases (default false). */\n includePrerelease?: boolean;\n}\n\ninterface Release {\n tag_name: string;\n name: string | null;\n body: string | null;\n updated_at: string;\n published_at: string;\n html_url: string;\n draft: boolean;\n prerelease: boolean;\n}\n\nfunction isRelease(value: unknown): value is Release {\n return (\n typeof value === 'object' &&\n value !== null &&\n 'tag_name' in value &&\n typeof value.tag_name === 'string' &&\n 'name' in value &&\n (value.name === null || typeof value.name === 'string') &&\n 'body' in value &&\n (value.body === null || typeof value.body === 'string') &&\n 'updated_at' in value &&\n typeof value.updated_at === 'string' &&\n 'published_at' in value &&\n typeof value.published_at === 'string' &&\n 'html_url' in value &&\n typeof value.html_url === 'string' &&\n 'draft' in value &&\n typeof value.draft === 'boolean' &&\n 'prerelease' in value &&\n typeof value.prerelease === 'boolean'\n );\n}\n\nfunction fs(path: string) {\n const [owner, repo, ...filePath] = path.split('/');\n return {\n readFile: async () => {\n const url = `https://api.github.com/repos/${owner}/${repo}/contents/${filePath.join('/')}`;\n const res = await fetch(url);\n const data: unknown = await res.json();\n if (\n typeof data !== 'object' ||\n data === null ||\n !('content' in data) ||\n typeof data.content !== 'string'\n ) {\n throw new Error(`GitHub returned no file content for ${path}`);\n }\n return atob(data.content);\n },\n release: async (repo: string, opts: ReleaseFetchOptions = {}) => {\n const [owner, repoName] = repo.split('/');\n const perPage = 100; // always fetch max per page for efficiency\n const maxPages = 10; // internal safety cap, not user configurable\n const releases: Release[] = [];\n let stop = false;\n\n for (let page = 1; page <= maxPages && !stop; page++) {\n const url = `https://api.github.com/repos/${owner}/${repoName}/releases?per_page=${perPage}&page=${page}`;\n const res = await fetch(url, {\n headers: { Accept: 'application/vnd.github+json' },\n });\n if (!res.ok) break;\n const pageData: unknown = await res.json();\n if (!Array.isArray(pageData) || !pageData.every(isRelease)) {\n throw new Error(`Unexpected GitHub releases response for ${repo}`);\n }\n if (!pageData.length) break;\n for (const rel of pageData) {\n if (!opts.includeDrafts && rel.draft) continue;\n if (!opts.includePrerelease && rel.prerelease) continue;\n\n releases.push(rel);\n if (opts.untilTag && rel.tag_name === opts.untilTag) {\n if (opts.inclusive === false) {\n releases.pop();\n }\n stop = true;\n break;\n }\n }\n }\n return releases;\n },\n };\n}\n\nexport const github = {\n file(filePath: string): Connector {\n const sourceId = `github:file:${filePath}`;\n const documentId = filePath;\n return {\n sourceId,\n sources: async function* () {\n const { readFile } = fs(filePath);\n yield {\n id: documentId,\n content: () => readFile(),\n };\n },\n };\n },\n /**\n * Create a connector over GitHub Releases with optional pagination & early-stop.\n * @param repo owner/repo string\n * @param options Release filtering & pagination controls\n */\n release: (repo: string, options: ReleaseFetchOptions = {}): Connector => {\n const sourceId = `github:releases:${repo}`;\n return {\n sourceId,\n sources: async function* () {\n const releases = await fs(repo).release(repo, options);\n for (const rel of releases) {\n yield {\n id: `${repo}:release:${rel.tag_name}`,\n content: async () => {\n const name = rel.name || rel.tag_name;\n const body = rel.body || '';\n return `Release: ${name}\\nTag: ${rel.tag_name}\\nPublished at: ${rel.published_at}\\nUpdated at: ${rel.updated_at}\\nURL: ${rel.html_url}\\nDraft: ${rel.draft}\\nPrerelease: ${rel.prerelease}\\n\\n${body}`;\n },\n };\n }\n },\n };\n },\n /**\n * Create a connector over a GitHub repository file tree using gitingest to produce a markdown digest.\n * @param repoUrl full https://github.com/OWNER/REPO or .../tree/BRANCH/subdir URL\n */\n repo: (\n repoUrl: string,\n opts: {\n includes: string[];\n excludes?: string[];\n branch?: string;\n includeGitignored?: boolean;\n includeSubmodules?: boolean;\n githubToken?: string;\n ingestWhen?: 'never' | 'contentChanged';\n },\n ): Connector => {\n const sourceId = `github:repo:${repoUrl}`;\n const {\n excludes = [\n '**/node_modules/**',\n '**/dist/**',\n '**/coverage/**',\n '**/*.test.ts',\n '**/*.test.tsx',\n '**/dist/**',\n '**/.git/**',\n '**/.github/**',\n '**/.vscode/**',\n '**/.idea/**',\n '**/build/**',\n '**/out/**',\n '**/vendor/**',\n '**/__tests__/**',\n '**/*.d.ts',\n ],\n branch,\n includeGitignored = false,\n includeSubmodules = false,\n githubToken,\n ingestWhen,\n includes,\n } = opts;\n\n async function gitingestDigest(): Promise<string> {\n const args = [\n 'gitingest',\n ...(branch ? ['-b', branch] : []),\n ...includes.flatMap((p) => ['-i', p]),\n ...excludes.flatMap((p) => ['-e', p]),\n ...(includeGitignored ? ['--include-gitignored'] : []),\n ...(includeSubmodules ? ['--include-submodules'] : []),\n repoUrl,\n '-o',\n '-',\n ];\n const { stdout } = await execa('uvx', args, {\n env: githubToken\n ? { ...process.env, GITHUB_TOKEN: githubToken }\n : process.env,\n stdout: 'pipe',\n stderr: 'inherit',\n });\n return stdout;\n }\n\n return {\n sourceId,\n ingestWhen,\n sources: async function* () {\n const content = await gitingestDigest();\n yield {\n id: 'digest.md',\n content: async () => content,\n };\n },\n };\n },\n};\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,aAAa;AA0BtB,SAAS,UAAU,OAAkC;AACnD,SACE,OAAO,UAAU,YACjB,UAAU,QACV,cAAc,SACd,OAAO,MAAM,aAAa,YAC1B,UAAU,UACT,MAAM,SAAS,QAAQ,OAAO,MAAM,SAAS,aAC9C,UAAU,UACT,MAAM,SAAS,QAAQ,OAAO,MAAM,SAAS,aAC9C,gBAAgB,SAChB,OAAO,MAAM,eAAe,YAC5B,kBAAkB,SAClB,OAAO,MAAM,iBAAiB,YAC9B,cAAc,SACd,OAAO,MAAM,aAAa,YAC1B,WAAW,SACX,OAAO,MAAM,UAAU,aACvB,gBAAgB,SAChB,OAAO,MAAM,eAAe;AAEhC;AAEA,SAAS,GAAG,MAAc;AACxB,QAAM,CAAC,OAAO,MAAM,GAAG,QAAQ,IAAI,KAAK,MAAM,GAAG;AACjD,SAAO;AAAA,IACL,UAAU,YAAY;AACpB,YAAM,MAAM,gCAAgC,KAAK,IAAI,IAAI,aAAa,SAAS,KAAK,GAAG,CAAC;AACxF,YAAM,MAAM,MAAM,MAAM,GAAG;AAC3B,YAAM,OAAgB,MAAM,IAAI,KAAK;AACrC,UACE,OAAO,SAAS,YAChB,SAAS,QACT,EAAE,aAAa,SACf,OAAO,KAAK,YAAY,UACxB;AACA,cAAM,IAAI,MAAM,uCAAuC,IAAI,EAAE;AAAA,MAC/D;AACA,aAAO,KAAK,KAAK,OAAO;AAAA,IAC1B;AAAA,IACA,SAAS,OAAOA,OAAc,OAA4B,CAAC,MAAM;AAC/D,YAAM,CAACC,QAAO,QAAQ,IAAID,MAAK,MAAM,GAAG;AACxC,YAAM,UAAU;AAChB,YAAM,WAAW;AACjB,YAAM,WAAsB,CAAC;AAC7B,UAAI,OAAO;AAEX,eAAS,OAAO,GAAG,QAAQ,YAAY,CAAC,MAAM,QAAQ;AACpD,cAAM,MAAM,gCAAgCC,MAAK,IAAI,QAAQ,sBAAsB,OAAO,SAAS,IAAI;AACvG,cAAM,MAAM,MAAM,MAAM,KAAK;AAAA,UAC3B,SAAS,EAAE,QAAQ,8BAA8B;AAAA,QACnD,CAAC;AACD,YAAI,CAAC,IAAI,GAAI;AACb,cAAM,WAAoB,MAAM,IAAI,KAAK;AACzC,YAAI,CAAC,MAAM,QAAQ,QAAQ,KAAK,CAAC,SAAS,MAAM,SAAS,GAAG;AAC1D,gBAAM,IAAI,MAAM,2CAA2CD,KAAI,EAAE;AAAA,QACnE;AACA,YAAI,CAAC,SAAS,OAAQ;AACtB,mBAAW,OAAO,UAAU;AAC1B,cAAI,CAAC,KAAK,iBAAiB,IAAI,MAAO;AACtC,cAAI,CAAC,KAAK,qBAAqB,IAAI,WAAY;AAE/C,mBAAS,KAAK,GAAG;AACjB,cAAI,KAAK,YAAY,IAAI,aAAa,KAAK,UAAU;AACnD,gBAAI,KAAK,cAAc,OAAO;AAC5B,uBAAS,IAAI;AAAA,YACf;AACA,mBAAO;AACP;AAAA,UACF;AAAA,QACF;AAAA,MACF;AACA,aAAO;AAAA,IACT;AAAA,EACF;AACF;AAEO,MAAM,SAAS;AAAA,EACpB,KAAK,UAA6B;AAChC,UAAM,WAAW,eAAe,QAAQ;AACxC,UAAM,aAAa;AACnB,WAAO;AAAA,MACL;AAAA,MACA,SAAS,mBAAmB;AAC1B,cAAM,EAAE,SAAS,IAAI,GAAG,QAAQ;AAChC,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,SAAS,MAAM,SAAS;AAAA,QAC1B;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMA,SAAS,CAAC,MAAc,UAA+B,CAAC,MAAiB;AACvE,UAAM,WAAW,mBAAmB,IAAI;AACxC,WAAO;AAAA,MACL;AAAA,MACA,SAAS,mBAAmB;AAC1B,cAAM,WAAW,MAAM,GAAG,IAAI,EAAE,QAAQ,MAAM,OAAO;AACrD,mBAAW,OAAO,UAAU;AAC1B,gBAAM;AAAA,YACJ,IAAI,GAAG,IAAI,YAAY,IAAI,QAAQ;AAAA,YACnC,SAAS,YAAY;AACnB,oBAAM,OAAO,IAAI,QAAQ,IAAI;AAC7B,oBAAM,OAAO,IAAI,QAAQ;AACzB,qBAAO,YAAY,IAAI;AAAA,OAAU,IAAI,QAAQ;AAAA,gBAAmB,IAAI,YAAY;AAAA,cAAiB,IAAI,UAAU;AAAA,OAAU,IAAI,QAAQ;AAAA,SAAY,IAAI,KAAK;AAAA,cAAiB,IAAI,UAAU;AAAA;AAAA,EAAO,IAAI;AAAA,YACtM;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,CACJ,SACA,SASc;AACd,UAAM,WAAW,eAAe,OAAO;AACvC,UAAM;AAAA,MACJ,WAAW;AAAA,QACT;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,MACA;AAAA,MACA,oBAAoB;AAAA,MACpB,oBAAoB;AAAA,MACpB;AAAA,MACA;AAAA,MACA;AAAA,IACF,IAAI;AAEJ,mBAAe,kBAAmC;AAChD,YAAM,OAAO;AAAA,QACX;AAAA,QACA,GAAI,SAAS,CAAC,MAAM,MAAM,IAAI,CAAC;AAAA,QAC/B,GAAG,SAAS,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AAAA,QACpC,GAAG,SAAS,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AAAA,QACpC,GAAI,oBAAoB,CAAC,sBAAsB,IAAI,CAAC;AAAA,QACpD,GAAI,oBAAoB,CAAC,sBAAsB,IAAI,CAAC;AAAA,QACpD;AAAA,QACA;AAAA,QACA;AAAA,MACF;AACA,YAAM,EAAE,OAAO,IAAI,MAAM,MAAM,OAAO,MAAM;AAAA,QAC1C,KAAK,cACD,EAAE,GAAG,QAAQ,KAAK,cAAc,YAAY,IAC5C,QAAQ;AAAA,QACZ,QAAQ;AAAA,QACR,QAAQ;AAAA,MACV,CAAC;AACD,aAAO;AAAA,IACT;AAEA,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,SAAS,mBAAmB;AAC1B,cAAM,UAAU,MAAM,gBAAgB;AACtC,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,SAAS,YAAY;AAAA,QACvB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;",
|
|
6
6
|
"names": ["repo", "owner"]
|
|
7
7
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../../src/lib/connectors/repo.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhD,wBAAgB,IAAI,CAClB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,EAAE,EACpB,UAAU,EAAE,SAAS,CAAC,YAAY,CAAC,GAClC,SAAS,CAqBX;AAED,wBAAuB,eAAe,CAAC,IAAI,EAAE,MAAM,yCAoClD;AAmBD,wBAAsB,cAAc,CAAC,IAAI,EAAE,MAAM,qBA4FhD;AAED,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,EACZ,UAAU,EAAE,MAAM,EAAE,GACnB,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,
|
|
1
|
+
{"version":3,"file":"repo.d.ts","sourceRoot":"","sources":["../../../src/lib/connectors/repo.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAEhD,wBAAgB,IAAI,CAClB,GAAG,EAAE,MAAM,EACX,UAAU,EAAE,MAAM,EAAE,EACpB,UAAU,EAAE,SAAS,CAAC,YAAY,CAAC,GAClC,SAAS,CAqBX;AAED,wBAAuB,eAAe,CAAC,IAAI,EAAE,MAAM,yCAoClD;AAmBD,wBAAsB,cAAc,CAAC,IAAI,EAAE,MAAM,qBA4FhD;AAED,wBAAsB,YAAY,CAChC,IAAI,EAAE,MAAM,EACZ,UAAU,EAAE,MAAM,EAAE,GACnB,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAqBhC"}
|
|
@@ -150,7 +150,7 @@ async function ignorePatterns(repo2) {
|
|
|
150
150
|
}
|
|
151
151
|
async function collectFiles(repo2, extensions) {
|
|
152
152
|
const exts = extensions.map((ext) => ext.replace(/^\./, ""));
|
|
153
|
-
|
|
153
|
+
const entries = fg.stream(
|
|
154
154
|
extensions.length > 1 ? `**/*.{${exts.join(",")}}` : `**/*.${exts[0]}`,
|
|
155
155
|
{
|
|
156
156
|
dot: false,
|
|
@@ -161,6 +161,14 @@ async function collectFiles(repo2, extensions) {
|
|
|
161
161
|
ignore: await ignorePatterns(repo2)
|
|
162
162
|
}
|
|
163
163
|
);
|
|
164
|
+
return (async function* () {
|
|
165
|
+
for await (const entry of entries) {
|
|
166
|
+
if (typeof entry !== "string") {
|
|
167
|
+
throw new TypeError("fast-glob yielded a non-path entry");
|
|
168
|
+
}
|
|
169
|
+
yield entry;
|
|
170
|
+
}
|
|
171
|
+
})();
|
|
164
172
|
}
|
|
165
173
|
export {
|
|
166
174
|
collectFiles,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../src/lib/connectors/repo.ts"],
|
|
4
|
-
"sourcesContent": ["import fg from 'fast-glob';\nimport { opendir, readFile, stat } from 'node:fs/promises';\nimport { join } from 'node:path';\n\nimport type { Connector } from './connector.js';\n\nexport function repo(\n dir: string,\n extensions: string[],\n ingestWhen: Connector['ingestWhen'],\n): Connector {\n const sourceId = `repo:${dir}`;\n return {\n sourceId,\n ingestWhen,\n sources: async function* () {\n const paths = await collectFiles(dir, extensions);\n for await (const path of paths) {\n const maxSize = 3 * 1024; // 3KB\n const st = await stat(path);\n if (st.size > maxSize) {\n continue;\n }\n yield {\n id: path,\n metadata: { repo: dir },\n content: () => readFile(path, 'utf8').catch(() => ''),\n };\n }\n },\n };\n}\n\nexport async function* findAllGitRepos(root: string) {\n const stack = [root];\n const skip = [\n 'node_modules',\n 'Library',\n 'Applications',\n 'Pictures',\n 'Movies',\n 'Music',\n 'Downloads',\n '.cache',\n '.npm',\n '.pnpm',\n 'development',\n ];\n while (stack.length) {\n const folder = stack.pop()!;\n const isGitRepo = await stat(join(folder, '.git'))\n .then((st) => st.isDirectory() || st.isFile())\n .catch(() => false);\n if (isGitRepo) {\n yield folder;\n continue;\n }\n const dir = await opendir(folder);\n for await (const dirent of dir) {\n if (\n dirent.isDirectory() &&\n !dirent.isSymbolicLink() &&\n !skip.includes(dirent.name) &&\n !dirent.name.startsWith('.')\n ) {\n stack.push(join(folder, dirent.name));\n }\n }\n }\n}\n\nfunction detectRepoTooling(repo: string) {\n const tools: string[] = [];\n return tools;\n}\n\nasync function gitignore(gitignoreFile: string) {\n const content = await readFile(gitignoreFile, 'utf8').catch(() => '');\n if (!content) {\n return [];\n }\n const patterns = content\n .split('\\n')\n .map((line) => line.trim())\n .filter((line) => line && !line.startsWith('#'));\n return patterns;\n}\n\nexport async function ignorePatterns(repo: string) {\n return [\n // Inherit repo-specific ignore patterns\n ...(await gitignore(join(repo, '.gitignore'))),\n\n // Package managers & dependency dirs\n 'node_modules/**',\n '**/node_modules/**',\n '**/.pnpm/**',\n '**/.npm/**',\n '**/.yarn/**',\n '**/vendor/**', // PHP / Go modules (when vendored)\n '**/3rdparty/**',\n\n // Version control + VCS metadata\n '**/.git/**',\n '**/.svn/**',\n '**/.hg/**',\n\n // OS / system junk\n '**/.DS_Store',\n '**/Thumbs.db',\n '**/Library/**',\n '**/Applications/**',\n '**/Pictures/**',\n '**/Movies/**',\n '**/Music/**',\n '**/Downloads/**',\n '**/.cache/**',\n\n // Environment / secrets (explicit)\n '**/.env',\n '**/.env.*',\n\n // Lockfiles & generated dependency state\n '**/*.lock',\n '**/yarn.lock',\n '**/package-lock.json',\n '**/pnpm-lock.yaml',\n\n // Build / compilation outputs\n '**/dist/**',\n '**/debug/**',\n '**/build/**',\n '**/out/**',\n '**/target/**', // Rust / JVM\n '**/bin/**',\n '**/obj/**',\n '**/classes/**',\n\n // Framework / tool specific build artifacts\n '**/.next/**',\n '**/.vercel/**',\n '**/.turbo/**',\n '**/.docusaurus/**',\n '**/.vite/**',\n '**/.parcel-cache/**',\n '**/.rollup.cache/**',\n '**/.vuepress/**',\n 'cdk.out/**',\n\n // Infra & deployment tooling\n '**/.serverless/**',\n '**/.terraform/**',\n '**/.terragrunt-cache/**',\n '**/.pulumi/**',\n\n // Coverage & testing caches\n '**/coverage/**',\n '**/.nyc_output/**',\n '**/jest-cache/**',\n '**/.pytest_cache/**',\n\n // Language / tooling caches\n '**/__pycache__/**',\n '**/.mypy_cache/**',\n '**/.tox/**',\n '**/.gradle/**',\n '**/.mvn/**',\n '**/.eslintcache',\n '**/.stylelintcache',\n\n // IDE / editor configs + history (we don't want to embed these)\n '**/.idea/**',\n '**/.vscode/**',\n '**/.fleet/**',\n '**/.history/**',\n\n // Virtual environments\n '**/.venv/**',\n '**/venv/**',\n ];\n}\n\nexport async function collectFiles(\n repo: string,\n extensions: string[],\n): Promise<AsyncIterable<string>> {\n const exts = extensions.map((ext) => ext.replace(/^\\./, ''));\n
|
|
5
|
-
"mappings": "AAAA,OAAO,QAAQ;AACf,SAAS,SAAS,UAAU,YAAY;AACxC,SAAS,YAAY;AAId,SAAS,KACd,KACA,YACA,YACW;AACX,QAAM,WAAW,QAAQ,GAAG;AAC5B,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,SAAS,mBAAmB;AAC1B,YAAM,QAAQ,MAAM,aAAa,KAAK,UAAU;AAChD,uBAAiB,QAAQ,OAAO;AAC9B,cAAM,UAAU,IAAI;AACpB,cAAM,KAAK,MAAM,KAAK,IAAI;AAC1B,YAAI,GAAG,OAAO,SAAS;AACrB;AAAA,QACF;AACA,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,UAAU,EAAE,MAAM,IAAI;AAAA,UACtB,SAAS,MAAM,SAAS,MAAM,MAAM,EAAE,MAAM,MAAM,EAAE;AAAA,QACtD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,gBAAuB,gBAAgB,MAAc;AACnD,QAAM,QAAQ,CAAC,IAAI;AACnB,QAAM,OAAO;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,SAAO,MAAM,QAAQ;AACnB,UAAM,SAAS,MAAM,IAAI;AACzB,UAAM,YAAY,MAAM,KAAK,KAAK,QAAQ,MAAM,CAAC,EAC9C,KAAK,CAAC,OAAO,GAAG,YAAY,KAAK,GAAG,OAAO,CAAC,EAC5C,MAAM,MAAM,KAAK;AACpB,QAAI,WAAW;AACb,YAAM;AACN;AAAA,IACF;AACA,UAAM,MAAM,MAAM,QAAQ,MAAM;AAChC,qBAAiB,UAAU,KAAK;AAC9B,UACE,OAAO,YAAY,KACnB,CAAC,OAAO,eAAe,KACvB,CAAC,KAAK,SAAS,OAAO,IAAI,KAC1B,CAAC,OAAO,KAAK,WAAW,GAAG,GAC3B;AACA,cAAM,KAAK,KAAK,QAAQ,OAAO,IAAI,CAAC;AAAA,MACtC;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,kBAAkBA,OAAc;AACvC,QAAM,QAAkB,CAAC;AACzB,SAAO;AACT;AAEA,eAAe,UAAU,eAAuB;AAC9C,QAAM,UAAU,MAAM,SAAS,eAAe,MAAM,EAAE,MAAM,MAAM,EAAE;AACpE,MAAI,CAAC,SAAS;AACZ,WAAO,CAAC;AAAA,EACV;AACA,QAAM,WAAW,QACd,MAAM,IAAI,EACV,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,OAAO,CAAC,SAAS,QAAQ,CAAC,KAAK,WAAW,GAAG,CAAC;AACjD,SAAO;AACT;AAEA,eAAsB,eAAeA,OAAc;AACjD,SAAO;AAAA;AAAA,IAEL,GAAI,MAAM,UAAU,KAAKA,OAAM,YAAY,CAAC;AAAA;AAAA,IAG5C;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,EACF;AACF;AAEA,eAAsB,aACpBA,OACA,YACgC;AAChC,QAAM,OAAO,WAAW,IAAI,CAAC,QAAQ,IAAI,QAAQ,OAAO,EAAE,CAAC;AAC3D,
|
|
4
|
+
"sourcesContent": ["import fg from 'fast-glob';\nimport { opendir, readFile, stat } from 'node:fs/promises';\nimport { join } from 'node:path';\n\nimport type { Connector } from './connector.js';\n\nexport function repo(\n dir: string,\n extensions: string[],\n ingestWhen: Connector['ingestWhen'],\n): Connector {\n const sourceId = `repo:${dir}`;\n return {\n sourceId,\n ingestWhen,\n sources: async function* () {\n const paths = await collectFiles(dir, extensions);\n for await (const path of paths) {\n const maxSize = 3 * 1024; // 3KB\n const st = await stat(path);\n if (st.size > maxSize) {\n continue;\n }\n yield {\n id: path,\n metadata: { repo: dir },\n content: () => readFile(path, 'utf8').catch(() => ''),\n };\n }\n },\n };\n}\n\nexport async function* findAllGitRepos(root: string) {\n const stack = [root];\n const skip = [\n 'node_modules',\n 'Library',\n 'Applications',\n 'Pictures',\n 'Movies',\n 'Music',\n 'Downloads',\n '.cache',\n '.npm',\n '.pnpm',\n 'development',\n ];\n while (stack.length) {\n const folder = stack.pop()!;\n const isGitRepo = await stat(join(folder, '.git'))\n .then((st) => st.isDirectory() || st.isFile())\n .catch(() => false);\n if (isGitRepo) {\n yield folder;\n continue;\n }\n const dir = await opendir(folder);\n for await (const dirent of dir) {\n if (\n dirent.isDirectory() &&\n !dirent.isSymbolicLink() &&\n !skip.includes(dirent.name) &&\n !dirent.name.startsWith('.')\n ) {\n stack.push(join(folder, dirent.name));\n }\n }\n }\n}\n\nfunction detectRepoTooling(repo: string) {\n const tools: string[] = [];\n return tools;\n}\n\nasync function gitignore(gitignoreFile: string) {\n const content = await readFile(gitignoreFile, 'utf8').catch(() => '');\n if (!content) {\n return [];\n }\n const patterns = content\n .split('\\n')\n .map((line) => line.trim())\n .filter((line) => line && !line.startsWith('#'));\n return patterns;\n}\n\nexport async function ignorePatterns(repo: string) {\n return [\n // Inherit repo-specific ignore patterns\n ...(await gitignore(join(repo, '.gitignore'))),\n\n // Package managers & dependency dirs\n 'node_modules/**',\n '**/node_modules/**',\n '**/.pnpm/**',\n '**/.npm/**',\n '**/.yarn/**',\n '**/vendor/**', // PHP / Go modules (when vendored)\n '**/3rdparty/**',\n\n // Version control + VCS metadata\n '**/.git/**',\n '**/.svn/**',\n '**/.hg/**',\n\n // OS / system junk\n '**/.DS_Store',\n '**/Thumbs.db',\n '**/Library/**',\n '**/Applications/**',\n '**/Pictures/**',\n '**/Movies/**',\n '**/Music/**',\n '**/Downloads/**',\n '**/.cache/**',\n\n // Environment / secrets (explicit)\n '**/.env',\n '**/.env.*',\n\n // Lockfiles & generated dependency state\n '**/*.lock',\n '**/yarn.lock',\n '**/package-lock.json',\n '**/pnpm-lock.yaml',\n\n // Build / compilation outputs\n '**/dist/**',\n '**/debug/**',\n '**/build/**',\n '**/out/**',\n '**/target/**', // Rust / JVM\n '**/bin/**',\n '**/obj/**',\n '**/classes/**',\n\n // Framework / tool specific build artifacts\n '**/.next/**',\n '**/.vercel/**',\n '**/.turbo/**',\n '**/.docusaurus/**',\n '**/.vite/**',\n '**/.parcel-cache/**',\n '**/.rollup.cache/**',\n '**/.vuepress/**',\n 'cdk.out/**',\n\n // Infra & deployment tooling\n '**/.serverless/**',\n '**/.terraform/**',\n '**/.terragrunt-cache/**',\n '**/.pulumi/**',\n\n // Coverage & testing caches\n '**/coverage/**',\n '**/.nyc_output/**',\n '**/jest-cache/**',\n '**/.pytest_cache/**',\n\n // Language / tooling caches\n '**/__pycache__/**',\n '**/.mypy_cache/**',\n '**/.tox/**',\n '**/.gradle/**',\n '**/.mvn/**',\n '**/.eslintcache',\n '**/.stylelintcache',\n\n // IDE / editor configs + history (we don't want to embed these)\n '**/.idea/**',\n '**/.vscode/**',\n '**/.fleet/**',\n '**/.history/**',\n\n // Virtual environments\n '**/.venv/**',\n '**/venv/**',\n ];\n}\n\nexport async function collectFiles(\n repo: string,\n extensions: string[],\n): Promise<AsyncIterable<string>> {\n const exts = extensions.map((ext) => ext.replace(/^\\./, ''));\n const entries = fg.stream(\n extensions.length > 1 ? `**/*.{${exts.join(',')}}` : `**/*.${exts[0]}`,\n {\n dot: false,\n onlyFiles: true,\n unique: true,\n absolute: true,\n cwd: repo,\n ignore: await ignorePatterns(repo),\n },\n );\n return (async function* () {\n for await (const entry of entries) {\n if (typeof entry !== 'string') {\n throw new TypeError('fast-glob yielded a non-path entry');\n }\n yield entry;\n }\n })();\n}\n"],
|
|
5
|
+
"mappings": "AAAA,OAAO,QAAQ;AACf,SAAS,SAAS,UAAU,YAAY;AACxC,SAAS,YAAY;AAId,SAAS,KACd,KACA,YACA,YACW;AACX,QAAM,WAAW,QAAQ,GAAG;AAC5B,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,SAAS,mBAAmB;AAC1B,YAAM,QAAQ,MAAM,aAAa,KAAK,UAAU;AAChD,uBAAiB,QAAQ,OAAO;AAC9B,cAAM,UAAU,IAAI;AACpB,cAAM,KAAK,MAAM,KAAK,IAAI;AAC1B,YAAI,GAAG,OAAO,SAAS;AACrB;AAAA,QACF;AACA,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,UAAU,EAAE,MAAM,IAAI;AAAA,UACtB,SAAS,MAAM,SAAS,MAAM,MAAM,EAAE,MAAM,MAAM,EAAE;AAAA,QACtD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,gBAAuB,gBAAgB,MAAc;AACnD,QAAM,QAAQ,CAAC,IAAI;AACnB,QAAM,OAAO;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,SAAO,MAAM,QAAQ;AACnB,UAAM,SAAS,MAAM,IAAI;AACzB,UAAM,YAAY,MAAM,KAAK,KAAK,QAAQ,MAAM,CAAC,EAC9C,KAAK,CAAC,OAAO,GAAG,YAAY,KAAK,GAAG,OAAO,CAAC,EAC5C,MAAM,MAAM,KAAK;AACpB,QAAI,WAAW;AACb,YAAM;AACN;AAAA,IACF;AACA,UAAM,MAAM,MAAM,QAAQ,MAAM;AAChC,qBAAiB,UAAU,KAAK;AAC9B,UACE,OAAO,YAAY,KACnB,CAAC,OAAO,eAAe,KACvB,CAAC,KAAK,SAAS,OAAO,IAAI,KAC1B,CAAC,OAAO,KAAK,WAAW,GAAG,GAC3B;AACA,cAAM,KAAK,KAAK,QAAQ,OAAO,IAAI,CAAC;AAAA,MACtC;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,kBAAkBA,OAAc;AACvC,QAAM,QAAkB,CAAC;AACzB,SAAO;AACT;AAEA,eAAe,UAAU,eAAuB;AAC9C,QAAM,UAAU,MAAM,SAAS,eAAe,MAAM,EAAE,MAAM,MAAM,EAAE;AACpE,MAAI,CAAC,SAAS;AACZ,WAAO,CAAC;AAAA,EACV;AACA,QAAM,WAAW,QACd,MAAM,IAAI,EACV,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,OAAO,CAAC,SAAS,QAAQ,CAAC,KAAK,WAAW,GAAG,CAAC;AACjD,SAAO;AACT;AAEA,eAAsB,eAAeA,OAAc;AACjD,SAAO;AAAA;AAAA,IAEL,GAAI,MAAM,UAAU,KAAKA,OAAM,YAAY,CAAC;AAAA;AAAA,IAG5C;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,EACF;AACF;AAEA,eAAsB,aACpBA,OACA,YACgC;AAChC,QAAM,OAAO,WAAW,IAAI,CAAC,QAAQ,IAAI,QAAQ,OAAO,EAAE,CAAC;AAC3D,QAAM,UAAU,GAAG;AAAA,IACjB,WAAW,SAAS,IAAI,SAAS,KAAK,KAAK,GAAG,CAAC,MAAM,QAAQ,KAAK,CAAC,CAAC;AAAA,IACpE;AAAA,MACE,KAAK;AAAA,MACL,WAAW;AAAA,MACX,QAAQ;AAAA,MACR,UAAU;AAAA,MACV,KAAKA;AAAA,MACL,QAAQ,MAAM,eAAeA,KAAI;AAAA,IACnC;AAAA,EACF;AACA,UAAQ,mBAAmB;AACzB,qBAAiB,SAAS,SAAS;AACjC,UAAI,OAAO,UAAU,UAAU;AAC7B,cAAM,IAAI,UAAU,oCAAoC;AAAA,MAC1D;AACA,YAAM;AAAA,IACR;AAAA,EACF,GAAG;AACL;",
|
|
6
6
|
"names": ["repo"]
|
|
7
7
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"rss.d.ts","sourceRoot":"","sources":["../../../src/lib/connectors/rss.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"rss.d.ts","sourceRoot":"","sources":["../../../src/lib/connectors/rss.ts"],"names":[],"mappings":"AAsHA,wBAAgB,GAAG,CACjB,OAAO,EAAE,MAAM,EACf,OAAO,GAAE;IACP,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,OAAO,CAAC;CACxB;;;;;;;EA0DP"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../src/lib/connectors/rss.ts"],
|
|
4
|
-
"sourcesContent": ["import { Readability } from '@mozilla/readability';\nimport Parser from 'rss-parser';\n\nconst rssParser = new Parser({\n customFields: {\n feed: ['copyright', 'language', 'managingEditor', 'webMaster'],\n item: ['category', 'creator', 'enclosure', 'guid'],\n },\n});\n\nasync function parseRSSFeed(feedUrl: string) {\n try {\n console.log(`Fetching RSS feed: ${feedUrl}`);\n const feed = await rssParser.parseURL(feedUrl);\n\n return {\n title: feed.title || '',\n description: feed.description || '',\n link: feed.link || '',\n language: feed.language || 'en',\n lastBuildDate:
|
|
5
|
-
"mappings": "AAAA,SAAS,mBAAmB;AAC5B,OAAO,YAAY;
|
|
4
|
+
"sourcesContent": ["import { Readability } from '@mozilla/readability';\nimport Parser from 'rss-parser';\n\ntype FeedFields = {\n copyright?: string;\n language?: string;\n managingEditor?: string;\n webMaster?: string;\n lastBuildDate?: string;\n};\n\ntype ItemFields = {\n category?: string;\n creator?: string;\n enclosure?: Parser.Enclosure;\n guid?: string;\n author?: string;\n 'content:encoded'?: string;\n};\n\nconst rssParser = new Parser<FeedFields, ItemFields>({\n customFields: {\n feed: ['copyright', 'language', 'managingEditor', 'webMaster'],\n item: ['category', 'creator', 'enclosure', 'guid'],\n },\n});\n\nasync function parseRSSFeed(feedUrl: string) {\n try {\n console.log(`Fetching RSS feed: ${feedUrl}`);\n const feed = await rssParser.parseURL(feedUrl);\n\n return {\n title: feed.title || '',\n description: feed.description || '',\n link: feed.link || '',\n language: feed.language || 'en',\n lastBuildDate: feed.lastBuildDate || new Date().toISOString(),\n items: feed.items.map((item) => ({\n title: item.title || '',\n description: item.content || item.summary || item.contentSnippet || '',\n link: item.link || '',\n pubDate: item.pubDate || item.isoDate || '',\n author: item.creator || item.author || '',\n categories: Array.isArray(item.categories)\n ? item.categories\n : item.category\n ? [item.category]\n : [],\n guid: item.guid || item.guid || '',\n contentEncoded: item['content:encoded'] || item.content || '',\n })),\n };\n } catch (error) {\n console.error(`Failed to parse RSS feed ${feedUrl}:`, error);\n throw new Error(\n `RSS parsing failed: ${error instanceof Error ? error.message : 'Unknown error'}`,\n );\n }\n}\n\nasync function fetchArticleContent(url: string): Promise<string> {\n try {\n console.log(`Fetching full article content from: ${url}`);\n\n const response = await fetch(url, {\n headers: {\n 'User-Agent': 'Mozilla/5.0 (compatible; RSS-RAG-Bot/1.0)',\n Accept:\n 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n 'Accept-Language': 'en-US,en;q=0.5',\n 'Accept-Encoding': 'gzip, deflate',\n Connection: 'keep-alive',\n },\n signal: AbortSignal.timeout(10000), // 10 second timeout\n });\n\n if (!response.ok) {\n throw new Error(`HTTP ${response.status}: ${response.statusText}`);\n }\n\n const html = await response.text();\n\n const { JSDOM } = await import('jsdom');\n const dom = new JSDOM(html, { url });\n const document = dom.window.document;\n\n // Use Mozilla's Readability to extract the main content\n const reader = new Readability(document);\n const article = reader.parse();\n\n if (!article) {\n throw new Error('Readability failed to extract article content');\n }\n\n // Combine title and content for comprehensive text\n const fullContent = `${article.title ? article.title + '\\n\\n' : ''}${article.textContent || article.content}`;\n\n // Ensure we have meaningful content (at least 200 characters for full articles)\n if (fullContent.length < 200) {\n throw new Error(\n `Extracted content too short (${fullContent.length} chars) - likely failed to find main content`,\n );\n }\n\n console.log(\n `\u2713 Successfully extracted article: \"${article.title}\" (${fullContent.length} chars, ~${Math.round(fullContent.split(' ').length)} words)`,\n );\n\n return fullContent;\n } catch (error) {\n console.warn(\n `Failed to fetch article content from ${url}:`,\n error instanceof Error ? error.message : error,\n );\n return ''; // Return empty string on failure rather than throwing\n }\n}\nexport function rss(\n feedUrl: string,\n options: {\n maxItems?: number;\n fetchFullArticles?: boolean;\n } = {},\n) {\n const sourceId = `rss:${feedUrl}`;\n const { maxItems = 50, fetchFullArticles = false } = options;\n\n return {\n sourceId,\n instructions: `You answer questions about articles and content from the RSS feed: ${feedUrl}.\n Always cite the article title and link when referencing specific content.\n The feed contains recent articles, blog posts, and news items.\n When referencing content, include the publication date and author when available.\n ${fetchFullArticles ? 'Full article content has been extracted from the original links for comprehensive analysis.' : 'Content includes RSS summaries and descriptions.'}`,\n sources: async function* () {\n const feed = await parseRSSFeed(feedUrl);\n // Add feed summary source\n const feedSummary = `RSS Feed: ${feed.title}\nDescription: ${feed.description}\nWebsite: ${feed.link}\nLanguage: ${feed.language}\nLast Updated: ${feed.lastBuildDate}\nTotal Items: ${feed.items.length}\n\nThis feed provides: ${feed.description}`;\n yield {\n id: 'feed-info',\n content: async () => feedSummary,\n };\n\n // Individual article sources (limit to maxItems)\n const itemsToProcess = feed.items.slice(0, maxItems);\n for (const item of itemsToProcess) {\n const documentId =\n item.guid || item.link || `${item.title}-${item.pubDate}`;\n yield {\n id: documentId,\n content: async () => {\n // Try full article fetch if enabled, fallback to RSS content\n let articleContent = item.contentEncoded || item.description;\n if (fetchFullArticles && item.link) {\n const fullContent = await fetchArticleContent(item.link);\n if (fullContent && fullContent.length > articleContent.length) {\n articleContent = fullContent;\n }\n }\n return `Title: ${item.title}\nAuthor: ${item.author}\nPublished: ${item.pubDate}\nCategories: ${item.categories.join(', ')}\nLink: ${item.link}\n${fetchFullArticles ? 'Full Article Content:' : 'Content:'}\n${articleContent}\n\nSummary: ${item.title} - ${item.description}`;\n },\n };\n }\n },\n };\n}\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,mBAAmB;AAC5B,OAAO,YAAY;AAmBnB,MAAM,YAAY,IAAI,OAA+B;AAAA,EACnD,cAAc;AAAA,IACZ,MAAM,CAAC,aAAa,YAAY,kBAAkB,WAAW;AAAA,IAC7D,MAAM,CAAC,YAAY,WAAW,aAAa,MAAM;AAAA,EACnD;AACF,CAAC;AAED,eAAe,aAAa,SAAiB;AAC3C,MAAI;AACF,YAAQ,IAAI,sBAAsB,OAAO,EAAE;AAC3C,UAAM,OAAO,MAAM,UAAU,SAAS,OAAO;AAE7C,WAAO;AAAA,MACL,OAAO,KAAK,SAAS;AAAA,MACrB,aAAa,KAAK,eAAe;AAAA,MACjC,MAAM,KAAK,QAAQ;AAAA,MACnB,UAAU,KAAK,YAAY;AAAA,MAC3B,eAAe,KAAK,kBAAiB,oBAAI,KAAK,GAAE,YAAY;AAAA,MAC5D,OAAO,KAAK,MAAM,IAAI,CAAC,UAAU;AAAA,QAC/B,OAAO,KAAK,SAAS;AAAA,QACrB,aAAa,KAAK,WAAW,KAAK,WAAW,KAAK,kBAAkB;AAAA,QACpE,MAAM,KAAK,QAAQ;AAAA,QACnB,SAAS,KAAK,WAAW,KAAK,WAAW;AAAA,QACzC,QAAQ,KAAK,WAAW,KAAK,UAAU;AAAA,QACvC,YAAY,MAAM,QAAQ,KAAK,UAAU,IACrC,KAAK,aACL,KAAK,WACH,CAAC,KAAK,QAAQ,IACd,CAAC;AAAA,QACP,MAAM,KAAK,QAAQ,KAAK,QAAQ;AAAA,QAChC,gBAAgB,KAAK,iBAAiB,KAAK,KAAK,WAAW;AAAA,MAC7D,EAAE;AAAA,IACJ;AAAA,EACF,SAAS,OAAO;AACd,YAAQ,MAAM,4BAA4B,OAAO,KAAK,KAAK;AAC3D,UAAM,IAAI;AAAA,MACR,uBAAuB,iBAAiB,QAAQ,MAAM,UAAU,eAAe;AAAA,IACjF;AAAA,EACF;AACF;AAEA,eAAe,oBAAoB,KAA8B;AAC/D,MAAI;AACF,YAAQ,IAAI,uCAAuC,GAAG,EAAE;AAExD,UAAM,WAAW,MAAM,MAAM,KAAK;AAAA,MAChC,SAAS;AAAA,QACP,cAAc;AAAA,QACd,QACE;AAAA,QACF,mBAAmB;AAAA,QACnB,mBAAmB;AAAA,QACnB,YAAY;AAAA,MACd;AAAA,MACA,QAAQ,YAAY,QAAQ,GAAK;AAAA;AAAA,IACnC,CAAC;AAED,QAAI,CAAC,SAAS,IAAI;AAChB,YAAM,IAAI,MAAM,QAAQ,SAAS,MAAM,KAAK,SAAS,UAAU,EAAE;AAAA,IACnE;AAEA,UAAM,OAAO,MAAM,SAAS,KAAK;AAEjC,UAAM,EAAE,MAAM,IAAI,MAAM,OAAO,OAAO;AACtC,UAAM,MAAM,IAAI,MAAM,MAAM,EAAE,IAAI,CAAC;AACnC,UAAM,WAAW,IAAI,OAAO;AAG5B,UAAM,SAAS,IAAI,YAAY,QAAQ;AACvC,UAAM,UAAU,OAAO,MAAM;AAE7B,QAAI,CAAC,SAAS;AACZ,YAAM,IAAI,MAAM,+CAA+C;AAAA,IACjE;AAGA,UAAM,cAAc,GAAG,QAAQ,QAAQ,QAAQ,QAAQ,SAAS,EAAE,GAAG,QAAQ,eAAe,QAAQ,OAAO;AAG3G,QAAI,YAAY,SAAS,KAAK;AAC5B,YAAM,IAAI;AAAA,QACR,gCAAgC,YAAY,MAAM;AAAA,MACpD;AAAA,IACF;AAEA,YAAQ;AAAA,MACN,2CAAsC,QAAQ,KAAK,MAAM,YAAY,MAAM,YAAY,KAAK,MAAM,YAAY,MAAM,GAAG,EAAE,MAAM,CAAC;AAAA,IAClI;AAEA,WAAO;AAAA,EACT,SAAS,OAAO;AACd,YAAQ;AAAA,MACN,wCAAwC,GAAG;AAAA,MAC3C,iBAAiB,QAAQ,MAAM,UAAU;AAAA,IAC3C;AACA,WAAO;AAAA,EACT;AACF;AACO,SAAS,IACd,SACA,UAGI,CAAC,GACL;AACA,QAAM,WAAW,OAAO,OAAO;AAC/B,QAAM,EAAE,WAAW,IAAI,oBAAoB,MAAM,IAAI;AAErD,SAAO;AAAA,IACL;AAAA,IACA,cAAc,sEAAsE,OAAO;AAAA;AAAA;AAAA;AAAA,QAIvF,oBAAoB,gGAAgG,kDAAkD;AAAA,IAC1K,SAAS,mBAAmB;AAC1B,YAAM,OAAO,MAAM,aAAa,OAAO;AAEvC,YAAM,cAAc,aAAa,KAAK,KAAK;AAAA,eAClC,KAAK,WAAW;AAAA,WACpB,KAAK,IAAI;AAAA,YACR,KAAK,QAAQ;AAAA,gBACT,KAAK,aAAa;AAAA,eACnB,KAAK,MAAM,MAAM;AAAA;AAAA,sBAEV,KAAK,WAAW;AAChC,YAAM;AAAA,QACJ,IAAI;AAAA,QACJ,SAAS,YAAY;AAAA,MACvB;AAGA,YAAM,iBAAiB,KAAK,MAAM,MAAM,GAAG,QAAQ;AACnD,iBAAW,QAAQ,gBAAgB;AACjC,cAAM,aACJ,KAAK,QAAQ,KAAK,QAAQ,GAAG,KAAK,KAAK,IAAI,KAAK,OAAO;AACzD,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,SAAS,YAAY;AAEnB,gBAAI,iBAAiB,KAAK,kBAAkB,KAAK;AACjD,gBAAI,qBAAqB,KAAK,MAAM;AAClC,oBAAM,cAAc,MAAM,oBAAoB,KAAK,IAAI;AACvD,kBAAI,eAAe,YAAY,SAAS,eAAe,QAAQ;AAC7D,iCAAiB;AAAA,cACnB;AAAA,YACF;AACA,mBAAO,UAAU,KAAK,KAAK;AAAA,UAC7B,KAAK,MAAM;AAAA,aACR,KAAK,OAAO;AAAA,cACX,KAAK,WAAW,KAAK,IAAI,CAAC;AAAA,QAChC,KAAK,IAAI;AAAA,EACf,oBAAoB,0BAA0B,UAAU;AAAA,EACxD,cAAc;AAAA;AAAA,WAEL,KAAK,KAAK,MAAM,KAAK,WAAW;AAAA,UACjC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
|
@@ -2,15 +2,16 @@ import type { Embedder } from '../stores/store.js';
|
|
|
2
2
|
/**
|
|
3
3
|
* Stable model identifiers retained from the former `fastembed` adapter.
|
|
4
4
|
*/
|
|
5
|
-
export declare
|
|
6
|
-
AllMiniLML6V2
|
|
7
|
-
BGEBaseEN
|
|
8
|
-
BGEBaseENV15
|
|
9
|
-
BGESmallEN
|
|
10
|
-
BGESmallENV15
|
|
11
|
-
BGESmallZH
|
|
12
|
-
MLE5Large
|
|
13
|
-
}
|
|
5
|
+
export declare const EmbeddingModels: {
|
|
6
|
+
readonly AllMiniLML6V2: "fast-all-MiniLM-L6-v2";
|
|
7
|
+
readonly BGEBaseEN: "fast-bge-base-en";
|
|
8
|
+
readonly BGEBaseENV15: "fast-bge-base-en-v1.5";
|
|
9
|
+
readonly BGESmallEN: "fast-bge-small-en";
|
|
10
|
+
readonly BGESmallENV15: "fast-bge-small-en-v1.5";
|
|
11
|
+
readonly BGESmallZH: "fast-bge-small-zh-v1.5";
|
|
12
|
+
readonly MLE5Large: "fast-multilingual-e5-large";
|
|
13
|
+
};
|
|
14
|
+
export type EmbeddingModel = (typeof EmbeddingModels)[keyof typeof EmbeddingModels];
|
|
14
15
|
export interface FastEmbedOptions {
|
|
15
16
|
model?: EmbeddingModel;
|
|
16
17
|
batchSize?: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fastembed.d.ts","sourceRoot":"","sources":["../../../src/lib/embedders/fastembed.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAGnD;;GAEG;AACH,
|
|
1
|
+
{"version":3,"file":"fastembed.d.ts","sourceRoot":"","sources":["../../../src/lib/embedders/fastembed.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAGnD;;GAEG;AACH,eAAO,MAAM,eAAe;;;;;;;;CAQlB,CAAC;AACX,MAAM,MAAM,cAAc,GACxB,CAAC,OAAO,eAAe,CAAC,CAAC,MAAM,OAAO,eAAe,CAAC,CAAC;AAYzD,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,cAAc,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;;;GAIG;AACH,wBAAgB,SAAS,CAAC,OAAO,GAAE,gBAAqB,GAAG,QAAQ,CA4ClE"}
|
|
@@ -1,26 +1,25 @@
|
|
|
1
1
|
import { tensorToEmbeddings } from "./huggingface.js";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
})(EmbeddingModel || {});
|
|
2
|
+
const EmbeddingModels = {
|
|
3
|
+
AllMiniLML6V2: "fast-all-MiniLM-L6-v2",
|
|
4
|
+
BGEBaseEN: "fast-bge-base-en",
|
|
5
|
+
BGEBaseENV15: "fast-bge-base-en-v1.5",
|
|
6
|
+
BGESmallEN: "fast-bge-small-en",
|
|
7
|
+
BGESmallENV15: "fast-bge-small-en-v1.5",
|
|
8
|
+
BGESmallZH: "fast-bge-small-zh-v1.5",
|
|
9
|
+
MLE5Large: "fast-multilingual-e5-large"
|
|
10
|
+
};
|
|
12
11
|
const transformerModels = {
|
|
13
|
-
[
|
|
14
|
-
[
|
|
15
|
-
[
|
|
16
|
-
[
|
|
17
|
-
[
|
|
18
|
-
[
|
|
19
|
-
[
|
|
12
|
+
[EmbeddingModels.AllMiniLML6V2]: "Xenova/all-MiniLM-L6-v2",
|
|
13
|
+
[EmbeddingModels.BGEBaseEN]: "Xenova/bge-base-en",
|
|
14
|
+
[EmbeddingModels.BGEBaseENV15]: "Xenova/bge-base-en-v1.5",
|
|
15
|
+
[EmbeddingModels.BGESmallEN]: "Xenova/bge-small-en",
|
|
16
|
+
[EmbeddingModels.BGESmallENV15]: "Xenova/bge-small-en-v1.5",
|
|
17
|
+
[EmbeddingModels.BGESmallZH]: "Xenova/bge-small-zh-v1.5",
|
|
18
|
+
[EmbeddingModels.MLE5Large]: "Xenova/multilingual-e5-large"
|
|
20
19
|
};
|
|
21
20
|
function fastembed(options = {}) {
|
|
22
21
|
const {
|
|
23
|
-
model: modelId =
|
|
22
|
+
model: modelId = EmbeddingModels.BGESmallENV15,
|
|
24
23
|
batchSize = 256,
|
|
25
24
|
cacheDir
|
|
26
25
|
} = options;
|
|
@@ -58,7 +57,7 @@ function fastembed(options = {}) {
|
|
|
58
57
|
};
|
|
59
58
|
}
|
|
60
59
|
export {
|
|
61
|
-
|
|
60
|
+
EmbeddingModels,
|
|
62
61
|
fastembed
|
|
63
62
|
};
|
|
64
63
|
//# sourceMappingURL=fastembed.js.map
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../src/lib/embedders/fastembed.ts"],
|
|
4
|
-
"sourcesContent": ["import type { FeatureExtractionPipeline } from '@huggingface/transformers';\n\nimport type { Embedder } from '../stores/store.js';\nimport { tensorToEmbeddings } from './huggingface.js';\n\n/**\n * Stable model identifiers retained from the former `fastembed` adapter.\n */\nexport
|
|
5
|
-
"mappings": "AAGA,SAAS,0BAA0B;AAK5B,
|
|
6
|
-
"names": [
|
|
4
|
+
"sourcesContent": ["import type { FeatureExtractionPipeline } from '@huggingface/transformers';\n\nimport type { Embedder } from '../stores/store.js';\nimport { tensorToEmbeddings } from './huggingface.js';\n\n/**\n * Stable model identifiers retained from the former `fastembed` adapter.\n */\nexport const EmbeddingModels = {\n AllMiniLML6V2: 'fast-all-MiniLM-L6-v2',\n BGEBaseEN: 'fast-bge-base-en',\n BGEBaseENV15: 'fast-bge-base-en-v1.5',\n BGESmallEN: 'fast-bge-small-en',\n BGESmallENV15: 'fast-bge-small-en-v1.5',\n BGESmallZH: 'fast-bge-small-zh-v1.5',\n MLE5Large: 'fast-multilingual-e5-large',\n} as const;\nexport type EmbeddingModel =\n (typeof EmbeddingModels)[keyof typeof EmbeddingModels];\n\nconst transformerModels: Record<EmbeddingModel, string> = {\n [EmbeddingModels.AllMiniLML6V2]: 'Xenova/all-MiniLM-L6-v2',\n [EmbeddingModels.BGEBaseEN]: 'Xenova/bge-base-en',\n [EmbeddingModels.BGEBaseENV15]: 'Xenova/bge-base-en-v1.5',\n [EmbeddingModels.BGESmallEN]: 'Xenova/bge-small-en',\n [EmbeddingModels.BGESmallENV15]: 'Xenova/bge-small-en-v1.5',\n [EmbeddingModels.BGESmallZH]: 'Xenova/bge-small-zh-v1.5',\n [EmbeddingModels.MLE5Large]: 'Xenova/multilingual-e5-large',\n};\n\nexport interface FastEmbedOptions {\n model?: EmbeddingModel;\n batchSize?: number;\n cacheDir?: string;\n}\n\n/**\n * Local feature extraction backed by Transformers.js. The historical function\n * and model identifiers remain stable while avoiding FastEmbed's archived tar\n * extraction dependency.\n */\nexport function fastembed(options: FastEmbedOptions = {}): Embedder {\n const {\n model: modelId = EmbeddingModels.BGESmallENV15,\n batchSize = 256,\n cacheDir,\n } = options;\n\n if (!Number.isInteger(batchSize) || batchSize <= 0) {\n throw new RangeError('batchSize must be a positive integer');\n }\n\n let modelPromise: Promise<FeatureExtractionPipeline> | null = null;\n const getModel = () => {\n if (!modelPromise) {\n modelPromise = import('@huggingface/transformers').then(({ pipeline }) =>\n pipeline('feature-extraction', transformerModels[modelId], {\n cache_dir: cacheDir,\n }),\n );\n }\n return modelPromise;\n };\n\n return async (documents: string[]) => {\n if (documents.length === 0) {\n return { embeddings: [], dimensions: 0 };\n }\n\n const model = await getModel();\n const embeddings: Float32Array[] = [];\n let dimensions = 0;\n\n for (let start = 0; start < documents.length; start += batchSize) {\n const tensor = await model(documents.slice(start, start + batchSize), {\n pooling: 'mean',\n normalize: true,\n });\n const batch = tensorToEmbeddings(tensor);\n dimensions ||= batch.dimensions;\n embeddings.push(...batch.embeddings);\n }\n\n return { embeddings, dimensions };\n };\n}\n"],
|
|
5
|
+
"mappings": "AAGA,SAAS,0BAA0B;AAK5B,MAAM,kBAAkB;AAAA,EAC7B,eAAe;AAAA,EACf,WAAW;AAAA,EACX,cAAc;AAAA,EACd,YAAY;AAAA,EACZ,eAAe;AAAA,EACf,YAAY;AAAA,EACZ,WAAW;AACb;AAIA,MAAM,oBAAoD;AAAA,EACxD,CAAC,gBAAgB,aAAa,GAAG;AAAA,EACjC,CAAC,gBAAgB,SAAS,GAAG;AAAA,EAC7B,CAAC,gBAAgB,YAAY,GAAG;AAAA,EAChC,CAAC,gBAAgB,UAAU,GAAG;AAAA,EAC9B,CAAC,gBAAgB,aAAa,GAAG;AAAA,EACjC,CAAC,gBAAgB,UAAU,GAAG;AAAA,EAC9B,CAAC,gBAAgB,SAAS,GAAG;AAC/B;AAaO,SAAS,UAAU,UAA4B,CAAC,GAAa;AAClE,QAAM;AAAA,IACJ,OAAO,UAAU,gBAAgB;AAAA,IACjC,YAAY;AAAA,IACZ;AAAA,EACF,IAAI;AAEJ,MAAI,CAAC,OAAO,UAAU,SAAS,KAAK,aAAa,GAAG;AAClD,UAAM,IAAI,WAAW,sCAAsC;AAAA,EAC7D;AAEA,MAAI,eAA0D;AAC9D,QAAM,WAAW,MAAM;AACrB,QAAI,CAAC,cAAc;AACjB,qBAAe,OAAO,2BAA2B,EAAE;AAAA,QAAK,CAAC,EAAE,SAAS,MAClE,SAAS,sBAAsB,kBAAkB,OAAO,GAAG;AAAA,UACzD,WAAW;AAAA,QACb,CAAC;AAAA,MACH;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAEA,SAAO,OAAO,cAAwB;AACpC,QAAI,UAAU,WAAW,GAAG;AAC1B,aAAO,EAAE,YAAY,CAAC,GAAG,YAAY,EAAE;AAAA,IACzC;AAEA,UAAM,QAAQ,MAAM,SAAS;AAC7B,UAAM,aAA6B,CAAC;AACpC,QAAI,aAAa;AAEjB,aAAS,QAAQ,GAAG,QAAQ,UAAU,QAAQ,SAAS,WAAW;AAChE,YAAM,SAAS,MAAM,MAAM,UAAU,MAAM,OAAO,QAAQ,SAAS,GAAG;AAAA,QACpE,SAAS;AAAA,QACT,WAAW;AAAA,MACb,CAAC;AACD,YAAM,QAAQ,mBAAmB,MAAM;AACvC,qBAAe,MAAM;AACrB,iBAAW,KAAK,GAAG,MAAM,UAAU;AAAA,IACrC;AAEA,WAAO,EAAE,YAAY,WAAW;AAAA,EAClC;AACF;",
|
|
6
|
+
"names": []
|
|
7
7
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"huggingface.d.ts","sourceRoot":"","sources":["../../../src/lib/embedders/huggingface.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,yBAAyB,EACzB,MAAM,EACP,MAAM,2BAA2B,CAAC;AAEnC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAEnD,MAAM,MAAM,mBAAmB,GAAG,yBAAyB,CAAC;AAE5D,MAAM,WAAW,gBAAgB;IAC/B,WAAW,EAAE,
|
|
1
|
+
{"version":3,"file":"huggingface.d.ts","sourceRoot":"","sources":["../../../src/lib/embedders/huggingface.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,yBAAyB,EACzB,MAAM,EACP,MAAM,2BAA2B,CAAC;AAEnC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAEnD,MAAM,MAAM,mBAAmB,GAAG,yBAAyB,CAAC;AAE5D,MAAM,WAAW,gBAAgB;IAC/B,WAAW,EAAE,MACX,OAAO,CAAC,yBAAyB,CAAC,GAAG,yBAAyB,CAAC;IACjE,OAAO,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;IACzB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAeD,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM;;;EAyBhD;AAED,wBAAgB,WAAW,CACzB,WAAW,EAAE,MACX,OAAO,CAAC,yBAAyB,CAAC,GAAG,yBAAyB,GAC/D,QAAQ,CAEV"}
|
|
@@ -20,13 +20,15 @@ function tensorToEmbeddings(tensor) {
|
|
|
20
20
|
`Data length mismatch: got ${tensor.data.length}, expected ${expectedLen} (batch=${batchSize}, hidden=${hiddenSize})`
|
|
21
21
|
);
|
|
22
22
|
}
|
|
23
|
+
const data = tensor.data;
|
|
24
|
+
if (!(data instanceof Float32Array)) {
|
|
25
|
+
throw new Error(`Expected float32 embeddings, got ${tensor.type}`);
|
|
26
|
+
}
|
|
23
27
|
const embeddings = [];
|
|
24
28
|
const dimensions = tensor.dims[tensor.dims.length - 1];
|
|
25
29
|
for (let i = 0; i < batchSize; i++) {
|
|
26
30
|
const start = i * hiddenSize;
|
|
27
|
-
embeddings.push(
|
|
28
|
-
tensor.data.subarray(start, start + hiddenSize)
|
|
29
|
-
);
|
|
31
|
+
embeddings.push(data.subarray(start, start + hiddenSize));
|
|
30
32
|
}
|
|
31
33
|
return { embeddings, dimensions };
|
|
32
34
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../src/lib/embedders/huggingface.ts"],
|
|
4
|
-
"sourcesContent": ["import type {\n FeatureExtractionPipeline,\n Tensor,\n} from '@huggingface/transformers';\n\nimport type { Embedder } from '../stores/store.js';\n\nexport type FeatureExtractionFn = FeatureExtractionPipeline;\n\nexport interface EmbeddingOptions {\n extractorFn: () =>\n
|
|
5
|
-
"mappings": "
|
|
4
|
+
"sourcesContent": ["import type {\n FeatureExtractionPipeline,\n Tensor,\n} from '@huggingface/transformers';\n\nimport type { Embedder } from '../stores/store.js';\n\nexport type FeatureExtractionFn = FeatureExtractionPipeline;\n\nexport interface EmbeddingOptions {\n extractorFn: () =>\n Promise<FeatureExtractionPipeline> | FeatureExtractionPipeline;\n pooling?: 'mean' | 'cls';\n normalize?: boolean;\n}\n\nasync function textEmbeddings(\n inputs: string | string[],\n { extractorFn, pooling, normalize }: EmbeddingOptions,\n) {\n inputs = (Array.isArray(inputs) ? inputs : [inputs]).map((it) => it.trim());\n const extractor = await extractorFn();\n const tensor = await extractor(inputs, {\n pooling: pooling ?? 'mean',\n normalize: normalize ?? true,\n });\n return tensorToEmbeddings(tensor);\n}\n\nexport function tensorToEmbeddings(tensor: Tensor) {\n const dims = tensor.dims;\n if (!Array.isArray(dims) || dims.length < 2) {\n throw new Error(`Unexpected tensor dims: ${JSON.stringify(dims)}`);\n }\n const batchSize = dims[0];\n const hiddenSize = dims[dims.length - 1];\n const expectedLen = batchSize * hiddenSize;\n if (tensor.data.length !== expectedLen) {\n throw new Error(\n `Data length mismatch: got ${tensor.data.length}, expected ${expectedLen} (batch=${batchSize}, hidden=${hiddenSize})`,\n );\n }\n const data = tensor.data;\n if (!(data instanceof Float32Array)) {\n throw new Error(`Expected float32 embeddings, got ${tensor.type}`);\n }\n // Reuse the underlying typed array without copying; we'll copy when creating Buffer for SQLite\n const embeddings: Float32Array[] = [];\n const dimensions = tensor.dims[tensor.dims.length - 1];\n for (let i = 0; i < batchSize; i++) {\n const start = i * hiddenSize;\n embeddings.push(data.subarray(start, start + hiddenSize));\n }\n return { embeddings, dimensions };\n}\n\nexport function huggingface(\n extractorFn: () =>\n Promise<FeatureExtractionPipeline> | FeatureExtractionPipeline,\n): Embedder {\n return (documents) => textEmbeddings(documents, { extractorFn });\n}\n"],
|
|
5
|
+
"mappings": "AAgBA,eAAe,eACb,QACA,EAAE,aAAa,SAAS,UAAU,GAClC;AACA,YAAU,MAAM,QAAQ,MAAM,IAAI,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;AAC1E,QAAM,YAAY,MAAM,YAAY;AACpC,QAAM,SAAS,MAAM,UAAU,QAAQ;AAAA,IACrC,SAAS,WAAW;AAAA,IACpB,WAAW,aAAa;AAAA,EAC1B,CAAC;AACD,SAAO,mBAAmB,MAAM;AAClC;AAEO,SAAS,mBAAmB,QAAgB;AACjD,QAAM,OAAO,OAAO;AACpB,MAAI,CAAC,MAAM,QAAQ,IAAI,KAAK,KAAK,SAAS,GAAG;AAC3C,UAAM,IAAI,MAAM,2BAA2B,KAAK,UAAU,IAAI,CAAC,EAAE;AAAA,EACnE;AACA,QAAM,YAAY,KAAK,CAAC;AACxB,QAAM,aAAa,KAAK,KAAK,SAAS,CAAC;AACvC,QAAM,cAAc,YAAY;AAChC,MAAI,OAAO,KAAK,WAAW,aAAa;AACtC,UAAM,IAAI;AAAA,MACR,6BAA6B,OAAO,KAAK,MAAM,cAAc,WAAW,WAAW,SAAS,YAAY,UAAU;AAAA,IACpH;AAAA,EACF;AACA,QAAM,OAAO,OAAO;AACpB,MAAI,EAAE,gBAAgB,eAAe;AACnC,UAAM,IAAI,MAAM,oCAAoC,OAAO,IAAI,EAAE;AAAA,EACnE;AAEA,QAAM,aAA6B,CAAC;AACpC,QAAM,aAAa,OAAO,KAAK,OAAO,KAAK,SAAS,CAAC;AACrD,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,IAAI;AAClB,eAAW,KAAK,KAAK,SAAS,OAAO,QAAQ,UAAU,CAAC;AAAA,EAC1D;AACA,SAAO,EAAE,YAAY,WAAW;AAClC;AAEO,SAAS,YACd,aAEU;AACV,SAAO,CAAC,cAAc,eAAe,WAAW,EAAE,YAAY,CAAC;AACjE;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
|
@@ -1,3 +1,9 @@
|
|
|
1
1
|
import { type IngestionConfig } from './ingest.js';
|
|
2
|
-
export declare function similaritySearch(query: string, config: Omit<IngestionConfig, 'splitter'>): Promise<
|
|
2
|
+
export declare function similaritySearch(query: string, config: Omit<IngestionConfig, 'splitter'>): Promise<{
|
|
3
|
+
similarity: number;
|
|
4
|
+
distance: number;
|
|
5
|
+
content: string;
|
|
6
|
+
document_id: string;
|
|
7
|
+
metadata: Record<string, unknown> | null;
|
|
8
|
+
}[]>;
|
|
3
9
|
//# sourceMappingURL=similiarty-search.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"similiarty-search.d.ts","sourceRoot":"","sources":["../../src/lib/similiarty-search.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,eAAe,EAAU,MAAM,aAAa,CAAC;AAE3D,wBAAsB,gBAAgB,CACpC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,IAAI,CAAC,eAAe,EAAE,UAAU,CAAC
|
|
1
|
+
{"version":3,"file":"similiarty-search.d.ts","sourceRoot":"","sources":["../../src/lib/similiarty-search.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,eAAe,EAAU,MAAM,aAAa,CAAC;AAE3D,wBAAsB,gBAAgB,CACpC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,IAAI,CAAC,eAAe,EAAE,UAAU,CAAC;;;;;;KAgD1C"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../src/lib/similiarty-search.ts"],
|
|
4
|
-
"sourcesContent": ["import { embedMany } from 'ai';\n\nimport { type IngestionConfig, ingest } from './ingest.js';\n\nexport async function similaritySearch(\n query: string,\n config: Omit<IngestionConfig, 'splitter'>,\n) {\n // Ingest if needed, then perform vector similarity search via the configured store\n const mode = config.connector.ingestWhen ?? 'contentChanged';\n let shouldIngest = true;\n\n if (mode === 'never') {\n if (await config.store.sourceExists(config.connector.sourceId)) {\n console.log(\n `Skipping ingestion for source ${config.connector.sourceId} (ingestWhen=never and source exists)`,\n );\n shouldIngest = false;\n }\n } else if (mode === 'expired') {\n const sourceExists = await config.store.sourceExists(\n config.connector.sourceId,\n );\n if (\n sourceExists &&\n !(await config.store.sourceExpired(config.connector.sourceId))\n ) {\n console.log(\n `Skipping ingestion for source ${config.connector.sourceId} (ingestWhen=expired and source not expired)`,\n );\n shouldIngest = false;\n }\n }\n\n // Calculate expiry date if connector specifies expiresAfter\n const expiryDate = config.connector.expiresAfter\n ? new Date(Date.now() + config.connector.expiresAfter)\n : undefined;\n if (shouldIngest) {\n await ingest(config);\n }\n return config.store\n .search(\n query,\n { sourceId: config.connector.sourceId, topN: 50 },\n config.embedder,\n )\n .then(
|
|
5
|
-
"mappings": "AAAA,SAAS,iBAAiB;AAE1B,SAA+B,cAAc;AAE7C,eAAsB,iBACpB,OACA,QACA;AAEA,QAAM,OAAO,OAAO,UAAU,cAAc;AAC5C,MAAI,eAAe;AAEnB,MAAI,SAAS,SAAS;AACpB,QAAI,MAAM,OAAO,MAAM,aAAa,OAAO,UAAU,QAAQ,GAAG;AAC9D,cAAQ;AAAA,QACN,iCAAiC,OAAO,UAAU,QAAQ;AAAA,MAC5D;AACA,qBAAe;AAAA,IACjB;AAAA,EACF,WAAW,SAAS,WAAW;AAC7B,UAAM,eAAe,MAAM,OAAO,MAAM;AAAA,MACtC,OAAO,UAAU;AAAA,IACnB;AACA,QACE,gBACA,CAAE,MAAM,OAAO,MAAM,cAAc,OAAO,UAAU,QAAQ,GAC5D;AACA,cAAQ;AAAA,QACN,iCAAiC,OAAO,UAAU,QAAQ;AAAA,MAC5D;AACA,qBAAe;AAAA,IACjB;AAAA,EACF;AAGA,QAAM,aAAa,OAAO,UAAU,eAChC,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,UAAU,YAAY,IACnD;AACJ,MAAI,cAAc;AAChB,UAAM,OAAO,MAAM;AAAA,EACrB;AACA,SAAO,OAAO,MACX;AAAA,IACC;AAAA,IACA,EAAE,UAAU,OAAO,UAAU,UAAU,MAAM,GAAG;AAAA,IAChD,OAAO;AAAA,EACT,EACC;AAAA,
|
|
4
|
+
"sourcesContent": ["import { embedMany } from 'ai';\n\nimport { type IngestionConfig, ingest } from './ingest.js';\n\nexport async function similaritySearch(\n query: string,\n config: Omit<IngestionConfig, 'splitter'>,\n) {\n // Ingest if needed, then perform vector similarity search via the configured store\n const mode = config.connector.ingestWhen ?? 'contentChanged';\n let shouldIngest = true;\n\n if (mode === 'never') {\n if (await config.store.sourceExists(config.connector.sourceId)) {\n console.log(\n `Skipping ingestion for source ${config.connector.sourceId} (ingestWhen=never and source exists)`,\n );\n shouldIngest = false;\n }\n } else if (mode === 'expired') {\n const sourceExists = await config.store.sourceExists(\n config.connector.sourceId,\n );\n if (\n sourceExists &&\n !(await config.store.sourceExpired(config.connector.sourceId))\n ) {\n console.log(\n `Skipping ingestion for source ${config.connector.sourceId} (ingestWhen=expired and source not expired)`,\n );\n shouldIngest = false;\n }\n }\n\n // Calculate expiry date if connector specifies expiresAfter\n const expiryDate = config.connector.expiresAfter\n ? new Date(Date.now() + config.connector.expiresAfter)\n : undefined;\n if (shouldIngest) {\n await ingest(config);\n }\n return config.store\n .search(\n query,\n { sourceId: config.connector.sourceId, topN: 50 },\n config.embedder,\n )\n .then((results) =>\n results.map((it) => ({\n ...it,\n similarity: 1 - it.distance,\n distance: it.distance,\n })),\n );\n}\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,iBAAiB;AAE1B,SAA+B,cAAc;AAE7C,eAAsB,iBACpB,OACA,QACA;AAEA,QAAM,OAAO,OAAO,UAAU,cAAc;AAC5C,MAAI,eAAe;AAEnB,MAAI,SAAS,SAAS;AACpB,QAAI,MAAM,OAAO,MAAM,aAAa,OAAO,UAAU,QAAQ,GAAG;AAC9D,cAAQ;AAAA,QACN,iCAAiC,OAAO,UAAU,QAAQ;AAAA,MAC5D;AACA,qBAAe;AAAA,IACjB;AAAA,EACF,WAAW,SAAS,WAAW;AAC7B,UAAM,eAAe,MAAM,OAAO,MAAM;AAAA,MACtC,OAAO,UAAU;AAAA,IACnB;AACA,QACE,gBACA,CAAE,MAAM,OAAO,MAAM,cAAc,OAAO,UAAU,QAAQ,GAC5D;AACA,cAAQ;AAAA,QACN,iCAAiC,OAAO,UAAU,QAAQ;AAAA,MAC5D;AACA,qBAAe;AAAA,IACjB;AAAA,EACF;AAGA,QAAM,aAAa,OAAO,UAAU,eAChC,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,UAAU,YAAY,IACnD;AACJ,MAAI,cAAc;AAChB,UAAM,OAAO,MAAM;AAAA,EACrB;AACA,SAAO,OAAO,MACX;AAAA,IACC;AAAA,IACA,EAAE,UAAU,OAAO,UAAU,UAAU,MAAM,GAAG;AAAA,IAChD,OAAO;AAAA,EACT,EACC;AAAA,IAAK,CAAC,YACL,QAAQ,IAAI,CAAC,QAAQ;AAAA,MACnB,GAAG;AAAA,MACH,YAAY,IAAI,GAAG;AAAA,MACnB,UAAU,GAAG;AAAA,IACf,EAAE;AAAA,EACJ;AACJ;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { Chunk, Corpus, Embedder, SearchOptions, Store } from '../store.js';
|
|
1
|
+
import type { Chunk, Corpus, Embedder, SearchOptions, SearchResult, Store } from '../store.js';
|
|
2
2
|
interface DB {
|
|
3
3
|
prepare: (sql: string) => {
|
|
4
4
|
run: (...args: any[]) => any;
|
|
@@ -10,7 +10,7 @@ interface DB {
|
|
|
10
10
|
export declare class SQLiteStore implements Store {
|
|
11
11
|
#private;
|
|
12
12
|
constructor(db: DB, dimension: number);
|
|
13
|
-
search(query: string, options: SearchOptions, embedder: Embedder): Promise<
|
|
13
|
+
search(query: string, options: SearchOptions, embedder: Embedder): Promise<SearchResult[]>;
|
|
14
14
|
upsertDoc(inputs: {
|
|
15
15
|
documentId: string;
|
|
16
16
|
sourceId: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sqlite.d.ts","sourceRoot":"","sources":["../../../../src/lib/stores/sqlite/sqlite.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,KAAK,EACL,MAAM,EACN,QAAQ,EACR,aAAa,EACb,KAAK,EACN,MAAM,aAAa,CAAC;AAKrB,UAAU,EAAE;IACV,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK;QACxB,GAAG,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,CAAC;QAC7B,GAAG,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,EAAE,CAAC;QAC/B,GAAG,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,CAAC;KAC9B,CAAC;IACF,IAAI,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAC7B;AAED,qBAAa,WAAY,YAAW,KAAK;;gBAE3B,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM;IAwD/B,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ;
|
|
1
|
+
{"version":3,"file":"sqlite.d.ts","sourceRoot":"","sources":["../../../../src/lib/stores/sqlite/sqlite.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACV,KAAK,EACL,MAAM,EACN,QAAQ,EACR,aAAa,EACb,YAAY,EACZ,KAAK,EACN,MAAM,aAAa,CAAC;AAKrB,UAAU,EAAE;IACV,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK;QACxB,GAAG,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,CAAC;QAC7B,GAAG,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,EAAE,CAAC;QAC/B,GAAG,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,CAAC;KAC9B,CAAC;IACF,IAAI,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,CAAC;CAC7B;AAED,qBAAa,WAAY,YAAW,KAAK;;gBAE3B,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM;IAwD/B,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ;IAyCtE,SAAS,CAAC,MAAM,EAAE;QAChB,UAAU,EAAE,MAAM,CAAC;QACnB,QAAQ,EAAE,MAAM,CAAC;QACjB,GAAG,EAAE,MAAM,CAAC;QACZ,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KAChC;IAiBD,SAAS,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,IAKhD,OAAO,KAAK;IAStB,MAAM,CAAC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE;IAMvD,YAAY,CAAC,QAAQ,EAAE,MAAM;IAO7B,aAAa,CAAC,QAAQ,EAAE,MAAM;IAW9B,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI;IAQ5C,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,IAAI;CA+ChE;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,YAAY,GAAG,MAAM,CASpE"}
|
|
@@ -63,20 +63,14 @@ class SQLiteStore {
|
|
|
63
63
|
embedding: vectorBlob,
|
|
64
64
|
k: topN ?? DEFAULT_TOP_N
|
|
65
65
|
});
|
|
66
|
-
return rows2.map(
|
|
67
|
-
...r,
|
|
68
|
-
metadata: safeParseMetadata(r.metadata)
|
|
69
|
-
}));
|
|
66
|
+
return rows2.map(toSearchResult);
|
|
70
67
|
}
|
|
71
68
|
const rows = this.#searchBySource({
|
|
72
69
|
sourceId: options.sourceId,
|
|
73
70
|
embedding: vectorBlob,
|
|
74
71
|
k: topN ?? DEFAULT_TOP_N
|
|
75
72
|
});
|
|
76
|
-
return rows.map(
|
|
77
|
-
...r,
|
|
78
|
-
metadata: safeParseMetadata(r.metadata)
|
|
79
|
-
}));
|
|
73
|
+
return rows.map(toSearchResult);
|
|
80
74
|
}
|
|
81
75
|
#upsertSource(inputs) {
|
|
82
76
|
const stmt = this.#db.prepare(dedent`
|
|
@@ -207,15 +201,29 @@ function vectorToBlob(vector) {
|
|
|
207
201
|
const floatArray = new Float32Array(vector);
|
|
208
202
|
return Buffer.from(floatArray.buffer);
|
|
209
203
|
}
|
|
210
|
-
function
|
|
211
|
-
if (
|
|
212
|
-
|
|
204
|
+
function toSearchResult(row) {
|
|
205
|
+
if (typeof row !== "object" || row === null || !("content" in row) || typeof row.content !== "string" || !("distance" in row) || typeof row.distance !== "number" || !("document_id" in row) || typeof row.document_id !== "string" || !("metadata" in row) || row.metadata !== null && typeof row.metadata !== "string") {
|
|
206
|
+
throw new Error("Unexpected vector search row");
|
|
207
|
+
}
|
|
208
|
+
return {
|
|
209
|
+
content: row.content,
|
|
210
|
+
distance: row.distance,
|
|
211
|
+
document_id: row.document_id,
|
|
212
|
+
metadata: parseMetadata(row.metadata)
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
function parseMetadata(value) {
|
|
216
|
+
if (value === null) return null;
|
|
213
217
|
try {
|
|
214
|
-
|
|
218
|
+
const parsed = JSON.parse(value);
|
|
219
|
+
return isRecord(parsed) ? parsed : null;
|
|
215
220
|
} catch {
|
|
216
221
|
return null;
|
|
217
222
|
}
|
|
218
223
|
}
|
|
224
|
+
function isRecord(value) {
|
|
225
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
226
|
+
}
|
|
219
227
|
export {
|
|
220
228
|
SQLiteStore,
|
|
221
229
|
vectorToBlob
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../../src/lib/stores/sqlite/sqlite.ts"],
|
|
4
|
-
"sourcesContent": ["import dedent from 'dedent';\nimport { template } from 'lodash-es';\n\nimport type {\n Chunk,\n Corpus,\n Embedder,\n SearchOptions,\n Store,\n} from '../store.js';\nimport sql from './sqlite.sql.js';\n\nconst DEFAULT_TOP_N = 10;\n\ninterface DB {\n prepare: (sql: string) => {\n run: (...args: any[]) => any;\n all: (...args: any[]) => any[];\n get: (...args: any[]) => any;\n };\n exec: (sql: string) => void;\n}\n\nexport class SQLiteStore implements Store {\n readonly #db: DB;\n constructor(db: DB, dimension: number) {\n this.#db = db;\n const compiled = template(sql);\n this.#db.exec(compiled({ DIMENSION: dimension }));\n }\n\n #transaction(callback: () => void) {\n try {\n this.#db.exec('BEGIN IMMEDIATE');\n callback();\n this.#db.exec('COMMIT');\n } catch (error) {\n this.#db.exec('ROLLBACK');\n throw error;\n }\n }\n\n #searchByDocument(inputs: {\n sourceId: string;\n documentId: string;\n embedding: Buffer<ArrayBufferLike>;\n k: number;\n }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT v.content, v.distance, v.document_id, d.metadata\n\t\t\t\tFROM vec_chunks v\n\t\t\t\tJOIN documents d ON d.id = v.document_id\n\t\t\t\tWHERE v.source_id = ?\n\t\t\t\t\tAND v.document_id = ?\n\t\t\t\t\tAND v.embedding MATCH vec_normalize(vec_f32(?))\n\t\t\t\t\tAND v.k = ?\n\t\t\t\tORDER BY v.distance ASC\n\t\t\t`);\n return stmt.all(\n inputs.sourceId,\n inputs.documentId,\n inputs.embedding,\n inputs.k,\n );\n }\n #searchBySource(inputs: {\n sourceId: string;\n embedding: Buffer<ArrayBufferLike>;\n k: number;\n }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT v.content, v.distance, v.document_id, d.metadata\n\t\t\t\tFROM vec_chunks v\n\t\t\t\tJOIN documents d ON d.id = v.document_id\n\t\t\t\tWHERE v.source_id = ?\n\t\t\t\t\tAND v.embedding MATCH vec_normalize(vec_f32(?))\n\t\t\t\t\tAND v.k = ?\n\t\t\t\tORDER BY v.distance ASC\n\t\t\t`);\n return stmt.all(inputs.sourceId, inputs.embedding, inputs.k);\n }\n async search(query: string, options: SearchOptions, embedder: Embedder) {\n const { embeddings } = await embedder([query]);\n if (!embeddings.length) {\n return [];\n }\n const vectorBlob = vectorToBlob(embeddings[0]);\n const topN = options.topN;\n\n if (options.documentId) {\n const rows = this.#searchByDocument({\n sourceId: options.sourceId,\n documentId: options.documentId,\n embedding: vectorBlob,\n k: topN ?? DEFAULT_TOP_N,\n });\n return rows.map(
|
|
5
|
-
"mappings": "AAAA,OAAO,YAAY;AACnB,SAAS,gBAAgB;
|
|
4
|
+
"sourcesContent": ["import dedent from 'dedent';\nimport { template } from 'lodash-es';\n\nimport type {\n Chunk,\n Corpus,\n Embedder,\n SearchOptions,\n SearchResult,\n Store,\n} from '../store.js';\nimport sql from './sqlite.sql.js';\n\nconst DEFAULT_TOP_N = 10;\n\ninterface DB {\n prepare: (sql: string) => {\n run: (...args: any[]) => any;\n all: (...args: any[]) => any[];\n get: (...args: any[]) => any;\n };\n exec: (sql: string) => void;\n}\n\nexport class SQLiteStore implements Store {\n readonly #db: DB;\n constructor(db: DB, dimension: number) {\n this.#db = db;\n const compiled = template(sql);\n this.#db.exec(compiled({ DIMENSION: dimension }));\n }\n\n #transaction(callback: () => void) {\n try {\n this.#db.exec('BEGIN IMMEDIATE');\n callback();\n this.#db.exec('COMMIT');\n } catch (error) {\n this.#db.exec('ROLLBACK');\n throw error;\n }\n }\n\n #searchByDocument(inputs: {\n sourceId: string;\n documentId: string;\n embedding: Buffer<ArrayBufferLike>;\n k: number;\n }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT v.content, v.distance, v.document_id, d.metadata\n\t\t\t\tFROM vec_chunks v\n\t\t\t\tJOIN documents d ON d.id = v.document_id\n\t\t\t\tWHERE v.source_id = ?\n\t\t\t\t\tAND v.document_id = ?\n\t\t\t\t\tAND v.embedding MATCH vec_normalize(vec_f32(?))\n\t\t\t\t\tAND v.k = ?\n\t\t\t\tORDER BY v.distance ASC\n\t\t\t`);\n return stmt.all(\n inputs.sourceId,\n inputs.documentId,\n inputs.embedding,\n inputs.k,\n );\n }\n #searchBySource(inputs: {\n sourceId: string;\n embedding: Buffer<ArrayBufferLike>;\n k: number;\n }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT v.content, v.distance, v.document_id, d.metadata\n\t\t\t\tFROM vec_chunks v\n\t\t\t\tJOIN documents d ON d.id = v.document_id\n\t\t\t\tWHERE v.source_id = ?\n\t\t\t\t\tAND v.embedding MATCH vec_normalize(vec_f32(?))\n\t\t\t\t\tAND v.k = ?\n\t\t\t\tORDER BY v.distance ASC\n\t\t\t`);\n return stmt.all(inputs.sourceId, inputs.embedding, inputs.k);\n }\n async search(query: string, options: SearchOptions, embedder: Embedder) {\n const { embeddings } = await embedder([query]);\n if (!embeddings.length) {\n return [];\n }\n const vectorBlob = vectorToBlob(embeddings[0]);\n const topN = options.topN;\n\n if (options.documentId) {\n const rows = this.#searchByDocument({\n sourceId: options.sourceId,\n documentId: options.documentId,\n embedding: vectorBlob,\n k: topN ?? DEFAULT_TOP_N,\n });\n return rows.map(toSearchResult);\n }\n\n const rows = this.#searchBySource({\n sourceId: options.sourceId,\n embedding: vectorBlob,\n k: topN ?? DEFAULT_TOP_N,\n });\n return rows.map(toSearchResult);\n }\n #upsertSource(inputs: { sourceId: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tINSERT INTO sources (source_id) VALUES (?)\n\t\t\t\tON CONFLICT(source_id) DO UPDATE SET updated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now')\n\t\t\t`);\n return stmt.run(inputs.sourceId);\n }\n #upsertSourceWithExpiry(inputs: { sourceId: string; expiresAt: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tINSERT INTO sources (source_id, expires_at) VALUES (?, ?)\n\t\t\t\tON CONFLICT(source_id) DO UPDATE SET\n\t\t\t\t\tupdated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now',\n\t\t\t\t\texpires_at=excluded.expires_at\n\t\t\t`);\n return stmt.run(inputs.sourceId, inputs.expiresAt);\n }\n upsertDoc(inputs: {\n documentId: string;\n sourceId: string;\n cid: string;\n metadata?: Record<string, any>;\n }) {\n const stmt = this.#db.prepare(dedent`\n INSERT INTO documents (id, source_id, cid, metadata)\n VALUES (?, ?, ?, json(?))\n ON CONFLICT(id) DO UPDATE SET\n cid=excluded.cid,\n metadata=COALESCE(excluded.metadata, documents.metadata),\n updated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now')\n WHERE documents.cid != excluded.cid;\n `);\n return stmt.run(\n inputs.documentId,\n inputs.sourceId,\n inputs.cid,\n inputs.metadata ? JSON.stringify(inputs.metadata) : null,\n );\n }\n insertDoc(inputs: { sourceId: string; documentId: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tINSERT INTO vec_chunks (source_id, document_id, content, embedding)\n\t\t\t\tVALUES (?, ?, ?, vec_normalize(vec_f32(?)))\n\t\t\t`);\n return (chunk: Chunk) => {\n stmt.run(\n inputs.sourceId,\n inputs.documentId,\n chunk.content,\n vectorToBlob(chunk.embedding),\n );\n };\n }\n delete(inputs: { sourceId: string; documentId: string }) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tDELETE FROM vec_chunks WHERE source_id = ? AND document_id = ?\n\t\t\t`);\n return stmt.run(inputs.sourceId, inputs.documentId);\n }\n sourceExists(sourceId: string) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT 1 FROM sources WHERE source_id = ? LIMIT 1\n\t\t\t`);\n const row = stmt.get(sourceId);\n return Boolean(row);\n }\n sourceExpired(sourceId: string) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tSELECT 1 FROM sources\n\t\t\t\tWHERE source_id = ?\n\t\t\t\t\tAND expires_at IS NOT NULL\n\t\t\t\t\tAND expires_at <= strftime('%Y-%m-%dT%H:%M:%fZ','now')\n\t\t\t\tLIMIT 1\n\t\t\t`);\n const row = stmt.run(sourceId);\n return Boolean(row);\n }\n setSourceExpiry(sourceId: string, expiryDate: Date) {\n const stmt = this.#db.prepare(dedent`\n\t\t\t\tUPDATE sources\n\t\t\t\tSET expires_at = ?, updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')\n\t\t\t\tWHERE source_id = ?\n\t\t\t`);\n return stmt.run(expiryDate.toISOString(), sourceId);\n }\n async index(sourceId: string, corpus: Corpus, expiryDate?: Date) {\n if (expiryDate) {\n this.#upsertSourceWithExpiry({\n expiresAt: expiryDate.toISOString(),\n sourceId,\n });\n } else {\n this.#upsertSource({ sourceId });\n }\n\n const info = this.upsertDoc({\n documentId: corpus.id,\n sourceId,\n cid: corpus.cid,\n metadata: corpus.metadata,\n });\n const changed = info.changes > 0;\n\n if (!changed) {\n return;\n }\n\n const insert = this.insertDoc({ sourceId, documentId: corpus.id });\n // Delete previous rows once before inserting\n this.#transaction(() => {\n this.delete({ sourceId, documentId: corpus.id });\n });\n\n const batchSize = 32;\n let batch: Chunk[] = [];\n const flush = () => {\n if (!batch.length) return;\n this.#transaction(() => {\n for (let i = 0; i < batch.length; i++) {\n insert(batch[i]);\n }\n });\n batch = [];\n };\n\n for await (const chunk of corpus.chunker()) {\n batch.push(chunk);\n if (batch.length >= batchSize) flush();\n }\n // flush any remaining\n flush();\n }\n}\n\nexport function vectorToBlob(vector: number[] | Float32Array): Buffer {\n if (vector instanceof Float32Array) {\n // Copy into a fresh Buffer to avoid retaining references to a larger batch tensor buffer\n const copied = new Float32Array(vector.length);\n copied.set(vector);\n return Buffer.from(copied.buffer);\n }\n const floatArray = new Float32Array(vector);\n return Buffer.from(floatArray.buffer);\n}\n\nfunction toSearchResult(row: unknown): SearchResult {\n if (\n typeof row !== 'object' ||\n row === null ||\n !('content' in row) ||\n typeof row.content !== 'string' ||\n !('distance' in row) ||\n typeof row.distance !== 'number' ||\n !('document_id' in row) ||\n typeof row.document_id !== 'string' ||\n !('metadata' in row) ||\n (row.metadata !== null && typeof row.metadata !== 'string')\n ) {\n throw new Error('Unexpected vector search row');\n }\n return {\n content: row.content,\n distance: row.distance,\n document_id: row.document_id,\n metadata: parseMetadata(row.metadata),\n };\n}\n\nfunction parseMetadata(value: string | null): Record<string, unknown> | null {\n if (value === null) return null;\n try {\n const parsed: unknown = JSON.parse(value);\n return isRecord(parsed) ? parsed : null;\n } catch {\n return null;\n }\n}\n\nfunction isRecord(value: unknown): value is Record<string, unknown> {\n return typeof value === 'object' && value !== null && !Array.isArray(value);\n}\n"],
|
|
5
|
+
"mappings": "AAAA,OAAO,YAAY;AACnB,SAAS,gBAAgB;AAUzB,OAAO,SAAS;AAEhB,MAAM,gBAAgB;AAWf,MAAM,YAA6B;AAAA,EAC/B;AAAA,EACT,YAAY,IAAQ,WAAmB;AACrC,SAAK,MAAM;AACX,UAAM,WAAW,SAAS,GAAG;AAC7B,SAAK,IAAI,KAAK,SAAS,EAAE,WAAW,UAAU,CAAC,CAAC;AAAA,EAClD;AAAA,EAEA,aAAa,UAAsB;AACjC,QAAI;AACF,WAAK,IAAI,KAAK,iBAAiB;AAC/B,eAAS;AACT,WAAK,IAAI,KAAK,QAAQ;AAAA,IACxB,SAAS,OAAO;AACd,WAAK,IAAI,KAAK,UAAU;AACxB,YAAM;AAAA,IACR;AAAA,EACF;AAAA,EAEA,kBAAkB,QAKf;AACD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAS9B;AACA,WAAO,KAAK;AAAA,MACV,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO;AAAA,IACT;AAAA,EACF;AAAA,EACA,gBAAgB,QAIb;AACD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAQ9B;AACA,WAAO,KAAK,IAAI,OAAO,UAAU,OAAO,WAAW,OAAO,CAAC;AAAA,EAC7D;AAAA,EACA,MAAM,OAAO,OAAe,SAAwB,UAAoB;AACtE,UAAM,EAAE,WAAW,IAAI,MAAM,SAAS,CAAC,KAAK,CAAC;AAC7C,QAAI,CAAC,WAAW,QAAQ;AACtB,aAAO,CAAC;AAAA,IACV;AACA,UAAM,aAAa,aAAa,WAAW,CAAC,CAAC;AAC7C,UAAM,OAAO,QAAQ;AAErB,QAAI,QAAQ,YAAY;AACtB,YAAMA,QAAO,KAAK,kBAAkB;AAAA,QAClC,UAAU,QAAQ;AAAA,QAClB,YAAY,QAAQ;AAAA,QACpB,WAAW;AAAA,QACX,GAAG,QAAQ;AAAA,MACb,CAAC;AACD,aAAOA,MAAK,IAAI,cAAc;AAAA,IAChC;AAEA,UAAM,OAAO,KAAK,gBAAgB;AAAA,MAChC,UAAU,QAAQ;AAAA,MAClB,WAAW;AAAA,MACX,GAAG,QAAQ;AAAA,IACb,CAAC;AACD,WAAO,KAAK,IAAI,cAAc;AAAA,EAChC;AAAA,EACA,cAAc,QAA8B;AAC1C,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA,IAG9B;AACA,WAAO,KAAK,IAAI,OAAO,QAAQ;AAAA,EACjC;AAAA,EACA,wBAAwB,QAAiD;AACvE,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,IAK9B;AACA,WAAO,KAAK,IAAI,OAAO,UAAU,OAAO,SAAS;AAAA,EACnD;AAAA,EACA,UAAU,QAKP;AACD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,OAQ3B;AACH,WAAO,KAAK;AAAA,MACV,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO;AAAA,MACP,OAAO,WAAW,KAAK,UAAU,OAAO,QAAQ,IAAI;AAAA,IACtD;AAAA,EACF;AAAA,EACA,UAAU,QAAkD;AAC1D,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA,IAG9B;AACA,WAAO,CAAC,UAAiB;AACvB,WAAK;AAAA,QACH,OAAO;AAAA,QACP,OAAO;AAAA,QACP,MAAM;AAAA,QACN,aAAa,MAAM,SAAS;AAAA,MAC9B;AAAA,IACF;AAAA,EACF;AAAA,EACA,OAAO,QAAkD;AACvD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA,IAE9B;AACA,WAAO,KAAK,IAAI,OAAO,UAAU,OAAO,UAAU;AAAA,EACpD;AAAA,EACA,aAAa,UAAkB;AAC7B,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA,IAE9B;AACA,UAAM,MAAM,KAAK,IAAI,QAAQ;AAC7B,WAAO,QAAQ,GAAG;AAAA,EACpB;AAAA,EACA,cAAc,UAAkB;AAC9B,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,IAM9B;AACA,UAAM,MAAM,KAAK,IAAI,QAAQ;AAC7B,WAAO,QAAQ,GAAG;AAAA,EACpB;AAAA,EACA,gBAAgB,UAAkB,YAAkB;AAClD,UAAM,OAAO,KAAK,IAAI,QAAQ;AAAA;AAAA;AAAA;AAAA,IAI9B;AACA,WAAO,KAAK,IAAI,WAAW,YAAY,GAAG,QAAQ;AAAA,EACpD;AAAA,EACA,MAAM,MAAM,UAAkB,QAAgB,YAAmB;AAC/D,QAAI,YAAY;AACd,WAAK,wBAAwB;AAAA,QAC3B,WAAW,WAAW,YAAY;AAAA,QAClC;AAAA,MACF,CAAC;AAAA,IACH,OAAO;AACL,WAAK,cAAc,EAAE,SAAS,CAAC;AAAA,IACjC;AAEA,UAAM,OAAO,KAAK,UAAU;AAAA,MAC1B,YAAY,OAAO;AAAA,MACnB;AAAA,MACA,KAAK,OAAO;AAAA,MACZ,UAAU,OAAO;AAAA,IACnB,CAAC;AACD,UAAM,UAAU,KAAK,UAAU;AAE/B,QAAI,CAAC,SAAS;AACZ;AAAA,IACF;AAEA,UAAM,SAAS,KAAK,UAAU,EAAE,UAAU,YAAY,OAAO,GAAG,CAAC;AAEjE,SAAK,aAAa,MAAM;AACtB,WAAK,OAAO,EAAE,UAAU,YAAY,OAAO,GAAG,CAAC;AAAA,IACjD,CAAC;AAED,UAAM,YAAY;AAClB,QAAI,QAAiB,CAAC;AACtB,UAAM,QAAQ,MAAM;AAClB,UAAI,CAAC,MAAM,OAAQ;AACnB,WAAK,aAAa,MAAM;AACtB,iBAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,iBAAO,MAAM,CAAC,CAAC;AAAA,QACjB;AAAA,MACF,CAAC;AACD,cAAQ,CAAC;AAAA,IACX;AAEA,qBAAiB,SAAS,OAAO,QAAQ,GAAG;AAC1C,YAAM,KAAK,KAAK;AAChB,UAAI,MAAM,UAAU,UAAW,OAAM;AAAA,IACvC;AAEA,UAAM;AAAA,EACR;AACF;AAEO,SAAS,aAAa,QAAyC;AACpE,MAAI,kBAAkB,cAAc;AAElC,UAAM,SAAS,IAAI,aAAa,OAAO,MAAM;AAC7C,WAAO,IAAI,MAAM;AACjB,WAAO,OAAO,KAAK,OAAO,MAAM;AAAA,EAClC;AACA,QAAM,aAAa,IAAI,aAAa,MAAM;AAC1C,SAAO,OAAO,KAAK,WAAW,MAAM;AACtC;AAEA,SAAS,eAAe,KAA4B;AAClD,MACE,OAAO,QAAQ,YACf,QAAQ,QACR,EAAE,aAAa,QACf,OAAO,IAAI,YAAY,YACvB,EAAE,cAAc,QAChB,OAAO,IAAI,aAAa,YACxB,EAAE,iBAAiB,QACnB,OAAO,IAAI,gBAAgB,YAC3B,EAAE,cAAc,QACf,IAAI,aAAa,QAAQ,OAAO,IAAI,aAAa,UAClD;AACA,UAAM,IAAI,MAAM,8BAA8B;AAAA,EAChD;AACA,SAAO;AAAA,IACL,SAAS,IAAI;AAAA,IACb,UAAU,IAAI;AAAA,IACd,aAAa,IAAI;AAAA,IACjB,UAAU,cAAc,IAAI,QAAQ;AAAA,EACtC;AACF;AAEA,SAAS,cAAc,OAAsD;AAC3E,MAAI,UAAU,KAAM,QAAO;AAC3B,MAAI;AACF,UAAM,SAAkB,KAAK,MAAM,KAAK;AACxC,WAAO,SAAS,MAAM,IAAI,SAAS;AAAA,EACrC,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,SAAS,SAAS,OAAkD;AAClE,SAAO,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK;AAC5E;",
|
|
6
6
|
"names": ["rows"]
|
|
7
7
|
}
|
|
@@ -18,8 +18,14 @@ export type Corpus = {
|
|
|
18
18
|
chunker: () => AsyncGenerator<Chunk>;
|
|
19
19
|
metadata?: Record<string, any> | undefined;
|
|
20
20
|
};
|
|
21
|
+
export interface SearchResult {
|
|
22
|
+
content: string;
|
|
23
|
+
distance: number;
|
|
24
|
+
document_id: string;
|
|
25
|
+
metadata: Record<string, unknown> | null;
|
|
26
|
+
}
|
|
21
27
|
export interface Store {
|
|
22
|
-
search: (query: string, options: SearchOptions, embedder: Embedder) => Promise<
|
|
28
|
+
search: (query: string, options: SearchOptions, embedder: Embedder) => Promise<SearchResult[]>;
|
|
23
29
|
sourceExists: (sourceId: string) => Promise<boolean> | boolean;
|
|
24
30
|
sourceExpired: (sourceId: string) => Promise<boolean> | boolean;
|
|
25
31
|
setSourceExpiry: (sourceId: string, expiryDate: Date) => Promise<void> | void;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../../src/lib/stores/store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAEpC,MAAM,MAAM,KAAK,GAAG;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,SAAS,GAAG,YAAY,CAAC;CACrC,CAAC;AACF,MAAM,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC;IACtD,UAAU,EAAE,CAAC,SAAS,GAAG,YAAY,CAAC,EAAE,CAAC;IACzC,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC,CAAC;AAEH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IAEjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,MAAM,MAAM,GAAG;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,cAAc,CAAC,KAAK,CAAC,CAAC;IACrC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS,CAAC;CAC5C,CAAC;AAEF,MAAM,WAAW,KAAK;IACpB,MAAM,EAAE,CACN,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,aAAa,EACtB,QAAQ,EAAE,QAAQ,KACf,OAAO,CAAC,
|
|
1
|
+
{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../../../src/lib/stores/store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,IAAI,CAAC;AAEpC,MAAM,MAAM,KAAK,GAAG;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,SAAS,GAAG,YAAY,CAAC;CACrC,CAAC;AACF,MAAM,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC;IACtD,UAAU,EAAE,CAAC,SAAS,GAAG,YAAY,CAAC,EAAE,CAAC;IACzC,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC,CAAC;AAEH,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IAEjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,MAAM,MAAM,GAAG;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,cAAc,CAAC,KAAK,CAAC,CAAC;IACrC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS,CAAC;CAC5C,CAAC;AAEF,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAC;CAC1C;AAED,MAAM,WAAW,KAAK;IACpB,MAAM,EAAE,CACN,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,aAAa,EACtB,QAAQ,EAAE,QAAQ,KACf,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;IAC7B,YAAY,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;IAC/D,aAAa,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;IAChE,eAAe,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC9E,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,IAAI,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC/E"}
|