@deepagents/retrieval 7.2.0 → 7.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/dist/index.d.ts +7 -7
  2. package/dist/index.js +561 -7
  3. package/dist/index.js.map +4 -4
  4. package/dist/lib/connectors/index.d.ts +7 -7
  5. package/dist/lib/connectors/index.js +649 -7
  6. package/dist/lib/connectors/index.js.map +4 -4
  7. package/dist/lib/connectors/local.d.ts +1 -1
  8. package/dist/lib/connectors/local.d.ts.map +1 -1
  9. package/dist/lib/connectors/pdf.d.ts +1 -1
  10. package/dist/lib/connectors/pdf.d.ts.map +1 -1
  11. package/dist/lib/connectors/repo.d.ts +1 -1
  12. package/dist/lib/embedders/fastembed.d.ts +1 -1
  13. package/dist/lib/embedders/huggingface.d.ts +1 -1
  14. package/dist/lib/ingest.d.ts +3 -3
  15. package/dist/lib/similiarty-search.d.ts +1 -1
  16. package/dist/lib/stores/sqlite/ddl.sqlite.d.ts +1 -1
  17. package/dist/lib/stores/sqlite/ddl.sqlite.d.ts.map +1 -1
  18. package/dist/lib/stores/sqlite/node-sqlite.d.ts +1 -1
  19. package/dist/lib/stores/sqlite/open.d.ts +1 -2
  20. package/dist/lib/stores/sqlite/open.d.ts.map +1 -1
  21. package/dist/lib/stores/sqlite/sqlite.d.ts +1 -1
  22. package/package.json +4 -3
  23. package/dist/lib/connectors/connector.js +0 -1
  24. package/dist/lib/connectors/connector.js.map +0 -7
  25. package/dist/lib/connectors/github.js +0 -164
  26. package/dist/lib/connectors/github.js.map +0 -7
  27. package/dist/lib/connectors/linear.js +0 -29
  28. package/dist/lib/connectors/linear.js.map +0 -7
  29. package/dist/lib/connectors/local.js +0 -98
  30. package/dist/lib/connectors/local.js.map +0 -7
  31. package/dist/lib/connectors/pdf.js +0 -58
  32. package/dist/lib/connectors/pdf.js.map +0 -7
  33. package/dist/lib/connectors/repo.js +0 -179
  34. package/dist/lib/connectors/repo.js.map +0 -7
  35. package/dist/lib/connectors/rss.js +0 -128
  36. package/dist/lib/connectors/rss.js.map +0 -7
  37. package/dist/lib/embedders/fastembed.js +0 -63
  38. package/dist/lib/embedders/fastembed.js.map +0 -7
  39. package/dist/lib/embedders/huggingface.js +0 -42
  40. package/dist/lib/embedders/huggingface.js.map +0 -7
  41. package/dist/lib/ingest.js +0 -117
  42. package/dist/lib/ingest.js.map +0 -7
  43. package/dist/lib/pipeline.js +0 -1
  44. package/dist/lib/pipeline.js.map +0 -7
  45. package/dist/lib/sidecar.js +0 -1
  46. package/dist/lib/sidecar.js.map +0 -7
  47. package/dist/lib/similiarty-search.js +0 -42
  48. package/dist/lib/similiarty-search.js.map +0 -7
  49. package/dist/lib/stores/cid.js +0 -8
  50. package/dist/lib/stores/cid.js.map +0 -7
  51. package/dist/lib/stores/sqlite/ddl.sqlite.js +0 -53
  52. package/dist/lib/stores/sqlite/ddl.sqlite.js.map +0 -7
  53. package/dist/lib/stores/sqlite/node-sqlite.js +0 -9
  54. package/dist/lib/stores/sqlite/node-sqlite.js.map +0 -7
  55. package/dist/lib/stores/sqlite/open.js +0 -11
  56. package/dist/lib/stores/sqlite/open.js.map +0 -7
  57. package/dist/lib/stores/sqlite/sqlite.js +0 -245
  58. package/dist/lib/stores/sqlite/sqlite.js.map +0 -7
  59. package/dist/lib/stores/store.js +0 -1
  60. package/dist/lib/stores/store.js.map +0 -7
@@ -1,58 +0,0 @@
1
- import { glob, readFile } from "node:fs/promises";
2
- import { extractText, getDocumentProxy } from "unpdf";
3
- function pdf(pattern) {
4
- const sourceId = `pdf:${pattern}`;
5
- return {
6
- sourceId,
7
- sources: async function* () {
8
- const paths = await Array.fromAsync(
9
- glob(pattern, { exclude: ["**/node_modules/**", "**/.git/**"] })
10
- );
11
- for (const path of paths) {
12
- if (!path.toLowerCase().endsWith(".pdf")) continue;
13
- yield {
14
- id: path,
15
- content: async () => {
16
- const buffer = await readFile(path);
17
- const pdf2 = await getDocumentProxy(new Uint8Array(buffer));
18
- const { text } = await extractText(pdf2, { mergePages: true });
19
- return text;
20
- }
21
- };
22
- }
23
- }
24
- };
25
- }
26
- function pdfFile(source) {
27
- const isUrl = /^https?:\/\//.test(source);
28
- const sourceId = `pdf:${isUrl ? "url" : "file"}:${source}`;
29
- return {
30
- sourceId,
31
- sources: async function* () {
32
- yield {
33
- id: source,
34
- content: async () => {
35
- let buffer;
36
- if (isUrl) {
37
- const response = await fetch(source);
38
- if (!response.ok) {
39
- throw new Error(`HTTP ${response.status}: ${response.statusText}`);
40
- }
41
- buffer = new Uint8Array(await response.arrayBuffer());
42
- } else {
43
- const fileBuffer = await readFile(source);
44
- buffer = new Uint8Array(fileBuffer.buffer);
45
- }
46
- const pdf2 = await getDocumentProxy(new Uint8Array(buffer));
47
- const { text } = await extractText(pdf2, { mergePages: true });
48
- return text;
49
- }
50
- };
51
- }
52
- };
53
- }
54
- export {
55
- pdf,
56
- pdfFile
57
- };
58
- //# sourceMappingURL=pdf.js.map
@@ -1,7 +0,0 @@
1
- {
2
- "version": 3,
3
- "sources": ["../../../src/lib/connectors/pdf.ts"],
4
- "sourcesContent": ["import { glob, readFile } from 'node:fs/promises';\n\nimport { extractText, getDocumentProxy } from 'unpdf';\n\nimport type { Connector } from './connector.js';\n\nexport function pdf(pattern: string): Connector {\n const sourceId = `pdf:${pattern}`;\n\n return {\n sourceId,\n sources: async function* () {\n const paths = await Array.fromAsync(\n glob(pattern, { exclude: ['**/node_modules/**', '**/.git/**'] }),\n );\n for (const path of paths) {\n if (!path.toLowerCase().endsWith('.pdf')) continue;\n yield {\n id: path,\n content: async () => {\n const buffer = await readFile(path);\n const pdf = await getDocumentProxy(new Uint8Array(buffer));\n const { text } = await extractText(pdf, { mergePages: true });\n return text;\n },\n };\n }\n },\n };\n}\n\nexport function pdfFile(source: string): Connector {\n const isUrl = /^https?:\\/\\//.test(source);\n const sourceId = `pdf:${isUrl ? 'url' : 'file'}:${source}`;\n\n return {\n sourceId,\n sources: async function* () {\n yield {\n id: source,\n content: async () => {\n let buffer: Uint8Array;\n\n if (isUrl) {\n const response = await fetch(source);\n if (!response.ok) {\n throw new Error(`HTTP ${response.status}: ${response.statusText}`);\n }\n buffer = new Uint8Array(await response.arrayBuffer());\n } else {\n const fileBuffer = await readFile(source);\n buffer = new Uint8Array(fileBuffer.buffer);\n }\n\n const pdf = await getDocumentProxy(new Uint8Array(buffer));\n const { text } = await extractText(pdf, { mergePages: true });\n return text;\n },\n };\n },\n };\n}\n"],
5
- "mappings": "AAAA,SAAS,MAAM,gBAAgB;AAE/B,SAAS,aAAa,wBAAwB;AAIvC,SAAS,IAAI,SAA4B;AAC9C,QAAM,WAAW,OAAO,OAAO;AAE/B,SAAO;AAAA,IACL;AAAA,IACA,SAAS,mBAAmB;AAC1B,YAAM,QAAQ,MAAM,MAAM;AAAA,QACxB,KAAK,SAAS,EAAE,SAAS,CAAC,sBAAsB,YAAY,EAAE,CAAC;AAAA,MACjE;AACA,iBAAW,QAAQ,OAAO;AACxB,YAAI,CAAC,KAAK,YAAY,EAAE,SAAS,MAAM,EAAG;AAC1C,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,SAAS,YAAY;AACnB,kBAAM,SAAS,MAAM,SAAS,IAAI;AAClC,kBAAMA,OAAM,MAAM,iBAAiB,IAAI,WAAW,MAAM,CAAC;AACzD,kBAAM,EAAE,KAAK,IAAI,MAAM,YAAYA,MAAK,EAAE,YAAY,KAAK,CAAC;AAC5D,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEO,SAAS,QAAQ,QAA2B;AACjD,QAAM,QAAQ,eAAe,KAAK,MAAM;AACxC,QAAM,WAAW,OAAO,QAAQ,QAAQ,MAAM,IAAI,MAAM;AAExD,SAAO;AAAA,IACL;AAAA,IACA,SAAS,mBAAmB;AAC1B,YAAM;AAAA,QACJ,IAAI;AAAA,QACJ,SAAS,YAAY;AACnB,cAAI;AAEJ,cAAI,OAAO;AACT,kBAAM,WAAW,MAAM,MAAM,MAAM;AACnC,gBAAI,CAAC,SAAS,IAAI;AAChB,oBAAM,IAAI,MAAM,QAAQ,SAAS,MAAM,KAAK,SAAS,UAAU,EAAE;AAAA,YACnE;AACA,qBAAS,IAAI,WAAW,MAAM,SAAS,YAAY,CAAC;AAAA,UACtD,OAAO;AACL,kBAAM,aAAa,MAAM,SAAS,MAAM;AACxC,qBAAS,IAAI,WAAW,WAAW,MAAM;AAAA,UAC3C;AAEA,gBAAMA,OAAM,MAAM,iBAAiB,IAAI,WAAW,MAAM,CAAC;AACzD,gBAAM,EAAE,KAAK,IAAI,MAAM,YAAYA,MAAK,EAAE,YAAY,KAAK,CAAC;AAC5D,iBAAO;AAAA,QACT;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;",
6
- "names": ["pdf"]
7
- }
@@ -1,179 +0,0 @@
1
- import fg from "fast-glob";
2
- import { opendir, readFile, stat } from "node:fs/promises";
3
- import { join } from "node:path";
4
- function repo(dir, extensions, ingestWhen) {
5
- const sourceId = `repo:${dir}`;
6
- return {
7
- sourceId,
8
- ingestWhen,
9
- sources: async function* () {
10
- const paths = await collectFiles(dir, extensions);
11
- for await (const path of paths) {
12
- const maxSize = 3 * 1024;
13
- const st = await stat(path);
14
- if (st.size > maxSize) {
15
- continue;
16
- }
17
- yield {
18
- id: path,
19
- metadata: { repo: dir },
20
- content: () => readFile(path, "utf8").catch(() => "")
21
- };
22
- }
23
- }
24
- };
25
- }
26
- async function* findAllGitRepos(root) {
27
- const stack = [root];
28
- const skip = [
29
- "node_modules",
30
- "Library",
31
- "Applications",
32
- "Pictures",
33
- "Movies",
34
- "Music",
35
- "Downloads",
36
- ".cache",
37
- ".npm",
38
- ".pnpm",
39
- "development"
40
- ];
41
- while (stack.length) {
42
- const folder = stack.pop();
43
- const isGitRepo = await stat(join(folder, ".git")).then((st) => st.isDirectory() || st.isFile()).catch(() => false);
44
- if (isGitRepo) {
45
- yield folder;
46
- continue;
47
- }
48
- const dir = await opendir(folder);
49
- for await (const dirent of dir) {
50
- if (dirent.isDirectory() && !dirent.isSymbolicLink() && !skip.includes(dirent.name) && !dirent.name.startsWith(".")) {
51
- stack.push(join(folder, dirent.name));
52
- }
53
- }
54
- }
55
- }
56
- function detectRepoTooling(repo2) {
57
- const tools = [];
58
- return tools;
59
- }
60
- async function gitignore(gitignoreFile) {
61
- const content = await readFile(gitignoreFile, "utf8").catch(() => "");
62
- if (!content) {
63
- return [];
64
- }
65
- const patterns = content.split("\n").map((line) => line.trim()).filter((line) => line && !line.startsWith("#"));
66
- return patterns;
67
- }
68
- async function ignorePatterns(repo2) {
69
- return [
70
- // Inherit repo-specific ignore patterns
71
- ...await gitignore(join(repo2, ".gitignore")),
72
- // Package managers & dependency dirs
73
- "node_modules/**",
74
- "**/node_modules/**",
75
- "**/.pnpm/**",
76
- "**/.npm/**",
77
- "**/.yarn/**",
78
- "**/vendor/**",
79
- // PHP / Go modules (when vendored)
80
- "**/3rdparty/**",
81
- // Version control + VCS metadata
82
- "**/.git/**",
83
- "**/.svn/**",
84
- "**/.hg/**",
85
- // OS / system junk
86
- "**/.DS_Store",
87
- "**/Thumbs.db",
88
- "**/Library/**",
89
- "**/Applications/**",
90
- "**/Pictures/**",
91
- "**/Movies/**",
92
- "**/Music/**",
93
- "**/Downloads/**",
94
- "**/.cache/**",
95
- // Environment / secrets (explicit)
96
- "**/.env",
97
- "**/.env.*",
98
- // Lockfiles & generated dependency state
99
- "**/*.lock",
100
- "**/yarn.lock",
101
- "**/package-lock.json",
102
- "**/pnpm-lock.yaml",
103
- // Build / compilation outputs
104
- "**/dist/**",
105
- "**/debug/**",
106
- "**/build/**",
107
- "**/out/**",
108
- "**/target/**",
109
- // Rust / JVM
110
- "**/bin/**",
111
- "**/obj/**",
112
- "**/classes/**",
113
- // Framework / tool specific build artifacts
114
- "**/.next/**",
115
- "**/.vercel/**",
116
- "**/.turbo/**",
117
- "**/.docusaurus/**",
118
- "**/.vite/**",
119
- "**/.parcel-cache/**",
120
- "**/.rollup.cache/**",
121
- "**/.vuepress/**",
122
- "cdk.out/**",
123
- // Infra & deployment tooling
124
- "**/.serverless/**",
125
- "**/.terraform/**",
126
- "**/.terragrunt-cache/**",
127
- "**/.pulumi/**",
128
- // Coverage & testing caches
129
- "**/coverage/**",
130
- "**/.nyc_output/**",
131
- "**/jest-cache/**",
132
- "**/.pytest_cache/**",
133
- // Language / tooling caches
134
- "**/__pycache__/**",
135
- "**/.mypy_cache/**",
136
- "**/.tox/**",
137
- "**/.gradle/**",
138
- "**/.mvn/**",
139
- "**/.eslintcache",
140
- "**/.stylelintcache",
141
- // IDE / editor configs + history (we don't want to embed these)
142
- "**/.idea/**",
143
- "**/.vscode/**",
144
- "**/.fleet/**",
145
- "**/.history/**",
146
- // Virtual environments
147
- "**/.venv/**",
148
- "**/venv/**"
149
- ];
150
- }
151
- async function collectFiles(repo2, extensions) {
152
- const exts = extensions.map((ext) => ext.replace(/^\./, ""));
153
- const entries = fg.stream(
154
- extensions.length > 1 ? `**/*.{${exts.join(",")}}` : `**/*.${exts[0]}`,
155
- {
156
- dot: false,
157
- onlyFiles: true,
158
- unique: true,
159
- absolute: true,
160
- cwd: repo2,
161
- ignore: await ignorePatterns(repo2)
162
- }
163
- );
164
- return (async function* () {
165
- for await (const entry of entries) {
166
- if (typeof entry !== "string") {
167
- throw new TypeError("fast-glob yielded a non-path entry");
168
- }
169
- yield entry;
170
- }
171
- })();
172
- }
173
- export {
174
- collectFiles,
175
- findAllGitRepos,
176
- ignorePatterns,
177
- repo
178
- };
179
- //# sourceMappingURL=repo.js.map
@@ -1,7 +0,0 @@
1
- {
2
- "version": 3,
3
- "sources": ["../../../src/lib/connectors/repo.ts"],
4
- "sourcesContent": ["import fg from 'fast-glob';\nimport { opendir, readFile, stat } from 'node:fs/promises';\nimport { join } from 'node:path';\n\nimport type { Connector } from './connector.js';\n\nexport function repo(\n dir: string,\n extensions: string[],\n ingestWhen: Connector['ingestWhen'],\n): Connector {\n const sourceId = `repo:${dir}`;\n return {\n sourceId,\n ingestWhen,\n sources: async function* () {\n const paths = await collectFiles(dir, extensions);\n for await (const path of paths) {\n const maxSize = 3 * 1024; // 3KB\n const st = await stat(path);\n if (st.size > maxSize) {\n continue;\n }\n yield {\n id: path,\n metadata: { repo: dir },\n content: () => readFile(path, 'utf8').catch(() => ''),\n };\n }\n },\n };\n}\n\nexport async function* findAllGitRepos(root: string) {\n const stack = [root];\n const skip = [\n 'node_modules',\n 'Library',\n 'Applications',\n 'Pictures',\n 'Movies',\n 'Music',\n 'Downloads',\n '.cache',\n '.npm',\n '.pnpm',\n 'development',\n ];\n while (stack.length) {\n const folder = stack.pop()!;\n const isGitRepo = await stat(join(folder, '.git'))\n .then((st) => st.isDirectory() || st.isFile())\n .catch(() => false);\n if (isGitRepo) {\n yield folder;\n continue;\n }\n const dir = await opendir(folder);\n for await (const dirent of dir) {\n if (\n dirent.isDirectory() &&\n !dirent.isSymbolicLink() &&\n !skip.includes(dirent.name) &&\n !dirent.name.startsWith('.')\n ) {\n stack.push(join(folder, dirent.name));\n }\n }\n }\n}\n\nfunction detectRepoTooling(repo: string) {\n const tools: string[] = [];\n return tools;\n}\n\nasync function gitignore(gitignoreFile: string) {\n const content = await readFile(gitignoreFile, 'utf8').catch(() => '');\n if (!content) {\n return [];\n }\n const patterns = content\n .split('\\n')\n .map((line) => line.trim())\n .filter((line) => line && !line.startsWith('#'));\n return patterns;\n}\n\nexport async function ignorePatterns(repo: string) {\n return [\n // Inherit repo-specific ignore patterns\n ...(await gitignore(join(repo, '.gitignore'))),\n\n // Package managers & dependency dirs\n 'node_modules/**',\n '**/node_modules/**',\n '**/.pnpm/**',\n '**/.npm/**',\n '**/.yarn/**',\n '**/vendor/**', // PHP / Go modules (when vendored)\n '**/3rdparty/**',\n\n // Version control + VCS metadata\n '**/.git/**',\n '**/.svn/**',\n '**/.hg/**',\n\n // OS / system junk\n '**/.DS_Store',\n '**/Thumbs.db',\n '**/Library/**',\n '**/Applications/**',\n '**/Pictures/**',\n '**/Movies/**',\n '**/Music/**',\n '**/Downloads/**',\n '**/.cache/**',\n\n // Environment / secrets (explicit)\n '**/.env',\n '**/.env.*',\n\n // Lockfiles & generated dependency state\n '**/*.lock',\n '**/yarn.lock',\n '**/package-lock.json',\n '**/pnpm-lock.yaml',\n\n // Build / compilation outputs\n '**/dist/**',\n '**/debug/**',\n '**/build/**',\n '**/out/**',\n '**/target/**', // Rust / JVM\n '**/bin/**',\n '**/obj/**',\n '**/classes/**',\n\n // Framework / tool specific build artifacts\n '**/.next/**',\n '**/.vercel/**',\n '**/.turbo/**',\n '**/.docusaurus/**',\n '**/.vite/**',\n '**/.parcel-cache/**',\n '**/.rollup.cache/**',\n '**/.vuepress/**',\n 'cdk.out/**',\n\n // Infra & deployment tooling\n '**/.serverless/**',\n '**/.terraform/**',\n '**/.terragrunt-cache/**',\n '**/.pulumi/**',\n\n // Coverage & testing caches\n '**/coverage/**',\n '**/.nyc_output/**',\n '**/jest-cache/**',\n '**/.pytest_cache/**',\n\n // Language / tooling caches\n '**/__pycache__/**',\n '**/.mypy_cache/**',\n '**/.tox/**',\n '**/.gradle/**',\n '**/.mvn/**',\n '**/.eslintcache',\n '**/.stylelintcache',\n\n // IDE / editor configs + history (we don't want to embed these)\n '**/.idea/**',\n '**/.vscode/**',\n '**/.fleet/**',\n '**/.history/**',\n\n // Virtual environments\n '**/.venv/**',\n '**/venv/**',\n ];\n}\n\nexport async function collectFiles(\n repo: string,\n extensions: string[],\n): Promise<AsyncIterable<string>> {\n const exts = extensions.map((ext) => ext.replace(/^\\./, ''));\n const entries = fg.stream(\n extensions.length > 1 ? `**/*.{${exts.join(',')}}` : `**/*.${exts[0]}`,\n {\n dot: false,\n onlyFiles: true,\n unique: true,\n absolute: true,\n cwd: repo,\n ignore: await ignorePatterns(repo),\n },\n );\n return (async function* () {\n for await (const entry of entries) {\n if (typeof entry !== 'string') {\n throw new TypeError('fast-glob yielded a non-path entry');\n }\n yield entry;\n }\n })();\n}\n"],
5
- "mappings": "AAAA,OAAO,QAAQ;AACf,SAAS,SAAS,UAAU,YAAY;AACxC,SAAS,YAAY;AAId,SAAS,KACd,KACA,YACA,YACW;AACX,QAAM,WAAW,QAAQ,GAAG;AAC5B,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,SAAS,mBAAmB;AAC1B,YAAM,QAAQ,MAAM,aAAa,KAAK,UAAU;AAChD,uBAAiB,QAAQ,OAAO;AAC9B,cAAM,UAAU,IAAI;AACpB,cAAM,KAAK,MAAM,KAAK,IAAI;AAC1B,YAAI,GAAG,OAAO,SAAS;AACrB;AAAA,QACF;AACA,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,UAAU,EAAE,MAAM,IAAI;AAAA,UACtB,SAAS,MAAM,SAAS,MAAM,MAAM,EAAE,MAAM,MAAM,EAAE;AAAA,QACtD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,gBAAuB,gBAAgB,MAAc;AACnD,QAAM,QAAQ,CAAC,IAAI;AACnB,QAAM,OAAO;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,SAAO,MAAM,QAAQ;AACnB,UAAM,SAAS,MAAM,IAAI;AACzB,UAAM,YAAY,MAAM,KAAK,KAAK,QAAQ,MAAM,CAAC,EAC9C,KAAK,CAAC,OAAO,GAAG,YAAY,KAAK,GAAG,OAAO,CAAC,EAC5C,MAAM,MAAM,KAAK;AACpB,QAAI,WAAW;AACb,YAAM;AACN;AAAA,IACF;AACA,UAAM,MAAM,MAAM,QAAQ,MAAM;AAChC,qBAAiB,UAAU,KAAK;AAC9B,UACE,OAAO,YAAY,KACnB,CAAC,OAAO,eAAe,KACvB,CAAC,KAAK,SAAS,OAAO,IAAI,KAC1B,CAAC,OAAO,KAAK,WAAW,GAAG,GAC3B;AACA,cAAM,KAAK,KAAK,QAAQ,OAAO,IAAI,CAAC;AAAA,MACtC;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,kBAAkBA,OAAc;AACvC,QAAM,QAAkB,CAAC;AACzB,SAAO;AACT;AAEA,eAAe,UAAU,eAAuB;AAC9C,QAAM,UAAU,MAAM,SAAS,eAAe,MAAM,EAAE,MAAM,MAAM,EAAE;AACpE,MAAI,CAAC,SAAS;AACZ,WAAO,CAAC;AAAA,EACV;AACA,QAAM,WAAW,QACd,MAAM,IAAI,EACV,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,OAAO,CAAC,SAAS,QAAQ,CAAC,KAAK,WAAW,GAAG,CAAC;AACjD,SAAO;AACT;AAEA,eAAsB,eAAeA,OAAc;AACjD,SAAO;AAAA;AAAA,IAEL,GAAI,MAAM,UAAU,KAAKA,OAAM,YAAY,CAAC;AAAA;AAAA,IAG5C;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,EACF;AACF;AAEA,eAAsB,aACpBA,OACA,YACgC;AAChC,QAAM,OAAO,WAAW,IAAI,CAAC,QAAQ,IAAI,QAAQ,OAAO,EAAE,CAAC;AAC3D,QAAM,UAAU,GAAG;AAAA,IACjB,WAAW,SAAS,IAAI,SAAS,KAAK,KAAK,GAAG,CAAC,MAAM,QAAQ,KAAK,CAAC,CAAC;AAAA,IACpE;AAAA,MACE,KAAK;AAAA,MACL,WAAW;AAAA,MACX,QAAQ;AAAA,MACR,UAAU;AAAA,MACV,KAAKA;AAAA,MACL,QAAQ,MAAM,eAAeA,KAAI;AAAA,IACnC;AAAA,EACF;AACA,UAAQ,mBAAmB;AACzB,qBAAiB,SAAS,SAAS;AACjC,UAAI,OAAO,UAAU,UAAU;AAC7B,cAAM,IAAI,UAAU,oCAAoC;AAAA,MAC1D;AACA,YAAM;AAAA,IACR;AAAA,EACF,GAAG;AACL;",
6
- "names": ["repo"]
7
- }
@@ -1,128 +0,0 @@
1
- import { experimental_createWebFetchTool } from "@ai-sdk/tool-web-fetch";
2
- import assert from "node:assert/strict";
3
- import Parser from "rss-parser";
4
- const webFetch = experimental_createWebFetchTool({
5
- // Ingestion stores the complete article in one result.
6
- maxLength: Number.MAX_SAFE_INTEGER
7
- });
8
- const rssParser = new Parser({
9
- customFields: {
10
- feed: ["copyright", "language", "managingEditor", "webMaster"],
11
- item: ["category", "creator", "enclosure", "guid"]
12
- }
13
- });
14
- async function parseRSSFeed(feedUrl) {
15
- try {
16
- console.log(`Fetching RSS feed: ${feedUrl}`);
17
- const feed = await rssParser.parseURL(feedUrl);
18
- return {
19
- title: feed.title || "",
20
- description: feed.description || "",
21
- link: feed.link || "",
22
- language: feed.language || "en",
23
- lastBuildDate: feed.lastBuildDate || (/* @__PURE__ */ new Date()).toISOString(),
24
- items: feed.items.map((item) => ({
25
- title: item.title || "",
26
- description: item.content || item.summary || item.contentSnippet || "",
27
- link: item.link || "",
28
- pubDate: item.pubDate || item.isoDate || "",
29
- author: item.creator || item.author || "",
30
- categories: Array.isArray(item.categories) ? item.categories : item.category ? [item.category] : [],
31
- guid: item.guid || item.guid || "",
32
- contentEncoded: item["content:encoded"] || item.content || ""
33
- }))
34
- };
35
- } catch (error) {
36
- console.error(`Failed to parse RSS feed ${feedUrl}:`, error);
37
- throw new Error(
38
- `RSS parsing failed: ${error instanceof Error ? error.message : "Unknown error"}`
39
- );
40
- }
41
- }
42
- async function fetchArticleContent(url) {
43
- try {
44
- console.log(`Fetching full article content from: ${url}`);
45
- const article = await webFetch.execute(
46
- { url },
47
- { toolCallId: url, messages: [], context: {} }
48
- );
49
- assert.ok("type" in article, "Web fetch must return a complete result");
50
- if (article.type === "error") {
51
- throw new Error(article.message);
52
- }
53
- const fullContent = article.title ? `${article.title}
54
-
55
- ${article.content}` : article.content;
56
- if (fullContent.length < 200) {
57
- throw new Error(
58
- `Extracted content too short (${fullContent.length} chars) - likely failed to find main content`
59
- );
60
- }
61
- console.log(
62
- `\u2713 Successfully extracted article: "${article.title ?? ""}" (${fullContent.length} chars, ~${Math.round(fullContent.split(" ").length)} words)`
63
- );
64
- return fullContent;
65
- } catch (error) {
66
- console.warn(
67
- `Failed to fetch article content from ${url}:`,
68
- error instanceof Error ? error.message : error
69
- );
70
- return "";
71
- }
72
- }
73
- function rss(feedUrl, options = {}) {
74
- const sourceId = `rss:${feedUrl}`;
75
- const { maxItems = 50, fetchFullArticles = false } = options;
76
- return {
77
- sourceId,
78
- instructions: `You answer questions about articles and content from the RSS feed: ${feedUrl}.
79
- Always cite the article title and link when referencing specific content.
80
- The feed contains recent articles, blog posts, and news items.
81
- When referencing content, include the publication date and author when available.
82
- ${fetchFullArticles ? "Full article content has been extracted from the original links for comprehensive analysis." : "Content includes RSS summaries and descriptions."}`,
83
- sources: async function* () {
84
- const feed = await parseRSSFeed(feedUrl);
85
- const feedSummary = `RSS Feed: ${feed.title}
86
- Description: ${feed.description}
87
- Website: ${feed.link}
88
- Language: ${feed.language}
89
- Last Updated: ${feed.lastBuildDate}
90
- Total Items: ${feed.items.length}
91
-
92
- This feed provides: ${feed.description}`;
93
- yield {
94
- id: "feed-info",
95
- content: async () => feedSummary
96
- };
97
- const itemsToProcess = feed.items.slice(0, maxItems);
98
- for (const item of itemsToProcess) {
99
- const documentId = item.guid || item.link || `${item.title}-${item.pubDate}`;
100
- yield {
101
- id: documentId,
102
- content: async () => {
103
- let articleContent = item.contentEncoded || item.description;
104
- if (fetchFullArticles && item.link) {
105
- const fullContent = await fetchArticleContent(item.link);
106
- if (fullContent && fullContent.length > articleContent.length) {
107
- articleContent = fullContent;
108
- }
109
- }
110
- return `Title: ${item.title}
111
- Author: ${item.author}
112
- Published: ${item.pubDate}
113
- Categories: ${item.categories.join(", ")}
114
- Link: ${item.link}
115
- ${fetchFullArticles ? "Full Article Content:" : "Content:"}
116
- ${articleContent}
117
-
118
- Summary: ${item.title} - ${item.description}`;
119
- }
120
- };
121
- }
122
- }
123
- };
124
- }
125
- export {
126
- rss
127
- };
128
- //# sourceMappingURL=rss.js.map
@@ -1,7 +0,0 @@
1
- {
2
- "version": 3,
3
- "sources": ["../../../src/lib/connectors/rss.ts"],
4
- "sourcesContent": ["import { experimental_createWebFetchTool } from '@ai-sdk/tool-web-fetch';\nimport assert from 'node:assert/strict';\nimport Parser from 'rss-parser';\n\nconst webFetch = experimental_createWebFetchTool({\n // Ingestion stores the complete article in one result.\n maxLength: Number.MAX_SAFE_INTEGER,\n});\n\ntype FeedFields = {\n copyright?: string;\n language?: string;\n managingEditor?: string;\n webMaster?: string;\n lastBuildDate?: string;\n};\n\ntype ItemFields = {\n category?: string;\n creator?: string;\n enclosure?: Parser.Enclosure;\n guid?: string;\n author?: string;\n 'content:encoded'?: string;\n};\n\nconst rssParser = new Parser<FeedFields, ItemFields>({\n customFields: {\n feed: ['copyright', 'language', 'managingEditor', 'webMaster'],\n item: ['category', 'creator', 'enclosure', 'guid'],\n },\n});\n\nasync function parseRSSFeed(feedUrl: string) {\n try {\n console.log(`Fetching RSS feed: ${feedUrl}`);\n const feed = await rssParser.parseURL(feedUrl);\n\n return {\n title: feed.title || '',\n description: feed.description || '',\n link: feed.link || '',\n language: feed.language || 'en',\n lastBuildDate: feed.lastBuildDate || new Date().toISOString(),\n items: feed.items.map((item) => ({\n title: item.title || '',\n description: item.content || item.summary || item.contentSnippet || '',\n link: item.link || '',\n pubDate: item.pubDate || item.isoDate || '',\n author: item.creator || item.author || '',\n categories: Array.isArray(item.categories)\n ? item.categories\n : item.category\n ? [item.category]\n : [],\n guid: item.guid || item.guid || '',\n contentEncoded: item['content:encoded'] || item.content || '',\n })),\n };\n } catch (error) {\n console.error(`Failed to parse RSS feed ${feedUrl}:`, error);\n throw new Error(\n `RSS parsing failed: ${error instanceof Error ? error.message : 'Unknown error'}`,\n );\n }\n}\n\nasync function fetchArticleContent(url: string): Promise<string> {\n try {\n console.log(`Fetching full article content from: ${url}`);\n\n const article = await webFetch.execute(\n { url },\n { toolCallId: url, messages: [], context: {} },\n );\n assert.ok('type' in article, 'Web fetch must return a complete result');\n if (article.type === 'error') {\n throw new Error(article.message);\n }\n\n // Combine title and content for comprehensive text\n const fullContent = article.title\n ? `${article.title}\\n\\n${article.content}`\n : article.content;\n\n // Ensure we have meaningful content (at least 200 characters for full articles)\n if (fullContent.length < 200) {\n throw new Error(\n `Extracted content too short (${fullContent.length} chars) - likely failed to find main content`,\n );\n }\n\n console.log(\n `\u2713 Successfully extracted article: \"${article.title ?? ''}\" (${fullContent.length} chars, ~${Math.round(fullContent.split(' ').length)} words)`,\n );\n\n return fullContent;\n } catch (error) {\n console.warn(\n `Failed to fetch article content from ${url}:`,\n error instanceof Error ? error.message : error,\n );\n return ''; // Return empty string on failure rather than throwing\n }\n}\nexport function rss(\n feedUrl: string,\n options: {\n maxItems?: number;\n fetchFullArticles?: boolean;\n } = {},\n) {\n const sourceId = `rss:${feedUrl}`;\n const { maxItems = 50, fetchFullArticles = false } = options;\n\n return {\n sourceId,\n instructions: `You answer questions about articles and content from the RSS feed: ${feedUrl}.\n Always cite the article title and link when referencing specific content.\n The feed contains recent articles, blog posts, and news items.\n When referencing content, include the publication date and author when available.\n ${fetchFullArticles ? 'Full article content has been extracted from the original links for comprehensive analysis.' : 'Content includes RSS summaries and descriptions.'}`,\n sources: async function* () {\n const feed = await parseRSSFeed(feedUrl);\n // Add feed summary source\n const feedSummary = `RSS Feed: ${feed.title}\nDescription: ${feed.description}\nWebsite: ${feed.link}\nLanguage: ${feed.language}\nLast Updated: ${feed.lastBuildDate}\nTotal Items: ${feed.items.length}\n\nThis feed provides: ${feed.description}`;\n yield {\n id: 'feed-info',\n content: async () => feedSummary,\n };\n\n // Individual article sources (limit to maxItems)\n const itemsToProcess = feed.items.slice(0, maxItems);\n for (const item of itemsToProcess) {\n const documentId =\n item.guid || item.link || `${item.title}-${item.pubDate}`;\n yield {\n id: documentId,\n content: async () => {\n // Try full article fetch if enabled, fallback to RSS content\n let articleContent = item.contentEncoded || item.description;\n if (fetchFullArticles && item.link) {\n const fullContent = await fetchArticleContent(item.link);\n if (fullContent && fullContent.length > articleContent.length) {\n articleContent = fullContent;\n }\n }\n return `Title: ${item.title}\nAuthor: ${item.author}\nPublished: ${item.pubDate}\nCategories: ${item.categories.join(', ')}\nLink: ${item.link}\n${fetchFullArticles ? 'Full Article Content:' : 'Content:'}\n${articleContent}\n\nSummary: ${item.title} - ${item.description}`;\n },\n };\n }\n },\n };\n}\n"],
5
- "mappings": "AAAA,SAAS,uCAAuC;AAChD,OAAO,YAAY;AACnB,OAAO,YAAY;AAEnB,MAAM,WAAW,gCAAgC;AAAA;AAAA,EAE/C,WAAW,OAAO;AACpB,CAAC;AAmBD,MAAM,YAAY,IAAI,OAA+B;AAAA,EACnD,cAAc;AAAA,IACZ,MAAM,CAAC,aAAa,YAAY,kBAAkB,WAAW;AAAA,IAC7D,MAAM,CAAC,YAAY,WAAW,aAAa,MAAM;AAAA,EACnD;AACF,CAAC;AAED,eAAe,aAAa,SAAiB;AAC3C,MAAI;AACF,YAAQ,IAAI,sBAAsB,OAAO,EAAE;AAC3C,UAAM,OAAO,MAAM,UAAU,SAAS,OAAO;AAE7C,WAAO;AAAA,MACL,OAAO,KAAK,SAAS;AAAA,MACrB,aAAa,KAAK,eAAe;AAAA,MACjC,MAAM,KAAK,QAAQ;AAAA,MACnB,UAAU,KAAK,YAAY;AAAA,MAC3B,eAAe,KAAK,kBAAiB,oBAAI,KAAK,GAAE,YAAY;AAAA,MAC5D,OAAO,KAAK,MAAM,IAAI,CAAC,UAAU;AAAA,QAC/B,OAAO,KAAK,SAAS;AAAA,QACrB,aAAa,KAAK,WAAW,KAAK,WAAW,KAAK,kBAAkB;AAAA,QACpE,MAAM,KAAK,QAAQ;AAAA,QACnB,SAAS,KAAK,WAAW,KAAK,WAAW;AAAA,QACzC,QAAQ,KAAK,WAAW,KAAK,UAAU;AAAA,QACvC,YAAY,MAAM,QAAQ,KAAK,UAAU,IACrC,KAAK,aACL,KAAK,WACH,CAAC,KAAK,QAAQ,IACd,CAAC;AAAA,QACP,MAAM,KAAK,QAAQ,KAAK,QAAQ;AAAA,QAChC,gBAAgB,KAAK,iBAAiB,KAAK,KAAK,WAAW;AAAA,MAC7D,EAAE;AAAA,IACJ;AAAA,EACF,SAAS,OAAO;AACd,YAAQ,MAAM,4BAA4B,OAAO,KAAK,KAAK;AAC3D,UAAM,IAAI;AAAA,MACR,uBAAuB,iBAAiB,QAAQ,MAAM,UAAU,eAAe;AAAA,IACjF;AAAA,EACF;AACF;AAEA,eAAe,oBAAoB,KAA8B;AAC/D,MAAI;AACF,YAAQ,IAAI,uCAAuC,GAAG,EAAE;AAExD,UAAM,UAAU,MAAM,SAAS;AAAA,MAC7B,EAAE,IAAI;AAAA,MACN,EAAE,YAAY,KAAK,UAAU,CAAC,GAAG,SAAS,CAAC,EAAE;AAAA,IAC/C;AACA,WAAO,GAAG,UAAU,SAAS,yCAAyC;AACtE,QAAI,QAAQ,SAAS,SAAS;AAC5B,YAAM,IAAI,MAAM,QAAQ,OAAO;AAAA,IACjC;AAGA,UAAM,cAAc,QAAQ,QACxB,GAAG,QAAQ,KAAK;AAAA;AAAA,EAAO,QAAQ,OAAO,KACtC,QAAQ;AAGZ,QAAI,YAAY,SAAS,KAAK;AAC5B,YAAM,IAAI;AAAA,QACR,gCAAgC,YAAY,MAAM;AAAA,MACpD;AAAA,IACF;AAEA,YAAQ;AAAA,MACN,2CAAsC,QAAQ,SAAS,EAAE,MAAM,YAAY,MAAM,YAAY,KAAK,MAAM,YAAY,MAAM,GAAG,EAAE,MAAM,CAAC;AAAA,IACxI;AAEA,WAAO;AAAA,EACT,SAAS,OAAO;AACd,YAAQ;AAAA,MACN,wCAAwC,GAAG;AAAA,MAC3C,iBAAiB,QAAQ,MAAM,UAAU;AAAA,IAC3C;AACA,WAAO;AAAA,EACT;AACF;AACO,SAAS,IACd,SACA,UAGI,CAAC,GACL;AACA,QAAM,WAAW,OAAO,OAAO;AAC/B,QAAM,EAAE,WAAW,IAAI,oBAAoB,MAAM,IAAI;AAErD,SAAO;AAAA,IACL;AAAA,IACA,cAAc,sEAAsE,OAAO;AAAA;AAAA;AAAA;AAAA,QAIvF,oBAAoB,gGAAgG,kDAAkD;AAAA,IAC1K,SAAS,mBAAmB;AAC1B,YAAM,OAAO,MAAM,aAAa,OAAO;AAEvC,YAAM,cAAc,aAAa,KAAK,KAAK;AAAA,eAClC,KAAK,WAAW;AAAA,WACpB,KAAK,IAAI;AAAA,YACR,KAAK,QAAQ;AAAA,gBACT,KAAK,aAAa;AAAA,eACnB,KAAK,MAAM,MAAM;AAAA;AAAA,sBAEV,KAAK,WAAW;AAChC,YAAM;AAAA,QACJ,IAAI;AAAA,QACJ,SAAS,YAAY;AAAA,MACvB;AAGA,YAAM,iBAAiB,KAAK,MAAM,MAAM,GAAG,QAAQ;AACnD,iBAAW,QAAQ,gBAAgB;AACjC,cAAM,aACJ,KAAK,QAAQ,KAAK,QAAQ,GAAG,KAAK,KAAK,IAAI,KAAK,OAAO;AACzD,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,SAAS,YAAY;AAEnB,gBAAI,iBAAiB,KAAK,kBAAkB,KAAK;AACjD,gBAAI,qBAAqB,KAAK,MAAM;AAClC,oBAAM,cAAc,MAAM,oBAAoB,KAAK,IAAI;AACvD,kBAAI,eAAe,YAAY,SAAS,eAAe,QAAQ;AAC7D,iCAAiB;AAAA,cACnB;AAAA,YACF;AACA,mBAAO,UAAU,KAAK,KAAK;AAAA,UAC7B,KAAK,MAAM;AAAA,aACR,KAAK,OAAO;AAAA,cACX,KAAK,WAAW,KAAK,IAAI,CAAC;AAAA,QAChC,KAAK,IAAI;AAAA,EACf,oBAAoB,0BAA0B,UAAU;AAAA,EACxD,cAAc;AAAA;AAAA,WAEL,KAAK,KAAK,MAAM,KAAK,WAAW;AAAA,UACjC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;",
6
- "names": []
7
- }
@@ -1,63 +0,0 @@
1
- import { tensorToEmbeddings } from "./huggingface.js";
2
- const EmbeddingModels = {
3
- AllMiniLML6V2: "fast-all-MiniLM-L6-v2",
4
- BGEBaseEN: "fast-bge-base-en",
5
- BGEBaseENV15: "fast-bge-base-en-v1.5",
6
- BGESmallEN: "fast-bge-small-en",
7
- BGESmallENV15: "fast-bge-small-en-v1.5",
8
- BGESmallZH: "fast-bge-small-zh-v1.5",
9
- MLE5Large: "fast-multilingual-e5-large"
10
- };
11
- const transformerModels = {
12
- [EmbeddingModels.AllMiniLML6V2]: "Xenova/all-MiniLM-L6-v2",
13
- [EmbeddingModels.BGEBaseEN]: "Xenova/bge-base-en",
14
- [EmbeddingModels.BGEBaseENV15]: "Xenova/bge-base-en-v1.5",
15
- [EmbeddingModels.BGESmallEN]: "Xenova/bge-small-en",
16
- [EmbeddingModels.BGESmallENV15]: "Xenova/bge-small-en-v1.5",
17
- [EmbeddingModels.BGESmallZH]: "Xenova/bge-small-zh-v1.5",
18
- [EmbeddingModels.MLE5Large]: "Xenova/multilingual-e5-large"
19
- };
20
- function fastembed(options = {}) {
21
- const {
22
- model: modelId = EmbeddingModels.BGESmallENV15,
23
- batchSize = 256,
24
- cacheDir
25
- } = options;
26
- if (!Number.isInteger(batchSize) || batchSize <= 0) {
27
- throw new RangeError("batchSize must be a positive integer");
28
- }
29
- let modelPromise = null;
30
- const getModel = () => {
31
- if (!modelPromise) {
32
- modelPromise = import("@huggingface/transformers").then(
33
- ({ pipeline }) => pipeline("feature-extraction", transformerModels[modelId], {
34
- cache_dir: cacheDir
35
- })
36
- );
37
- }
38
- return modelPromise;
39
- };
40
- return async (documents) => {
41
- if (documents.length === 0) {
42
- return { embeddings: [], dimensions: 0 };
43
- }
44
- const model = await getModel();
45
- const embeddings = [];
46
- let dimensions = 0;
47
- for (let start = 0; start < documents.length; start += batchSize) {
48
- const tensor = await model(documents.slice(start, start + batchSize), {
49
- pooling: "mean",
50
- normalize: true
51
- });
52
- const batch = tensorToEmbeddings(tensor);
53
- dimensions ||= batch.dimensions;
54
- embeddings.push(...batch.embeddings);
55
- }
56
- return { embeddings, dimensions };
57
- };
58
- }
59
- export {
60
- EmbeddingModels,
61
- fastembed
62
- };
63
- //# sourceMappingURL=fastembed.js.map
@@ -1,7 +0,0 @@
1
- {
2
- "version": 3,
3
- "sources": ["../../../src/lib/embedders/fastembed.ts"],
4
- "sourcesContent": ["import type { FeatureExtractionPipeline } from '@huggingface/transformers';\n\nimport type { Embedder } from '../stores/store.js';\nimport { tensorToEmbeddings } from './huggingface.js';\n\n/**\n * Stable model identifiers retained from the former `fastembed` adapter.\n */\nexport const EmbeddingModels = {\n AllMiniLML6V2: 'fast-all-MiniLM-L6-v2',\n BGEBaseEN: 'fast-bge-base-en',\n BGEBaseENV15: 'fast-bge-base-en-v1.5',\n BGESmallEN: 'fast-bge-small-en',\n BGESmallENV15: 'fast-bge-small-en-v1.5',\n BGESmallZH: 'fast-bge-small-zh-v1.5',\n MLE5Large: 'fast-multilingual-e5-large',\n} as const;\nexport type EmbeddingModel =\n (typeof EmbeddingModels)[keyof typeof EmbeddingModels];\n\nconst transformerModels: Record<EmbeddingModel, string> = {\n [EmbeddingModels.AllMiniLML6V2]: 'Xenova/all-MiniLM-L6-v2',\n [EmbeddingModels.BGEBaseEN]: 'Xenova/bge-base-en',\n [EmbeddingModels.BGEBaseENV15]: 'Xenova/bge-base-en-v1.5',\n [EmbeddingModels.BGESmallEN]: 'Xenova/bge-small-en',\n [EmbeddingModels.BGESmallENV15]: 'Xenova/bge-small-en-v1.5',\n [EmbeddingModels.BGESmallZH]: 'Xenova/bge-small-zh-v1.5',\n [EmbeddingModels.MLE5Large]: 'Xenova/multilingual-e5-large',\n};\n\nexport interface FastEmbedOptions {\n model?: EmbeddingModel;\n batchSize?: number;\n cacheDir?: string;\n}\n\n/**\n * Local feature extraction backed by Transformers.js. The historical function\n * and model identifiers remain stable while avoiding FastEmbed's archived tar\n * extraction dependency.\n */\nexport function fastembed(options: FastEmbedOptions = {}): Embedder {\n const {\n model: modelId = EmbeddingModels.BGESmallENV15,\n batchSize = 256,\n cacheDir,\n } = options;\n\n if (!Number.isInteger(batchSize) || batchSize <= 0) {\n throw new RangeError('batchSize must be a positive integer');\n }\n\n let modelPromise: Promise<FeatureExtractionPipeline> | null = null;\n const getModel = () => {\n if (!modelPromise) {\n modelPromise = import('@huggingface/transformers').then(({ pipeline }) =>\n pipeline('feature-extraction', transformerModels[modelId], {\n cache_dir: cacheDir,\n }),\n );\n }\n return modelPromise;\n };\n\n return async (documents: string[]) => {\n if (documents.length === 0) {\n return { embeddings: [], dimensions: 0 };\n }\n\n const model = await getModel();\n const embeddings: Float32Array[] = [];\n let dimensions = 0;\n\n for (let start = 0; start < documents.length; start += batchSize) {\n const tensor = await model(documents.slice(start, start + batchSize), {\n pooling: 'mean',\n normalize: true,\n });\n const batch = tensorToEmbeddings(tensor);\n dimensions ||= batch.dimensions;\n embeddings.push(...batch.embeddings);\n }\n\n return { embeddings, dimensions };\n };\n}\n"],
5
- "mappings": "AAGA,SAAS,0BAA0B;AAK5B,MAAM,kBAAkB;AAAA,EAC7B,eAAe;AAAA,EACf,WAAW;AAAA,EACX,cAAc;AAAA,EACd,YAAY;AAAA,EACZ,eAAe;AAAA,EACf,YAAY;AAAA,EACZ,WAAW;AACb;AAIA,MAAM,oBAAoD;AAAA,EACxD,CAAC,gBAAgB,aAAa,GAAG;AAAA,EACjC,CAAC,gBAAgB,SAAS,GAAG;AAAA,EAC7B,CAAC,gBAAgB,YAAY,GAAG;AAAA,EAChC,CAAC,gBAAgB,UAAU,GAAG;AAAA,EAC9B,CAAC,gBAAgB,aAAa,GAAG;AAAA,EACjC,CAAC,gBAAgB,UAAU,GAAG;AAAA,EAC9B,CAAC,gBAAgB,SAAS,GAAG;AAC/B;AAaO,SAAS,UAAU,UAA4B,CAAC,GAAa;AAClE,QAAM;AAAA,IACJ,OAAO,UAAU,gBAAgB;AAAA,IACjC,YAAY;AAAA,IACZ;AAAA,EACF,IAAI;AAEJ,MAAI,CAAC,OAAO,UAAU,SAAS,KAAK,aAAa,GAAG;AAClD,UAAM,IAAI,WAAW,sCAAsC;AAAA,EAC7D;AAEA,MAAI,eAA0D;AAC9D,QAAM,WAAW,MAAM;AACrB,QAAI,CAAC,cAAc;AACjB,qBAAe,OAAO,2BAA2B,EAAE;AAAA,QAAK,CAAC,EAAE,SAAS,MAClE,SAAS,sBAAsB,kBAAkB,OAAO,GAAG;AAAA,UACzD,WAAW;AAAA,QACb,CAAC;AAAA,MACH;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAEA,SAAO,OAAO,cAAwB;AACpC,QAAI,UAAU,WAAW,GAAG;AAC1B,aAAO,EAAE,YAAY,CAAC,GAAG,YAAY,EAAE;AAAA,IACzC;AAEA,UAAM,QAAQ,MAAM,SAAS;AAC7B,UAAM,aAA6B,CAAC;AACpC,QAAI,aAAa;AAEjB,aAAS,QAAQ,GAAG,QAAQ,UAAU,QAAQ,SAAS,WAAW;AAChE,YAAM,SAAS,MAAM,MAAM,UAAU,MAAM,OAAO,QAAQ,SAAS,GAAG;AAAA,QACpE,SAAS;AAAA,QACT,WAAW;AAAA,MACb,CAAC;AACD,YAAM,QAAQ,mBAAmB,MAAM;AACvC,qBAAe,MAAM;AACrB,iBAAW,KAAK,GAAG,MAAM,UAAU;AAAA,IACrC;AAEA,WAAO,EAAE,YAAY,WAAW;AAAA,EAClC;AACF;",
6
- "names": []
7
- }
@@ -1,42 +0,0 @@
1
- async function textEmbeddings(inputs, { extractorFn, pooling, normalize }) {
2
- inputs = (Array.isArray(inputs) ? inputs : [inputs]).map((it) => it.trim());
3
- const extractor = await extractorFn();
4
- const tensor = await extractor(inputs, {
5
- pooling: pooling ?? "mean",
6
- normalize: normalize ?? true
7
- });
8
- return tensorToEmbeddings(tensor);
9
- }
10
- function tensorToEmbeddings(tensor) {
11
- const dims = tensor.dims;
12
- if (!Array.isArray(dims) || dims.length < 2) {
13
- throw new Error(`Unexpected tensor dims: ${JSON.stringify(dims)}`);
14
- }
15
- const batchSize = dims[0];
16
- const hiddenSize = dims[dims.length - 1];
17
- const expectedLen = batchSize * hiddenSize;
18
- if (tensor.data.length !== expectedLen) {
19
- throw new Error(
20
- `Data length mismatch: got ${tensor.data.length}, expected ${expectedLen} (batch=${batchSize}, hidden=${hiddenSize})`
21
- );
22
- }
23
- const data = tensor.data;
24
- if (!(data instanceof Float32Array)) {
25
- throw new Error(`Expected float32 embeddings, got ${tensor.type}`);
26
- }
27
- const embeddings = [];
28
- const dimensions = tensor.dims[tensor.dims.length - 1];
29
- for (let i = 0; i < batchSize; i++) {
30
- const start = i * hiddenSize;
31
- embeddings.push(data.subarray(start, start + hiddenSize));
32
- }
33
- return { embeddings, dimensions };
34
- }
35
- function huggingface(extractorFn) {
36
- return (documents) => textEmbeddings(documents, { extractorFn });
37
- }
38
- export {
39
- huggingface,
40
- tensorToEmbeddings
41
- };
42
- //# sourceMappingURL=huggingface.js.map
@@ -1,7 +0,0 @@
1
- {
2
- "version": 3,
3
- "sources": ["../../../src/lib/embedders/huggingface.ts"],
4
- "sourcesContent": ["import type {\n FeatureExtractionPipeline,\n Tensor,\n} from '@huggingface/transformers';\n\nimport type { Embedder } from '../stores/store.js';\n\nexport type FeatureExtractionFn = FeatureExtractionPipeline;\n\nexport interface EmbeddingOptions {\n extractorFn: () =>\n Promise<FeatureExtractionPipeline> | FeatureExtractionPipeline;\n pooling?: 'mean' | 'cls';\n normalize?: boolean;\n}\n\nasync function textEmbeddings(\n inputs: string | string[],\n { extractorFn, pooling, normalize }: EmbeddingOptions,\n) {\n inputs = (Array.isArray(inputs) ? inputs : [inputs]).map((it) => it.trim());\n const extractor = await extractorFn();\n const tensor = await extractor(inputs, {\n pooling: pooling ?? 'mean',\n normalize: normalize ?? true,\n });\n return tensorToEmbeddings(tensor);\n}\n\nexport function tensorToEmbeddings(tensor: Tensor) {\n const dims = tensor.dims;\n if (!Array.isArray(dims) || dims.length < 2) {\n throw new Error(`Unexpected tensor dims: ${JSON.stringify(dims)}`);\n }\n const batchSize = dims[0];\n const hiddenSize = dims[dims.length - 1];\n const expectedLen = batchSize * hiddenSize;\n if (tensor.data.length !== expectedLen) {\n throw new Error(\n `Data length mismatch: got ${tensor.data.length}, expected ${expectedLen} (batch=${batchSize}, hidden=${hiddenSize})`,\n );\n }\n const data = tensor.data;\n if (!(data instanceof Float32Array)) {\n throw new Error(`Expected float32 embeddings, got ${tensor.type}`);\n }\n // Reuse the underlying typed array without copying; we'll copy when creating Buffer for SQLite\n const embeddings: Float32Array[] = [];\n const dimensions = tensor.dims[tensor.dims.length - 1];\n for (let i = 0; i < batchSize; i++) {\n const start = i * hiddenSize;\n embeddings.push(data.subarray(start, start + hiddenSize));\n }\n return { embeddings, dimensions };\n}\n\nexport function huggingface(\n extractorFn: () =>\n Promise<FeatureExtractionPipeline> | FeatureExtractionPipeline,\n): Embedder {\n return (documents) => textEmbeddings(documents, { extractorFn });\n}\n"],
5
- "mappings": "AAgBA,eAAe,eACb,QACA,EAAE,aAAa,SAAS,UAAU,GAClC;AACA,YAAU,MAAM,QAAQ,MAAM,IAAI,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;AAC1E,QAAM,YAAY,MAAM,YAAY;AACpC,QAAM,SAAS,MAAM,UAAU,QAAQ;AAAA,IACrC,SAAS,WAAW;AAAA,IACpB,WAAW,aAAa;AAAA,EAC1B,CAAC;AACD,SAAO,mBAAmB,MAAM;AAClC;AAEO,SAAS,mBAAmB,QAAgB;AACjD,QAAM,OAAO,OAAO;AACpB,MAAI,CAAC,MAAM,QAAQ,IAAI,KAAK,KAAK,SAAS,GAAG;AAC3C,UAAM,IAAI,MAAM,2BAA2B,KAAK,UAAU,IAAI,CAAC,EAAE;AAAA,EACnE;AACA,QAAM,YAAY,KAAK,CAAC;AACxB,QAAM,aAAa,KAAK,KAAK,SAAS,CAAC;AACvC,QAAM,cAAc,YAAY;AAChC,MAAI,OAAO,KAAK,WAAW,aAAa;AACtC,UAAM,IAAI;AAAA,MACR,6BAA6B,OAAO,KAAK,MAAM,cAAc,WAAW,WAAW,SAAS,YAAY,UAAU;AAAA,IACpH;AAAA,EACF;AACA,QAAM,OAAO,OAAO;AACpB,MAAI,EAAE,gBAAgB,eAAe;AACnC,UAAM,IAAI,MAAM,oCAAoC,OAAO,IAAI,EAAE;AAAA,EACnE;AAEA,QAAM,aAA6B,CAAC;AACpC,QAAM,aAAa,OAAO,KAAK,OAAO,KAAK,SAAS,CAAC;AACrD,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,IAAI;AAClB,eAAW,KAAK,KAAK,SAAS,OAAO,QAAQ,UAAU,CAAC;AAAA,EAC1D;AACA,SAAO,EAAE,YAAY,WAAW;AAClC;AAEO,SAAS,YACd,aAEU;AACV,SAAO,CAAC,cAAc,eAAe,WAAW,EAAE,YAAY,CAAC;AACjE;",
6
- "names": []
7
- }