@deepagents/retrieval 7.2.0 → 7.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +7 -7
- package/dist/index.js +561 -7
- package/dist/index.js.map +4 -4
- package/dist/lib/connectors/index.d.ts +7 -7
- package/dist/lib/connectors/index.js +649 -7
- package/dist/lib/connectors/index.js.map +4 -4
- package/dist/lib/connectors/local.d.ts +1 -1
- package/dist/lib/connectors/local.d.ts.map +1 -1
- package/dist/lib/connectors/pdf.d.ts +1 -1
- package/dist/lib/connectors/pdf.d.ts.map +1 -1
- package/dist/lib/connectors/repo.d.ts +1 -1
- package/dist/lib/embedders/fastembed.d.ts +1 -1
- package/dist/lib/embedders/huggingface.d.ts +1 -1
- package/dist/lib/ingest.d.ts +3 -3
- package/dist/lib/similiarty-search.d.ts +1 -1
- package/dist/lib/stores/sqlite/ddl.sqlite.d.ts +1 -1
- package/dist/lib/stores/sqlite/ddl.sqlite.d.ts.map +1 -1
- package/dist/lib/stores/sqlite/node-sqlite.d.ts +1 -1
- package/dist/lib/stores/sqlite/open.d.ts +1 -2
- package/dist/lib/stores/sqlite/open.d.ts.map +1 -1
- package/dist/lib/stores/sqlite/sqlite.d.ts +1 -1
- package/package.json +4 -3
- package/dist/lib/connectors/connector.js +0 -1
- package/dist/lib/connectors/connector.js.map +0 -7
- package/dist/lib/connectors/github.js +0 -164
- package/dist/lib/connectors/github.js.map +0 -7
- package/dist/lib/connectors/linear.js +0 -29
- package/dist/lib/connectors/linear.js.map +0 -7
- package/dist/lib/connectors/local.js +0 -98
- package/dist/lib/connectors/local.js.map +0 -7
- package/dist/lib/connectors/pdf.js +0 -58
- package/dist/lib/connectors/pdf.js.map +0 -7
- package/dist/lib/connectors/repo.js +0 -179
- package/dist/lib/connectors/repo.js.map +0 -7
- package/dist/lib/connectors/rss.js +0 -128
- package/dist/lib/connectors/rss.js.map +0 -7
- package/dist/lib/embedders/fastembed.js +0 -63
- package/dist/lib/embedders/fastembed.js.map +0 -7
- package/dist/lib/embedders/huggingface.js +0 -42
- package/dist/lib/embedders/huggingface.js.map +0 -7
- package/dist/lib/ingest.js +0 -117
- package/dist/lib/ingest.js.map +0 -7
- package/dist/lib/pipeline.js +0 -1
- package/dist/lib/pipeline.js.map +0 -7
- package/dist/lib/sidecar.js +0 -1
- package/dist/lib/sidecar.js.map +0 -7
- package/dist/lib/similiarty-search.js +0 -42
- package/dist/lib/similiarty-search.js.map +0 -7
- package/dist/lib/stores/cid.js +0 -8
- package/dist/lib/stores/cid.js.map +0 -7
- package/dist/lib/stores/sqlite/ddl.sqlite.js +0 -53
- package/dist/lib/stores/sqlite/ddl.sqlite.js.map +0 -7
- package/dist/lib/stores/sqlite/node-sqlite.js +0 -9
- package/dist/lib/stores/sqlite/node-sqlite.js.map +0 -7
- package/dist/lib/stores/sqlite/open.js +0 -11
- package/dist/lib/stores/sqlite/open.js.map +0 -7
- package/dist/lib/stores/sqlite/sqlite.js +0 -245
- package/dist/lib/stores/sqlite/sqlite.js.map +0 -7
- package/dist/lib/stores/store.js +0 -1
- package/dist/lib/stores/store.js.map +0 -7
|
@@ -1,58 +0,0 @@
|
|
|
1
|
-
import { glob, readFile } from "node:fs/promises";
|
|
2
|
-
import { extractText, getDocumentProxy } from "unpdf";
|
|
3
|
-
function pdf(pattern) {
|
|
4
|
-
const sourceId = `pdf:${pattern}`;
|
|
5
|
-
return {
|
|
6
|
-
sourceId,
|
|
7
|
-
sources: async function* () {
|
|
8
|
-
const paths = await Array.fromAsync(
|
|
9
|
-
glob(pattern, { exclude: ["**/node_modules/**", "**/.git/**"] })
|
|
10
|
-
);
|
|
11
|
-
for (const path of paths) {
|
|
12
|
-
if (!path.toLowerCase().endsWith(".pdf")) continue;
|
|
13
|
-
yield {
|
|
14
|
-
id: path,
|
|
15
|
-
content: async () => {
|
|
16
|
-
const buffer = await readFile(path);
|
|
17
|
-
const pdf2 = await getDocumentProxy(new Uint8Array(buffer));
|
|
18
|
-
const { text } = await extractText(pdf2, { mergePages: true });
|
|
19
|
-
return text;
|
|
20
|
-
}
|
|
21
|
-
};
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
};
|
|
25
|
-
}
|
|
26
|
-
function pdfFile(source) {
|
|
27
|
-
const isUrl = /^https?:\/\//.test(source);
|
|
28
|
-
const sourceId = `pdf:${isUrl ? "url" : "file"}:${source}`;
|
|
29
|
-
return {
|
|
30
|
-
sourceId,
|
|
31
|
-
sources: async function* () {
|
|
32
|
-
yield {
|
|
33
|
-
id: source,
|
|
34
|
-
content: async () => {
|
|
35
|
-
let buffer;
|
|
36
|
-
if (isUrl) {
|
|
37
|
-
const response = await fetch(source);
|
|
38
|
-
if (!response.ok) {
|
|
39
|
-
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
40
|
-
}
|
|
41
|
-
buffer = new Uint8Array(await response.arrayBuffer());
|
|
42
|
-
} else {
|
|
43
|
-
const fileBuffer = await readFile(source);
|
|
44
|
-
buffer = new Uint8Array(fileBuffer.buffer);
|
|
45
|
-
}
|
|
46
|
-
const pdf2 = await getDocumentProxy(new Uint8Array(buffer));
|
|
47
|
-
const { text } = await extractText(pdf2, { mergePages: true });
|
|
48
|
-
return text;
|
|
49
|
-
}
|
|
50
|
-
};
|
|
51
|
-
}
|
|
52
|
-
};
|
|
53
|
-
}
|
|
54
|
-
export {
|
|
55
|
-
pdf,
|
|
56
|
-
pdfFile
|
|
57
|
-
};
|
|
58
|
-
//# sourceMappingURL=pdf.js.map
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": 3,
|
|
3
|
-
"sources": ["../../../src/lib/connectors/pdf.ts"],
|
|
4
|
-
"sourcesContent": ["import { glob, readFile } from 'node:fs/promises';\n\nimport { extractText, getDocumentProxy } from 'unpdf';\n\nimport type { Connector } from './connector.js';\n\nexport function pdf(pattern: string): Connector {\n const sourceId = `pdf:${pattern}`;\n\n return {\n sourceId,\n sources: async function* () {\n const paths = await Array.fromAsync(\n glob(pattern, { exclude: ['**/node_modules/**', '**/.git/**'] }),\n );\n for (const path of paths) {\n if (!path.toLowerCase().endsWith('.pdf')) continue;\n yield {\n id: path,\n content: async () => {\n const buffer = await readFile(path);\n const pdf = await getDocumentProxy(new Uint8Array(buffer));\n const { text } = await extractText(pdf, { mergePages: true });\n return text;\n },\n };\n }\n },\n };\n}\n\nexport function pdfFile(source: string): Connector {\n const isUrl = /^https?:\\/\\//.test(source);\n const sourceId = `pdf:${isUrl ? 'url' : 'file'}:${source}`;\n\n return {\n sourceId,\n sources: async function* () {\n yield {\n id: source,\n content: async () => {\n let buffer: Uint8Array;\n\n if (isUrl) {\n const response = await fetch(source);\n if (!response.ok) {\n throw new Error(`HTTP ${response.status}: ${response.statusText}`);\n }\n buffer = new Uint8Array(await response.arrayBuffer());\n } else {\n const fileBuffer = await readFile(source);\n buffer = new Uint8Array(fileBuffer.buffer);\n }\n\n const pdf = await getDocumentProxy(new Uint8Array(buffer));\n const { text } = await extractText(pdf, { mergePages: true });\n return text;\n },\n };\n },\n };\n}\n"],
|
|
5
|
-
"mappings": "AAAA,SAAS,MAAM,gBAAgB;AAE/B,SAAS,aAAa,wBAAwB;AAIvC,SAAS,IAAI,SAA4B;AAC9C,QAAM,WAAW,OAAO,OAAO;AAE/B,SAAO;AAAA,IACL;AAAA,IACA,SAAS,mBAAmB;AAC1B,YAAM,QAAQ,MAAM,MAAM;AAAA,QACxB,KAAK,SAAS,EAAE,SAAS,CAAC,sBAAsB,YAAY,EAAE,CAAC;AAAA,MACjE;AACA,iBAAW,QAAQ,OAAO;AACxB,YAAI,CAAC,KAAK,YAAY,EAAE,SAAS,MAAM,EAAG;AAC1C,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,SAAS,YAAY;AACnB,kBAAM,SAAS,MAAM,SAAS,IAAI;AAClC,kBAAMA,OAAM,MAAM,iBAAiB,IAAI,WAAW,MAAM,CAAC;AACzD,kBAAM,EAAE,KAAK,IAAI,MAAM,YAAYA,MAAK,EAAE,YAAY,KAAK,CAAC;AAC5D,mBAAO;AAAA,UACT;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEO,SAAS,QAAQ,QAA2B;AACjD,QAAM,QAAQ,eAAe,KAAK,MAAM;AACxC,QAAM,WAAW,OAAO,QAAQ,QAAQ,MAAM,IAAI,MAAM;AAExD,SAAO;AAAA,IACL;AAAA,IACA,SAAS,mBAAmB;AAC1B,YAAM;AAAA,QACJ,IAAI;AAAA,QACJ,SAAS,YAAY;AACnB,cAAI;AAEJ,cAAI,OAAO;AACT,kBAAM,WAAW,MAAM,MAAM,MAAM;AACnC,gBAAI,CAAC,SAAS,IAAI;AAChB,oBAAM,IAAI,MAAM,QAAQ,SAAS,MAAM,KAAK,SAAS,UAAU,EAAE;AAAA,YACnE;AACA,qBAAS,IAAI,WAAW,MAAM,SAAS,YAAY,CAAC;AAAA,UACtD,OAAO;AACL,kBAAM,aAAa,MAAM,SAAS,MAAM;AACxC,qBAAS,IAAI,WAAW,WAAW,MAAM;AAAA,UAC3C;AAEA,gBAAMA,OAAM,MAAM,iBAAiB,IAAI,WAAW,MAAM,CAAC;AACzD,gBAAM,EAAE,KAAK,IAAI,MAAM,YAAYA,MAAK,EAAE,YAAY,KAAK,CAAC;AAC5D,iBAAO;AAAA,QACT;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;",
|
|
6
|
-
"names": ["pdf"]
|
|
7
|
-
}
|
|
@@ -1,179 +0,0 @@
|
|
|
1
|
-
import fg from "fast-glob";
|
|
2
|
-
import { opendir, readFile, stat } from "node:fs/promises";
|
|
3
|
-
import { join } from "node:path";
|
|
4
|
-
function repo(dir, extensions, ingestWhen) {
|
|
5
|
-
const sourceId = `repo:${dir}`;
|
|
6
|
-
return {
|
|
7
|
-
sourceId,
|
|
8
|
-
ingestWhen,
|
|
9
|
-
sources: async function* () {
|
|
10
|
-
const paths = await collectFiles(dir, extensions);
|
|
11
|
-
for await (const path of paths) {
|
|
12
|
-
const maxSize = 3 * 1024;
|
|
13
|
-
const st = await stat(path);
|
|
14
|
-
if (st.size > maxSize) {
|
|
15
|
-
continue;
|
|
16
|
-
}
|
|
17
|
-
yield {
|
|
18
|
-
id: path,
|
|
19
|
-
metadata: { repo: dir },
|
|
20
|
-
content: () => readFile(path, "utf8").catch(() => "")
|
|
21
|
-
};
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
};
|
|
25
|
-
}
|
|
26
|
-
async function* findAllGitRepos(root) {
|
|
27
|
-
const stack = [root];
|
|
28
|
-
const skip = [
|
|
29
|
-
"node_modules",
|
|
30
|
-
"Library",
|
|
31
|
-
"Applications",
|
|
32
|
-
"Pictures",
|
|
33
|
-
"Movies",
|
|
34
|
-
"Music",
|
|
35
|
-
"Downloads",
|
|
36
|
-
".cache",
|
|
37
|
-
".npm",
|
|
38
|
-
".pnpm",
|
|
39
|
-
"development"
|
|
40
|
-
];
|
|
41
|
-
while (stack.length) {
|
|
42
|
-
const folder = stack.pop();
|
|
43
|
-
const isGitRepo = await stat(join(folder, ".git")).then((st) => st.isDirectory() || st.isFile()).catch(() => false);
|
|
44
|
-
if (isGitRepo) {
|
|
45
|
-
yield folder;
|
|
46
|
-
continue;
|
|
47
|
-
}
|
|
48
|
-
const dir = await opendir(folder);
|
|
49
|
-
for await (const dirent of dir) {
|
|
50
|
-
if (dirent.isDirectory() && !dirent.isSymbolicLink() && !skip.includes(dirent.name) && !dirent.name.startsWith(".")) {
|
|
51
|
-
stack.push(join(folder, dirent.name));
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
function detectRepoTooling(repo2) {
|
|
57
|
-
const tools = [];
|
|
58
|
-
return tools;
|
|
59
|
-
}
|
|
60
|
-
async function gitignore(gitignoreFile) {
|
|
61
|
-
const content = await readFile(gitignoreFile, "utf8").catch(() => "");
|
|
62
|
-
if (!content) {
|
|
63
|
-
return [];
|
|
64
|
-
}
|
|
65
|
-
const patterns = content.split("\n").map((line) => line.trim()).filter((line) => line && !line.startsWith("#"));
|
|
66
|
-
return patterns;
|
|
67
|
-
}
|
|
68
|
-
async function ignorePatterns(repo2) {
|
|
69
|
-
return [
|
|
70
|
-
// Inherit repo-specific ignore patterns
|
|
71
|
-
...await gitignore(join(repo2, ".gitignore")),
|
|
72
|
-
// Package managers & dependency dirs
|
|
73
|
-
"node_modules/**",
|
|
74
|
-
"**/node_modules/**",
|
|
75
|
-
"**/.pnpm/**",
|
|
76
|
-
"**/.npm/**",
|
|
77
|
-
"**/.yarn/**",
|
|
78
|
-
"**/vendor/**",
|
|
79
|
-
// PHP / Go modules (when vendored)
|
|
80
|
-
"**/3rdparty/**",
|
|
81
|
-
// Version control + VCS metadata
|
|
82
|
-
"**/.git/**",
|
|
83
|
-
"**/.svn/**",
|
|
84
|
-
"**/.hg/**",
|
|
85
|
-
// OS / system junk
|
|
86
|
-
"**/.DS_Store",
|
|
87
|
-
"**/Thumbs.db",
|
|
88
|
-
"**/Library/**",
|
|
89
|
-
"**/Applications/**",
|
|
90
|
-
"**/Pictures/**",
|
|
91
|
-
"**/Movies/**",
|
|
92
|
-
"**/Music/**",
|
|
93
|
-
"**/Downloads/**",
|
|
94
|
-
"**/.cache/**",
|
|
95
|
-
// Environment / secrets (explicit)
|
|
96
|
-
"**/.env",
|
|
97
|
-
"**/.env.*",
|
|
98
|
-
// Lockfiles & generated dependency state
|
|
99
|
-
"**/*.lock",
|
|
100
|
-
"**/yarn.lock",
|
|
101
|
-
"**/package-lock.json",
|
|
102
|
-
"**/pnpm-lock.yaml",
|
|
103
|
-
// Build / compilation outputs
|
|
104
|
-
"**/dist/**",
|
|
105
|
-
"**/debug/**",
|
|
106
|
-
"**/build/**",
|
|
107
|
-
"**/out/**",
|
|
108
|
-
"**/target/**",
|
|
109
|
-
// Rust / JVM
|
|
110
|
-
"**/bin/**",
|
|
111
|
-
"**/obj/**",
|
|
112
|
-
"**/classes/**",
|
|
113
|
-
// Framework / tool specific build artifacts
|
|
114
|
-
"**/.next/**",
|
|
115
|
-
"**/.vercel/**",
|
|
116
|
-
"**/.turbo/**",
|
|
117
|
-
"**/.docusaurus/**",
|
|
118
|
-
"**/.vite/**",
|
|
119
|
-
"**/.parcel-cache/**",
|
|
120
|
-
"**/.rollup.cache/**",
|
|
121
|
-
"**/.vuepress/**",
|
|
122
|
-
"cdk.out/**",
|
|
123
|
-
// Infra & deployment tooling
|
|
124
|
-
"**/.serverless/**",
|
|
125
|
-
"**/.terraform/**",
|
|
126
|
-
"**/.terragrunt-cache/**",
|
|
127
|
-
"**/.pulumi/**",
|
|
128
|
-
// Coverage & testing caches
|
|
129
|
-
"**/coverage/**",
|
|
130
|
-
"**/.nyc_output/**",
|
|
131
|
-
"**/jest-cache/**",
|
|
132
|
-
"**/.pytest_cache/**",
|
|
133
|
-
// Language / tooling caches
|
|
134
|
-
"**/__pycache__/**",
|
|
135
|
-
"**/.mypy_cache/**",
|
|
136
|
-
"**/.tox/**",
|
|
137
|
-
"**/.gradle/**",
|
|
138
|
-
"**/.mvn/**",
|
|
139
|
-
"**/.eslintcache",
|
|
140
|
-
"**/.stylelintcache",
|
|
141
|
-
// IDE / editor configs + history (we don't want to embed these)
|
|
142
|
-
"**/.idea/**",
|
|
143
|
-
"**/.vscode/**",
|
|
144
|
-
"**/.fleet/**",
|
|
145
|
-
"**/.history/**",
|
|
146
|
-
// Virtual environments
|
|
147
|
-
"**/.venv/**",
|
|
148
|
-
"**/venv/**"
|
|
149
|
-
];
|
|
150
|
-
}
|
|
151
|
-
async function collectFiles(repo2, extensions) {
|
|
152
|
-
const exts = extensions.map((ext) => ext.replace(/^\./, ""));
|
|
153
|
-
const entries = fg.stream(
|
|
154
|
-
extensions.length > 1 ? `**/*.{${exts.join(",")}}` : `**/*.${exts[0]}`,
|
|
155
|
-
{
|
|
156
|
-
dot: false,
|
|
157
|
-
onlyFiles: true,
|
|
158
|
-
unique: true,
|
|
159
|
-
absolute: true,
|
|
160
|
-
cwd: repo2,
|
|
161
|
-
ignore: await ignorePatterns(repo2)
|
|
162
|
-
}
|
|
163
|
-
);
|
|
164
|
-
return (async function* () {
|
|
165
|
-
for await (const entry of entries) {
|
|
166
|
-
if (typeof entry !== "string") {
|
|
167
|
-
throw new TypeError("fast-glob yielded a non-path entry");
|
|
168
|
-
}
|
|
169
|
-
yield entry;
|
|
170
|
-
}
|
|
171
|
-
})();
|
|
172
|
-
}
|
|
173
|
-
export {
|
|
174
|
-
collectFiles,
|
|
175
|
-
findAllGitRepos,
|
|
176
|
-
ignorePatterns,
|
|
177
|
-
repo
|
|
178
|
-
};
|
|
179
|
-
//# sourceMappingURL=repo.js.map
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": 3,
|
|
3
|
-
"sources": ["../../../src/lib/connectors/repo.ts"],
|
|
4
|
-
"sourcesContent": ["import fg from 'fast-glob';\nimport { opendir, readFile, stat } from 'node:fs/promises';\nimport { join } from 'node:path';\n\nimport type { Connector } from './connector.js';\n\nexport function repo(\n dir: string,\n extensions: string[],\n ingestWhen: Connector['ingestWhen'],\n): Connector {\n const sourceId = `repo:${dir}`;\n return {\n sourceId,\n ingestWhen,\n sources: async function* () {\n const paths = await collectFiles(dir, extensions);\n for await (const path of paths) {\n const maxSize = 3 * 1024; // 3KB\n const st = await stat(path);\n if (st.size > maxSize) {\n continue;\n }\n yield {\n id: path,\n metadata: { repo: dir },\n content: () => readFile(path, 'utf8').catch(() => ''),\n };\n }\n },\n };\n}\n\nexport async function* findAllGitRepos(root: string) {\n const stack = [root];\n const skip = [\n 'node_modules',\n 'Library',\n 'Applications',\n 'Pictures',\n 'Movies',\n 'Music',\n 'Downloads',\n '.cache',\n '.npm',\n '.pnpm',\n 'development',\n ];\n while (stack.length) {\n const folder = stack.pop()!;\n const isGitRepo = await stat(join(folder, '.git'))\n .then((st) => st.isDirectory() || st.isFile())\n .catch(() => false);\n if (isGitRepo) {\n yield folder;\n continue;\n }\n const dir = await opendir(folder);\n for await (const dirent of dir) {\n if (\n dirent.isDirectory() &&\n !dirent.isSymbolicLink() &&\n !skip.includes(dirent.name) &&\n !dirent.name.startsWith('.')\n ) {\n stack.push(join(folder, dirent.name));\n }\n }\n }\n}\n\nfunction detectRepoTooling(repo: string) {\n const tools: string[] = [];\n return tools;\n}\n\nasync function gitignore(gitignoreFile: string) {\n const content = await readFile(gitignoreFile, 'utf8').catch(() => '');\n if (!content) {\n return [];\n }\n const patterns = content\n .split('\\n')\n .map((line) => line.trim())\n .filter((line) => line && !line.startsWith('#'));\n return patterns;\n}\n\nexport async function ignorePatterns(repo: string) {\n return [\n // Inherit repo-specific ignore patterns\n ...(await gitignore(join(repo, '.gitignore'))),\n\n // Package managers & dependency dirs\n 'node_modules/**',\n '**/node_modules/**',\n '**/.pnpm/**',\n '**/.npm/**',\n '**/.yarn/**',\n '**/vendor/**', // PHP / Go modules (when vendored)\n '**/3rdparty/**',\n\n // Version control + VCS metadata\n '**/.git/**',\n '**/.svn/**',\n '**/.hg/**',\n\n // OS / system junk\n '**/.DS_Store',\n '**/Thumbs.db',\n '**/Library/**',\n '**/Applications/**',\n '**/Pictures/**',\n '**/Movies/**',\n '**/Music/**',\n '**/Downloads/**',\n '**/.cache/**',\n\n // Environment / secrets (explicit)\n '**/.env',\n '**/.env.*',\n\n // Lockfiles & generated dependency state\n '**/*.lock',\n '**/yarn.lock',\n '**/package-lock.json',\n '**/pnpm-lock.yaml',\n\n // Build / compilation outputs\n '**/dist/**',\n '**/debug/**',\n '**/build/**',\n '**/out/**',\n '**/target/**', // Rust / JVM\n '**/bin/**',\n '**/obj/**',\n '**/classes/**',\n\n // Framework / tool specific build artifacts\n '**/.next/**',\n '**/.vercel/**',\n '**/.turbo/**',\n '**/.docusaurus/**',\n '**/.vite/**',\n '**/.parcel-cache/**',\n '**/.rollup.cache/**',\n '**/.vuepress/**',\n 'cdk.out/**',\n\n // Infra & deployment tooling\n '**/.serverless/**',\n '**/.terraform/**',\n '**/.terragrunt-cache/**',\n '**/.pulumi/**',\n\n // Coverage & testing caches\n '**/coverage/**',\n '**/.nyc_output/**',\n '**/jest-cache/**',\n '**/.pytest_cache/**',\n\n // Language / tooling caches\n '**/__pycache__/**',\n '**/.mypy_cache/**',\n '**/.tox/**',\n '**/.gradle/**',\n '**/.mvn/**',\n '**/.eslintcache',\n '**/.stylelintcache',\n\n // IDE / editor configs + history (we don't want to embed these)\n '**/.idea/**',\n '**/.vscode/**',\n '**/.fleet/**',\n '**/.history/**',\n\n // Virtual environments\n '**/.venv/**',\n '**/venv/**',\n ];\n}\n\nexport async function collectFiles(\n repo: string,\n extensions: string[],\n): Promise<AsyncIterable<string>> {\n const exts = extensions.map((ext) => ext.replace(/^\\./, ''));\n const entries = fg.stream(\n extensions.length > 1 ? `**/*.{${exts.join(',')}}` : `**/*.${exts[0]}`,\n {\n dot: false,\n onlyFiles: true,\n unique: true,\n absolute: true,\n cwd: repo,\n ignore: await ignorePatterns(repo),\n },\n );\n return (async function* () {\n for await (const entry of entries) {\n if (typeof entry !== 'string') {\n throw new TypeError('fast-glob yielded a non-path entry');\n }\n yield entry;\n }\n })();\n}\n"],
|
|
5
|
-
"mappings": "AAAA,OAAO,QAAQ;AACf,SAAS,SAAS,UAAU,YAAY;AACxC,SAAS,YAAY;AAId,SAAS,KACd,KACA,YACA,YACW;AACX,QAAM,WAAW,QAAQ,GAAG;AAC5B,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,SAAS,mBAAmB;AAC1B,YAAM,QAAQ,MAAM,aAAa,KAAK,UAAU;AAChD,uBAAiB,QAAQ,OAAO;AAC9B,cAAM,UAAU,IAAI;AACpB,cAAM,KAAK,MAAM,KAAK,IAAI;AAC1B,YAAI,GAAG,OAAO,SAAS;AACrB;AAAA,QACF;AACA,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,UAAU,EAAE,MAAM,IAAI;AAAA,UACtB,SAAS,MAAM,SAAS,MAAM,MAAM,EAAE,MAAM,MAAM,EAAE;AAAA,QACtD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,gBAAuB,gBAAgB,MAAc;AACnD,QAAM,QAAQ,CAAC,IAAI;AACnB,QAAM,OAAO;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,SAAO,MAAM,QAAQ;AACnB,UAAM,SAAS,MAAM,IAAI;AACzB,UAAM,YAAY,MAAM,KAAK,KAAK,QAAQ,MAAM,CAAC,EAC9C,KAAK,CAAC,OAAO,GAAG,YAAY,KAAK,GAAG,OAAO,CAAC,EAC5C,MAAM,MAAM,KAAK;AACpB,QAAI,WAAW;AACb,YAAM;AACN;AAAA,IACF;AACA,UAAM,MAAM,MAAM,QAAQ,MAAM;AAChC,qBAAiB,UAAU,KAAK;AAC9B,UACE,OAAO,YAAY,KACnB,CAAC,OAAO,eAAe,KACvB,CAAC,KAAK,SAAS,OAAO,IAAI,KAC1B,CAAC,OAAO,KAAK,WAAW,GAAG,GAC3B;AACA,cAAM,KAAK,KAAK,QAAQ,OAAO,IAAI,CAAC;AAAA,MACtC;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,kBAAkBA,OAAc;AACvC,QAAM,QAAkB,CAAC;AACzB,SAAO;AACT;AAEA,eAAe,UAAU,eAAuB;AAC9C,QAAM,UAAU,MAAM,SAAS,eAAe,MAAM,EAAE,MAAM,MAAM,EAAE;AACpE,MAAI,CAAC,SAAS;AACZ,WAAO,CAAC;AAAA,EACV;AACA,QAAM,WAAW,QACd,MAAM,IAAI,EACV,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,OAAO,CAAC,SAAS,QAAQ,CAAC,KAAK,WAAW,GAAG,CAAC;AACjD,SAAO;AACT;AAEA,eAAsB,eAAeA,OAAc;AACjD,SAAO;AAAA;AAAA,IAEL,GAAI,MAAM,UAAU,KAAKA,OAAM,YAAY,CAAC;AAAA;AAAA,IAG5C;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,EACF;AACF;AAEA,eAAsB,aACpBA,OACA,YACgC;AAChC,QAAM,OAAO,WAAW,IAAI,CAAC,QAAQ,IAAI,QAAQ,OAAO,EAAE,CAAC;AAC3D,QAAM,UAAU,GAAG;AAAA,IACjB,WAAW,SAAS,IAAI,SAAS,KAAK,KAAK,GAAG,CAAC,MAAM,QAAQ,KAAK,CAAC,CAAC;AAAA,IACpE;AAAA,MACE,KAAK;AAAA,MACL,WAAW;AAAA,MACX,QAAQ;AAAA,MACR,UAAU;AAAA,MACV,KAAKA;AAAA,MACL,QAAQ,MAAM,eAAeA,KAAI;AAAA,IACnC;AAAA,EACF;AACA,UAAQ,mBAAmB;AACzB,qBAAiB,SAAS,SAAS;AACjC,UAAI,OAAO,UAAU,UAAU;AAC7B,cAAM,IAAI,UAAU,oCAAoC;AAAA,MAC1D;AACA,YAAM;AAAA,IACR;AAAA,EACF,GAAG;AACL;",
|
|
6
|
-
"names": ["repo"]
|
|
7
|
-
}
|
|
@@ -1,128 +0,0 @@
|
|
|
1
|
-
import { experimental_createWebFetchTool } from "@ai-sdk/tool-web-fetch";
|
|
2
|
-
import assert from "node:assert/strict";
|
|
3
|
-
import Parser from "rss-parser";
|
|
4
|
-
const webFetch = experimental_createWebFetchTool({
|
|
5
|
-
// Ingestion stores the complete article in one result.
|
|
6
|
-
maxLength: Number.MAX_SAFE_INTEGER
|
|
7
|
-
});
|
|
8
|
-
const rssParser = new Parser({
|
|
9
|
-
customFields: {
|
|
10
|
-
feed: ["copyright", "language", "managingEditor", "webMaster"],
|
|
11
|
-
item: ["category", "creator", "enclosure", "guid"]
|
|
12
|
-
}
|
|
13
|
-
});
|
|
14
|
-
async function parseRSSFeed(feedUrl) {
|
|
15
|
-
try {
|
|
16
|
-
console.log(`Fetching RSS feed: ${feedUrl}`);
|
|
17
|
-
const feed = await rssParser.parseURL(feedUrl);
|
|
18
|
-
return {
|
|
19
|
-
title: feed.title || "",
|
|
20
|
-
description: feed.description || "",
|
|
21
|
-
link: feed.link || "",
|
|
22
|
-
language: feed.language || "en",
|
|
23
|
-
lastBuildDate: feed.lastBuildDate || (/* @__PURE__ */ new Date()).toISOString(),
|
|
24
|
-
items: feed.items.map((item) => ({
|
|
25
|
-
title: item.title || "",
|
|
26
|
-
description: item.content || item.summary || item.contentSnippet || "",
|
|
27
|
-
link: item.link || "",
|
|
28
|
-
pubDate: item.pubDate || item.isoDate || "",
|
|
29
|
-
author: item.creator || item.author || "",
|
|
30
|
-
categories: Array.isArray(item.categories) ? item.categories : item.category ? [item.category] : [],
|
|
31
|
-
guid: item.guid || item.guid || "",
|
|
32
|
-
contentEncoded: item["content:encoded"] || item.content || ""
|
|
33
|
-
}))
|
|
34
|
-
};
|
|
35
|
-
} catch (error) {
|
|
36
|
-
console.error(`Failed to parse RSS feed ${feedUrl}:`, error);
|
|
37
|
-
throw new Error(
|
|
38
|
-
`RSS parsing failed: ${error instanceof Error ? error.message : "Unknown error"}`
|
|
39
|
-
);
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
async function fetchArticleContent(url) {
|
|
43
|
-
try {
|
|
44
|
-
console.log(`Fetching full article content from: ${url}`);
|
|
45
|
-
const article = await webFetch.execute(
|
|
46
|
-
{ url },
|
|
47
|
-
{ toolCallId: url, messages: [], context: {} }
|
|
48
|
-
);
|
|
49
|
-
assert.ok("type" in article, "Web fetch must return a complete result");
|
|
50
|
-
if (article.type === "error") {
|
|
51
|
-
throw new Error(article.message);
|
|
52
|
-
}
|
|
53
|
-
const fullContent = article.title ? `${article.title}
|
|
54
|
-
|
|
55
|
-
${article.content}` : article.content;
|
|
56
|
-
if (fullContent.length < 200) {
|
|
57
|
-
throw new Error(
|
|
58
|
-
`Extracted content too short (${fullContent.length} chars) - likely failed to find main content`
|
|
59
|
-
);
|
|
60
|
-
}
|
|
61
|
-
console.log(
|
|
62
|
-
`\u2713 Successfully extracted article: "${article.title ?? ""}" (${fullContent.length} chars, ~${Math.round(fullContent.split(" ").length)} words)`
|
|
63
|
-
);
|
|
64
|
-
return fullContent;
|
|
65
|
-
} catch (error) {
|
|
66
|
-
console.warn(
|
|
67
|
-
`Failed to fetch article content from ${url}:`,
|
|
68
|
-
error instanceof Error ? error.message : error
|
|
69
|
-
);
|
|
70
|
-
return "";
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
function rss(feedUrl, options = {}) {
|
|
74
|
-
const sourceId = `rss:${feedUrl}`;
|
|
75
|
-
const { maxItems = 50, fetchFullArticles = false } = options;
|
|
76
|
-
return {
|
|
77
|
-
sourceId,
|
|
78
|
-
instructions: `You answer questions about articles and content from the RSS feed: ${feedUrl}.
|
|
79
|
-
Always cite the article title and link when referencing specific content.
|
|
80
|
-
The feed contains recent articles, blog posts, and news items.
|
|
81
|
-
When referencing content, include the publication date and author when available.
|
|
82
|
-
${fetchFullArticles ? "Full article content has been extracted from the original links for comprehensive analysis." : "Content includes RSS summaries and descriptions."}`,
|
|
83
|
-
sources: async function* () {
|
|
84
|
-
const feed = await parseRSSFeed(feedUrl);
|
|
85
|
-
const feedSummary = `RSS Feed: ${feed.title}
|
|
86
|
-
Description: ${feed.description}
|
|
87
|
-
Website: ${feed.link}
|
|
88
|
-
Language: ${feed.language}
|
|
89
|
-
Last Updated: ${feed.lastBuildDate}
|
|
90
|
-
Total Items: ${feed.items.length}
|
|
91
|
-
|
|
92
|
-
This feed provides: ${feed.description}`;
|
|
93
|
-
yield {
|
|
94
|
-
id: "feed-info",
|
|
95
|
-
content: async () => feedSummary
|
|
96
|
-
};
|
|
97
|
-
const itemsToProcess = feed.items.slice(0, maxItems);
|
|
98
|
-
for (const item of itemsToProcess) {
|
|
99
|
-
const documentId = item.guid || item.link || `${item.title}-${item.pubDate}`;
|
|
100
|
-
yield {
|
|
101
|
-
id: documentId,
|
|
102
|
-
content: async () => {
|
|
103
|
-
let articleContent = item.contentEncoded || item.description;
|
|
104
|
-
if (fetchFullArticles && item.link) {
|
|
105
|
-
const fullContent = await fetchArticleContent(item.link);
|
|
106
|
-
if (fullContent && fullContent.length > articleContent.length) {
|
|
107
|
-
articleContent = fullContent;
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
return `Title: ${item.title}
|
|
111
|
-
Author: ${item.author}
|
|
112
|
-
Published: ${item.pubDate}
|
|
113
|
-
Categories: ${item.categories.join(", ")}
|
|
114
|
-
Link: ${item.link}
|
|
115
|
-
${fetchFullArticles ? "Full Article Content:" : "Content:"}
|
|
116
|
-
${articleContent}
|
|
117
|
-
|
|
118
|
-
Summary: ${item.title} - ${item.description}`;
|
|
119
|
-
}
|
|
120
|
-
};
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
};
|
|
124
|
-
}
|
|
125
|
-
export {
|
|
126
|
-
rss
|
|
127
|
-
};
|
|
128
|
-
//# sourceMappingURL=rss.js.map
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": 3,
|
|
3
|
-
"sources": ["../../../src/lib/connectors/rss.ts"],
|
|
4
|
-
"sourcesContent": ["import { experimental_createWebFetchTool } from '@ai-sdk/tool-web-fetch';\nimport assert from 'node:assert/strict';\nimport Parser from 'rss-parser';\n\nconst webFetch = experimental_createWebFetchTool({\n // Ingestion stores the complete article in one result.\n maxLength: Number.MAX_SAFE_INTEGER,\n});\n\ntype FeedFields = {\n copyright?: string;\n language?: string;\n managingEditor?: string;\n webMaster?: string;\n lastBuildDate?: string;\n};\n\ntype ItemFields = {\n category?: string;\n creator?: string;\n enclosure?: Parser.Enclosure;\n guid?: string;\n author?: string;\n 'content:encoded'?: string;\n};\n\nconst rssParser = new Parser<FeedFields, ItemFields>({\n customFields: {\n feed: ['copyright', 'language', 'managingEditor', 'webMaster'],\n item: ['category', 'creator', 'enclosure', 'guid'],\n },\n});\n\nasync function parseRSSFeed(feedUrl: string) {\n try {\n console.log(`Fetching RSS feed: ${feedUrl}`);\n const feed = await rssParser.parseURL(feedUrl);\n\n return {\n title: feed.title || '',\n description: feed.description || '',\n link: feed.link || '',\n language: feed.language || 'en',\n lastBuildDate: feed.lastBuildDate || new Date().toISOString(),\n items: feed.items.map((item) => ({\n title: item.title || '',\n description: item.content || item.summary || item.contentSnippet || '',\n link: item.link || '',\n pubDate: item.pubDate || item.isoDate || '',\n author: item.creator || item.author || '',\n categories: Array.isArray(item.categories)\n ? item.categories\n : item.category\n ? [item.category]\n : [],\n guid: item.guid || item.guid || '',\n contentEncoded: item['content:encoded'] || item.content || '',\n })),\n };\n } catch (error) {\n console.error(`Failed to parse RSS feed ${feedUrl}:`, error);\n throw new Error(\n `RSS parsing failed: ${error instanceof Error ? error.message : 'Unknown error'}`,\n );\n }\n}\n\nasync function fetchArticleContent(url: string): Promise<string> {\n try {\n console.log(`Fetching full article content from: ${url}`);\n\n const article = await webFetch.execute(\n { url },\n { toolCallId: url, messages: [], context: {} },\n );\n assert.ok('type' in article, 'Web fetch must return a complete result');\n if (article.type === 'error') {\n throw new Error(article.message);\n }\n\n // Combine title and content for comprehensive text\n const fullContent = article.title\n ? `${article.title}\\n\\n${article.content}`\n : article.content;\n\n // Ensure we have meaningful content (at least 200 characters for full articles)\n if (fullContent.length < 200) {\n throw new Error(\n `Extracted content too short (${fullContent.length} chars) - likely failed to find main content`,\n );\n }\n\n console.log(\n `\u2713 Successfully extracted article: \"${article.title ?? ''}\" (${fullContent.length} chars, ~${Math.round(fullContent.split(' ').length)} words)`,\n );\n\n return fullContent;\n } catch (error) {\n console.warn(\n `Failed to fetch article content from ${url}:`,\n error instanceof Error ? error.message : error,\n );\n return ''; // Return empty string on failure rather than throwing\n }\n}\nexport function rss(\n feedUrl: string,\n options: {\n maxItems?: number;\n fetchFullArticles?: boolean;\n } = {},\n) {\n const sourceId = `rss:${feedUrl}`;\n const { maxItems = 50, fetchFullArticles = false } = options;\n\n return {\n sourceId,\n instructions: `You answer questions about articles and content from the RSS feed: ${feedUrl}.\n Always cite the article title and link when referencing specific content.\n The feed contains recent articles, blog posts, and news items.\n When referencing content, include the publication date and author when available.\n ${fetchFullArticles ? 'Full article content has been extracted from the original links for comprehensive analysis.' : 'Content includes RSS summaries and descriptions.'}`,\n sources: async function* () {\n const feed = await parseRSSFeed(feedUrl);\n // Add feed summary source\n const feedSummary = `RSS Feed: ${feed.title}\nDescription: ${feed.description}\nWebsite: ${feed.link}\nLanguage: ${feed.language}\nLast Updated: ${feed.lastBuildDate}\nTotal Items: ${feed.items.length}\n\nThis feed provides: ${feed.description}`;\n yield {\n id: 'feed-info',\n content: async () => feedSummary,\n };\n\n // Individual article sources (limit to maxItems)\n const itemsToProcess = feed.items.slice(0, maxItems);\n for (const item of itemsToProcess) {\n const documentId =\n item.guid || item.link || `${item.title}-${item.pubDate}`;\n yield {\n id: documentId,\n content: async () => {\n // Try full article fetch if enabled, fallback to RSS content\n let articleContent = item.contentEncoded || item.description;\n if (fetchFullArticles && item.link) {\n const fullContent = await fetchArticleContent(item.link);\n if (fullContent && fullContent.length > articleContent.length) {\n articleContent = fullContent;\n }\n }\n return `Title: ${item.title}\nAuthor: ${item.author}\nPublished: ${item.pubDate}\nCategories: ${item.categories.join(', ')}\nLink: ${item.link}\n${fetchFullArticles ? 'Full Article Content:' : 'Content:'}\n${articleContent}\n\nSummary: ${item.title} - ${item.description}`;\n },\n };\n }\n },\n };\n}\n"],
|
|
5
|
-
"mappings": "AAAA,SAAS,uCAAuC;AAChD,OAAO,YAAY;AACnB,OAAO,YAAY;AAEnB,MAAM,WAAW,gCAAgC;AAAA;AAAA,EAE/C,WAAW,OAAO;AACpB,CAAC;AAmBD,MAAM,YAAY,IAAI,OAA+B;AAAA,EACnD,cAAc;AAAA,IACZ,MAAM,CAAC,aAAa,YAAY,kBAAkB,WAAW;AAAA,IAC7D,MAAM,CAAC,YAAY,WAAW,aAAa,MAAM;AAAA,EACnD;AACF,CAAC;AAED,eAAe,aAAa,SAAiB;AAC3C,MAAI;AACF,YAAQ,IAAI,sBAAsB,OAAO,EAAE;AAC3C,UAAM,OAAO,MAAM,UAAU,SAAS,OAAO;AAE7C,WAAO;AAAA,MACL,OAAO,KAAK,SAAS;AAAA,MACrB,aAAa,KAAK,eAAe;AAAA,MACjC,MAAM,KAAK,QAAQ;AAAA,MACnB,UAAU,KAAK,YAAY;AAAA,MAC3B,eAAe,KAAK,kBAAiB,oBAAI,KAAK,GAAE,YAAY;AAAA,MAC5D,OAAO,KAAK,MAAM,IAAI,CAAC,UAAU;AAAA,QAC/B,OAAO,KAAK,SAAS;AAAA,QACrB,aAAa,KAAK,WAAW,KAAK,WAAW,KAAK,kBAAkB;AAAA,QACpE,MAAM,KAAK,QAAQ;AAAA,QACnB,SAAS,KAAK,WAAW,KAAK,WAAW;AAAA,QACzC,QAAQ,KAAK,WAAW,KAAK,UAAU;AAAA,QACvC,YAAY,MAAM,QAAQ,KAAK,UAAU,IACrC,KAAK,aACL,KAAK,WACH,CAAC,KAAK,QAAQ,IACd,CAAC;AAAA,QACP,MAAM,KAAK,QAAQ,KAAK,QAAQ;AAAA,QAChC,gBAAgB,KAAK,iBAAiB,KAAK,KAAK,WAAW;AAAA,MAC7D,EAAE;AAAA,IACJ;AAAA,EACF,SAAS,OAAO;AACd,YAAQ,MAAM,4BAA4B,OAAO,KAAK,KAAK;AAC3D,UAAM,IAAI;AAAA,MACR,uBAAuB,iBAAiB,QAAQ,MAAM,UAAU,eAAe;AAAA,IACjF;AAAA,EACF;AACF;AAEA,eAAe,oBAAoB,KAA8B;AAC/D,MAAI;AACF,YAAQ,IAAI,uCAAuC,GAAG,EAAE;AAExD,UAAM,UAAU,MAAM,SAAS;AAAA,MAC7B,EAAE,IAAI;AAAA,MACN,EAAE,YAAY,KAAK,UAAU,CAAC,GAAG,SAAS,CAAC,EAAE;AAAA,IAC/C;AACA,WAAO,GAAG,UAAU,SAAS,yCAAyC;AACtE,QAAI,QAAQ,SAAS,SAAS;AAC5B,YAAM,IAAI,MAAM,QAAQ,OAAO;AAAA,IACjC;AAGA,UAAM,cAAc,QAAQ,QACxB,GAAG,QAAQ,KAAK;AAAA;AAAA,EAAO,QAAQ,OAAO,KACtC,QAAQ;AAGZ,QAAI,YAAY,SAAS,KAAK;AAC5B,YAAM,IAAI;AAAA,QACR,gCAAgC,YAAY,MAAM;AAAA,MACpD;AAAA,IACF;AAEA,YAAQ;AAAA,MACN,2CAAsC,QAAQ,SAAS,EAAE,MAAM,YAAY,MAAM,YAAY,KAAK,MAAM,YAAY,MAAM,GAAG,EAAE,MAAM,CAAC;AAAA,IACxI;AAEA,WAAO;AAAA,EACT,SAAS,OAAO;AACd,YAAQ;AAAA,MACN,wCAAwC,GAAG;AAAA,MAC3C,iBAAiB,QAAQ,MAAM,UAAU;AAAA,IAC3C;AACA,WAAO;AAAA,EACT;AACF;AACO,SAAS,IACd,SACA,UAGI,CAAC,GACL;AACA,QAAM,WAAW,OAAO,OAAO;AAC/B,QAAM,EAAE,WAAW,IAAI,oBAAoB,MAAM,IAAI;AAErD,SAAO;AAAA,IACL;AAAA,IACA,cAAc,sEAAsE,OAAO;AAAA;AAAA;AAAA;AAAA,QAIvF,oBAAoB,gGAAgG,kDAAkD;AAAA,IAC1K,SAAS,mBAAmB;AAC1B,YAAM,OAAO,MAAM,aAAa,OAAO;AAEvC,YAAM,cAAc,aAAa,KAAK,KAAK;AAAA,eAClC,KAAK,WAAW;AAAA,WACpB,KAAK,IAAI;AAAA,YACR,KAAK,QAAQ;AAAA,gBACT,KAAK,aAAa;AAAA,eACnB,KAAK,MAAM,MAAM;AAAA;AAAA,sBAEV,KAAK,WAAW;AAChC,YAAM;AAAA,QACJ,IAAI;AAAA,QACJ,SAAS,YAAY;AAAA,MACvB;AAGA,YAAM,iBAAiB,KAAK,MAAM,MAAM,GAAG,QAAQ;AACnD,iBAAW,QAAQ,gBAAgB;AACjC,cAAM,aACJ,KAAK,QAAQ,KAAK,QAAQ,GAAG,KAAK,KAAK,IAAI,KAAK,OAAO;AACzD,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,SAAS,YAAY;AAEnB,gBAAI,iBAAiB,KAAK,kBAAkB,KAAK;AACjD,gBAAI,qBAAqB,KAAK,MAAM;AAClC,oBAAM,cAAc,MAAM,oBAAoB,KAAK,IAAI;AACvD,kBAAI,eAAe,YAAY,SAAS,eAAe,QAAQ;AAC7D,iCAAiB;AAAA,cACnB;AAAA,YACF;AACA,mBAAO,UAAU,KAAK,KAAK;AAAA,UAC7B,KAAK,MAAM;AAAA,aACR,KAAK,OAAO;AAAA,cACX,KAAK,WAAW,KAAK,IAAI,CAAC;AAAA,QAChC,KAAK,IAAI;AAAA,EACf,oBAAoB,0BAA0B,UAAU;AAAA,EACxD,cAAc;AAAA;AAAA,WAEL,KAAK,KAAK,MAAM,KAAK,WAAW;AAAA,UACjC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;",
|
|
6
|
-
"names": []
|
|
7
|
-
}
|
|
@@ -1,63 +0,0 @@
|
|
|
1
|
-
import { tensorToEmbeddings } from "./huggingface.js";
|
|
2
|
-
const EmbeddingModels = {
|
|
3
|
-
AllMiniLML6V2: "fast-all-MiniLM-L6-v2",
|
|
4
|
-
BGEBaseEN: "fast-bge-base-en",
|
|
5
|
-
BGEBaseENV15: "fast-bge-base-en-v1.5",
|
|
6
|
-
BGESmallEN: "fast-bge-small-en",
|
|
7
|
-
BGESmallENV15: "fast-bge-small-en-v1.5",
|
|
8
|
-
BGESmallZH: "fast-bge-small-zh-v1.5",
|
|
9
|
-
MLE5Large: "fast-multilingual-e5-large"
|
|
10
|
-
};
|
|
11
|
-
const transformerModels = {
|
|
12
|
-
[EmbeddingModels.AllMiniLML6V2]: "Xenova/all-MiniLM-L6-v2",
|
|
13
|
-
[EmbeddingModels.BGEBaseEN]: "Xenova/bge-base-en",
|
|
14
|
-
[EmbeddingModels.BGEBaseENV15]: "Xenova/bge-base-en-v1.5",
|
|
15
|
-
[EmbeddingModels.BGESmallEN]: "Xenova/bge-small-en",
|
|
16
|
-
[EmbeddingModels.BGESmallENV15]: "Xenova/bge-small-en-v1.5",
|
|
17
|
-
[EmbeddingModels.BGESmallZH]: "Xenova/bge-small-zh-v1.5",
|
|
18
|
-
[EmbeddingModels.MLE5Large]: "Xenova/multilingual-e5-large"
|
|
19
|
-
};
|
|
20
|
-
function fastembed(options = {}) {
|
|
21
|
-
const {
|
|
22
|
-
model: modelId = EmbeddingModels.BGESmallENV15,
|
|
23
|
-
batchSize = 256,
|
|
24
|
-
cacheDir
|
|
25
|
-
} = options;
|
|
26
|
-
if (!Number.isInteger(batchSize) || batchSize <= 0) {
|
|
27
|
-
throw new RangeError("batchSize must be a positive integer");
|
|
28
|
-
}
|
|
29
|
-
let modelPromise = null;
|
|
30
|
-
const getModel = () => {
|
|
31
|
-
if (!modelPromise) {
|
|
32
|
-
modelPromise = import("@huggingface/transformers").then(
|
|
33
|
-
({ pipeline }) => pipeline("feature-extraction", transformerModels[modelId], {
|
|
34
|
-
cache_dir: cacheDir
|
|
35
|
-
})
|
|
36
|
-
);
|
|
37
|
-
}
|
|
38
|
-
return modelPromise;
|
|
39
|
-
};
|
|
40
|
-
return async (documents) => {
|
|
41
|
-
if (documents.length === 0) {
|
|
42
|
-
return { embeddings: [], dimensions: 0 };
|
|
43
|
-
}
|
|
44
|
-
const model = await getModel();
|
|
45
|
-
const embeddings = [];
|
|
46
|
-
let dimensions = 0;
|
|
47
|
-
for (let start = 0; start < documents.length; start += batchSize) {
|
|
48
|
-
const tensor = await model(documents.slice(start, start + batchSize), {
|
|
49
|
-
pooling: "mean",
|
|
50
|
-
normalize: true
|
|
51
|
-
});
|
|
52
|
-
const batch = tensorToEmbeddings(tensor);
|
|
53
|
-
dimensions ||= batch.dimensions;
|
|
54
|
-
embeddings.push(...batch.embeddings);
|
|
55
|
-
}
|
|
56
|
-
return { embeddings, dimensions };
|
|
57
|
-
};
|
|
58
|
-
}
|
|
59
|
-
export {
|
|
60
|
-
EmbeddingModels,
|
|
61
|
-
fastembed
|
|
62
|
-
};
|
|
63
|
-
//# sourceMappingURL=fastembed.js.map
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": 3,
|
|
3
|
-
"sources": ["../../../src/lib/embedders/fastembed.ts"],
|
|
4
|
-
"sourcesContent": ["import type { FeatureExtractionPipeline } from '@huggingface/transformers';\n\nimport type { Embedder } from '../stores/store.js';\nimport { tensorToEmbeddings } from './huggingface.js';\n\n/**\n * Stable model identifiers retained from the former `fastembed` adapter.\n */\nexport const EmbeddingModels = {\n AllMiniLML6V2: 'fast-all-MiniLM-L6-v2',\n BGEBaseEN: 'fast-bge-base-en',\n BGEBaseENV15: 'fast-bge-base-en-v1.5',\n BGESmallEN: 'fast-bge-small-en',\n BGESmallENV15: 'fast-bge-small-en-v1.5',\n BGESmallZH: 'fast-bge-small-zh-v1.5',\n MLE5Large: 'fast-multilingual-e5-large',\n} as const;\nexport type EmbeddingModel =\n (typeof EmbeddingModels)[keyof typeof EmbeddingModels];\n\nconst transformerModels: Record<EmbeddingModel, string> = {\n [EmbeddingModels.AllMiniLML6V2]: 'Xenova/all-MiniLM-L6-v2',\n [EmbeddingModels.BGEBaseEN]: 'Xenova/bge-base-en',\n [EmbeddingModels.BGEBaseENV15]: 'Xenova/bge-base-en-v1.5',\n [EmbeddingModels.BGESmallEN]: 'Xenova/bge-small-en',\n [EmbeddingModels.BGESmallENV15]: 'Xenova/bge-small-en-v1.5',\n [EmbeddingModels.BGESmallZH]: 'Xenova/bge-small-zh-v1.5',\n [EmbeddingModels.MLE5Large]: 'Xenova/multilingual-e5-large',\n};\n\nexport interface FastEmbedOptions {\n model?: EmbeddingModel;\n batchSize?: number;\n cacheDir?: string;\n}\n\n/**\n * Local feature extraction backed by Transformers.js. The historical function\n * and model identifiers remain stable while avoiding FastEmbed's archived tar\n * extraction dependency.\n */\nexport function fastembed(options: FastEmbedOptions = {}): Embedder {\n const {\n model: modelId = EmbeddingModels.BGESmallENV15,\n batchSize = 256,\n cacheDir,\n } = options;\n\n if (!Number.isInteger(batchSize) || batchSize <= 0) {\n throw new RangeError('batchSize must be a positive integer');\n }\n\n let modelPromise: Promise<FeatureExtractionPipeline> | null = null;\n const getModel = () => {\n if (!modelPromise) {\n modelPromise = import('@huggingface/transformers').then(({ pipeline }) =>\n pipeline('feature-extraction', transformerModels[modelId], {\n cache_dir: cacheDir,\n }),\n );\n }\n return modelPromise;\n };\n\n return async (documents: string[]) => {\n if (documents.length === 0) {\n return { embeddings: [], dimensions: 0 };\n }\n\n const model = await getModel();\n const embeddings: Float32Array[] = [];\n let dimensions = 0;\n\n for (let start = 0; start < documents.length; start += batchSize) {\n const tensor = await model(documents.slice(start, start + batchSize), {\n pooling: 'mean',\n normalize: true,\n });\n const batch = tensorToEmbeddings(tensor);\n dimensions ||= batch.dimensions;\n embeddings.push(...batch.embeddings);\n }\n\n return { embeddings, dimensions };\n };\n}\n"],
|
|
5
|
-
"mappings": "AAGA,SAAS,0BAA0B;AAK5B,MAAM,kBAAkB;AAAA,EAC7B,eAAe;AAAA,EACf,WAAW;AAAA,EACX,cAAc;AAAA,EACd,YAAY;AAAA,EACZ,eAAe;AAAA,EACf,YAAY;AAAA,EACZ,WAAW;AACb;AAIA,MAAM,oBAAoD;AAAA,EACxD,CAAC,gBAAgB,aAAa,GAAG;AAAA,EACjC,CAAC,gBAAgB,SAAS,GAAG;AAAA,EAC7B,CAAC,gBAAgB,YAAY,GAAG;AAAA,EAChC,CAAC,gBAAgB,UAAU,GAAG;AAAA,EAC9B,CAAC,gBAAgB,aAAa,GAAG;AAAA,EACjC,CAAC,gBAAgB,UAAU,GAAG;AAAA,EAC9B,CAAC,gBAAgB,SAAS,GAAG;AAC/B;AAaO,SAAS,UAAU,UAA4B,CAAC,GAAa;AAClE,QAAM;AAAA,IACJ,OAAO,UAAU,gBAAgB;AAAA,IACjC,YAAY;AAAA,IACZ;AAAA,EACF,IAAI;AAEJ,MAAI,CAAC,OAAO,UAAU,SAAS,KAAK,aAAa,GAAG;AAClD,UAAM,IAAI,WAAW,sCAAsC;AAAA,EAC7D;AAEA,MAAI,eAA0D;AAC9D,QAAM,WAAW,MAAM;AACrB,QAAI,CAAC,cAAc;AACjB,qBAAe,OAAO,2BAA2B,EAAE;AAAA,QAAK,CAAC,EAAE,SAAS,MAClE,SAAS,sBAAsB,kBAAkB,OAAO,GAAG;AAAA,UACzD,WAAW;AAAA,QACb,CAAC;AAAA,MACH;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAEA,SAAO,OAAO,cAAwB;AACpC,QAAI,UAAU,WAAW,GAAG;AAC1B,aAAO,EAAE,YAAY,CAAC,GAAG,YAAY,EAAE;AAAA,IACzC;AAEA,UAAM,QAAQ,MAAM,SAAS;AAC7B,UAAM,aAA6B,CAAC;AACpC,QAAI,aAAa;AAEjB,aAAS,QAAQ,GAAG,QAAQ,UAAU,QAAQ,SAAS,WAAW;AAChE,YAAM,SAAS,MAAM,MAAM,UAAU,MAAM,OAAO,QAAQ,SAAS,GAAG;AAAA,QACpE,SAAS;AAAA,QACT,WAAW;AAAA,MACb,CAAC;AACD,YAAM,QAAQ,mBAAmB,MAAM;AACvC,qBAAe,MAAM;AACrB,iBAAW,KAAK,GAAG,MAAM,UAAU;AAAA,IACrC;AAEA,WAAO,EAAE,YAAY,WAAW;AAAA,EAClC;AACF;",
|
|
6
|
-
"names": []
|
|
7
|
-
}
|
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
async function textEmbeddings(inputs, { extractorFn, pooling, normalize }) {
|
|
2
|
-
inputs = (Array.isArray(inputs) ? inputs : [inputs]).map((it) => it.trim());
|
|
3
|
-
const extractor = await extractorFn();
|
|
4
|
-
const tensor = await extractor(inputs, {
|
|
5
|
-
pooling: pooling ?? "mean",
|
|
6
|
-
normalize: normalize ?? true
|
|
7
|
-
});
|
|
8
|
-
return tensorToEmbeddings(tensor);
|
|
9
|
-
}
|
|
10
|
-
function tensorToEmbeddings(tensor) {
|
|
11
|
-
const dims = tensor.dims;
|
|
12
|
-
if (!Array.isArray(dims) || dims.length < 2) {
|
|
13
|
-
throw new Error(`Unexpected tensor dims: ${JSON.stringify(dims)}`);
|
|
14
|
-
}
|
|
15
|
-
const batchSize = dims[0];
|
|
16
|
-
const hiddenSize = dims[dims.length - 1];
|
|
17
|
-
const expectedLen = batchSize * hiddenSize;
|
|
18
|
-
if (tensor.data.length !== expectedLen) {
|
|
19
|
-
throw new Error(
|
|
20
|
-
`Data length mismatch: got ${tensor.data.length}, expected ${expectedLen} (batch=${batchSize}, hidden=${hiddenSize})`
|
|
21
|
-
);
|
|
22
|
-
}
|
|
23
|
-
const data = tensor.data;
|
|
24
|
-
if (!(data instanceof Float32Array)) {
|
|
25
|
-
throw new Error(`Expected float32 embeddings, got ${tensor.type}`);
|
|
26
|
-
}
|
|
27
|
-
const embeddings = [];
|
|
28
|
-
const dimensions = tensor.dims[tensor.dims.length - 1];
|
|
29
|
-
for (let i = 0; i < batchSize; i++) {
|
|
30
|
-
const start = i * hiddenSize;
|
|
31
|
-
embeddings.push(data.subarray(start, start + hiddenSize));
|
|
32
|
-
}
|
|
33
|
-
return { embeddings, dimensions };
|
|
34
|
-
}
|
|
35
|
-
function huggingface(extractorFn) {
|
|
36
|
-
return (documents) => textEmbeddings(documents, { extractorFn });
|
|
37
|
-
}
|
|
38
|
-
export {
|
|
39
|
-
huggingface,
|
|
40
|
-
tensorToEmbeddings
|
|
41
|
-
};
|
|
42
|
-
//# sourceMappingURL=huggingface.js.map
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": 3,
|
|
3
|
-
"sources": ["../../../src/lib/embedders/huggingface.ts"],
|
|
4
|
-
"sourcesContent": ["import type {\n FeatureExtractionPipeline,\n Tensor,\n} from '@huggingface/transformers';\n\nimport type { Embedder } from '../stores/store.js';\n\nexport type FeatureExtractionFn = FeatureExtractionPipeline;\n\nexport interface EmbeddingOptions {\n extractorFn: () =>\n Promise<FeatureExtractionPipeline> | FeatureExtractionPipeline;\n pooling?: 'mean' | 'cls';\n normalize?: boolean;\n}\n\nasync function textEmbeddings(\n inputs: string | string[],\n { extractorFn, pooling, normalize }: EmbeddingOptions,\n) {\n inputs = (Array.isArray(inputs) ? inputs : [inputs]).map((it) => it.trim());\n const extractor = await extractorFn();\n const tensor = await extractor(inputs, {\n pooling: pooling ?? 'mean',\n normalize: normalize ?? true,\n });\n return tensorToEmbeddings(tensor);\n}\n\nexport function tensorToEmbeddings(tensor: Tensor) {\n const dims = tensor.dims;\n if (!Array.isArray(dims) || dims.length < 2) {\n throw new Error(`Unexpected tensor dims: ${JSON.stringify(dims)}`);\n }\n const batchSize = dims[0];\n const hiddenSize = dims[dims.length - 1];\n const expectedLen = batchSize * hiddenSize;\n if (tensor.data.length !== expectedLen) {\n throw new Error(\n `Data length mismatch: got ${tensor.data.length}, expected ${expectedLen} (batch=${batchSize}, hidden=${hiddenSize})`,\n );\n }\n const data = tensor.data;\n if (!(data instanceof Float32Array)) {\n throw new Error(`Expected float32 embeddings, got ${tensor.type}`);\n }\n // Reuse the underlying typed array without copying; we'll copy when creating Buffer for SQLite\n const embeddings: Float32Array[] = [];\n const dimensions = tensor.dims[tensor.dims.length - 1];\n for (let i = 0; i < batchSize; i++) {\n const start = i * hiddenSize;\n embeddings.push(data.subarray(start, start + hiddenSize));\n }\n return { embeddings, dimensions };\n}\n\nexport function huggingface(\n extractorFn: () =>\n Promise<FeatureExtractionPipeline> | FeatureExtractionPipeline,\n): Embedder {\n return (documents) => textEmbeddings(documents, { extractorFn });\n}\n"],
|
|
5
|
-
"mappings": "AAgBA,eAAe,eACb,QACA,EAAE,aAAa,SAAS,UAAU,GAClC;AACA,YAAU,MAAM,QAAQ,MAAM,IAAI,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;AAC1E,QAAM,YAAY,MAAM,YAAY;AACpC,QAAM,SAAS,MAAM,UAAU,QAAQ;AAAA,IACrC,SAAS,WAAW;AAAA,IACpB,WAAW,aAAa;AAAA,EAC1B,CAAC;AACD,SAAO,mBAAmB,MAAM;AAClC;AAEO,SAAS,mBAAmB,QAAgB;AACjD,QAAM,OAAO,OAAO;AACpB,MAAI,CAAC,MAAM,QAAQ,IAAI,KAAK,KAAK,SAAS,GAAG;AAC3C,UAAM,IAAI,MAAM,2BAA2B,KAAK,UAAU,IAAI,CAAC,EAAE;AAAA,EACnE;AACA,QAAM,YAAY,KAAK,CAAC;AACxB,QAAM,aAAa,KAAK,KAAK,SAAS,CAAC;AACvC,QAAM,cAAc,YAAY;AAChC,MAAI,OAAO,KAAK,WAAW,aAAa;AACtC,UAAM,IAAI;AAAA,MACR,6BAA6B,OAAO,KAAK,MAAM,cAAc,WAAW,WAAW,SAAS,YAAY,UAAU;AAAA,IACpH;AAAA,EACF;AACA,QAAM,OAAO,OAAO;AACpB,MAAI,EAAE,gBAAgB,eAAe;AACnC,UAAM,IAAI,MAAM,oCAAoC,OAAO,IAAI,EAAE;AAAA,EACnE;AAEA,QAAM,aAA6B,CAAC;AACpC,QAAM,aAAa,OAAO,KAAK,OAAO,KAAK,SAAS,CAAC;AACrD,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,IAAI;AAClB,eAAW,KAAK,KAAK,SAAS,OAAO,QAAQ,UAAU,CAAC;AAAA,EAC1D;AACA,SAAO,EAAE,YAAY,WAAW;AAClC;AAEO,SAAS,YACd,aAEU;AACV,SAAO,CAAC,cAAc,eAAe,WAAW,EAAE,YAAY,CAAC;AACjE;",
|
|
6
|
-
"names": []
|
|
7
|
-
}
|