@deepagents/retrieval 7.2.0 → 7.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +7 -7
- package/dist/index.js +561 -7
- package/dist/index.js.map +4 -4
- package/dist/lib/connectors/index.d.ts +7 -7
- package/dist/lib/connectors/index.js +649 -7
- package/dist/lib/connectors/index.js.map +4 -4
- package/dist/lib/connectors/local.d.ts +1 -1
- package/dist/lib/connectors/local.d.ts.map +1 -1
- package/dist/lib/connectors/pdf.d.ts +1 -1
- package/dist/lib/connectors/pdf.d.ts.map +1 -1
- package/dist/lib/connectors/repo.d.ts +1 -1
- package/dist/lib/embedders/fastembed.d.ts +1 -1
- package/dist/lib/embedders/huggingface.d.ts +1 -1
- package/dist/lib/ingest.d.ts +3 -3
- package/dist/lib/similiarty-search.d.ts +1 -1
- package/dist/lib/stores/sqlite/ddl.sqlite.d.ts +1 -1
- package/dist/lib/stores/sqlite/ddl.sqlite.d.ts.map +1 -1
- package/dist/lib/stores/sqlite/node-sqlite.d.ts +1 -1
- package/dist/lib/stores/sqlite/open.d.ts +1 -2
- package/dist/lib/stores/sqlite/open.d.ts.map +1 -1
- package/dist/lib/stores/sqlite/sqlite.d.ts +1 -1
- package/package.json +4 -3
- package/dist/lib/connectors/connector.js +0 -1
- package/dist/lib/connectors/connector.js.map +0 -7
- package/dist/lib/connectors/github.js +0 -164
- package/dist/lib/connectors/github.js.map +0 -7
- package/dist/lib/connectors/linear.js +0 -29
- package/dist/lib/connectors/linear.js.map +0 -7
- package/dist/lib/connectors/local.js +0 -98
- package/dist/lib/connectors/local.js.map +0 -7
- package/dist/lib/connectors/pdf.js +0 -58
- package/dist/lib/connectors/pdf.js.map +0 -7
- package/dist/lib/connectors/repo.js +0 -179
- package/dist/lib/connectors/repo.js.map +0 -7
- package/dist/lib/connectors/rss.js +0 -128
- package/dist/lib/connectors/rss.js.map +0 -7
- package/dist/lib/embedders/fastembed.js +0 -63
- package/dist/lib/embedders/fastembed.js.map +0 -7
- package/dist/lib/embedders/huggingface.js +0 -42
- package/dist/lib/embedders/huggingface.js.map +0 -7
- package/dist/lib/ingest.js +0 -117
- package/dist/lib/ingest.js.map +0 -7
- package/dist/lib/pipeline.js +0 -1
- package/dist/lib/pipeline.js.map +0 -7
- package/dist/lib/sidecar.js +0 -1
- package/dist/lib/sidecar.js.map +0 -7
- package/dist/lib/similiarty-search.js +0 -42
- package/dist/lib/similiarty-search.js.map +0 -7
- package/dist/lib/stores/cid.js +0 -8
- package/dist/lib/stores/cid.js.map +0 -7
- package/dist/lib/stores/sqlite/ddl.sqlite.js +0 -53
- package/dist/lib/stores/sqlite/ddl.sqlite.js.map +0 -7
- package/dist/lib/stores/sqlite/node-sqlite.js +0 -9
- package/dist/lib/stores/sqlite/node-sqlite.js.map +0 -7
- package/dist/lib/stores/sqlite/open.js +0 -11
- package/dist/lib/stores/sqlite/open.js.map +0 -7
- package/dist/lib/stores/sqlite/sqlite.js +0 -245
- package/dist/lib/stores/sqlite/sqlite.js.map +0 -7
- package/dist/lib/stores/store.js +0 -1
- package/dist/lib/stores/store.js.map +0 -7
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
export * from './connector.
|
|
2
|
-
export * from './github.
|
|
3
|
-
export * from './linear.
|
|
4
|
-
export * from './local.
|
|
5
|
-
export * from './pdf.
|
|
6
|
-
export * from './rss.
|
|
7
|
-
export * from './repo.
|
|
1
|
+
export * from './connector.ts';
|
|
2
|
+
export * from './github.ts';
|
|
3
|
+
export * from './linear.ts';
|
|
4
|
+
export * from './local.ts';
|
|
5
|
+
export * from './pdf.ts';
|
|
6
|
+
export * from './rss.ts';
|
|
7
|
+
export * from './repo.ts';
|
|
8
8
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1,8 +1,650 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
1
|
+
// packages/retrieval/src/lib/connectors/github.ts
|
|
2
|
+
import { execa } from "execa";
|
|
3
|
+
function isRelease(value) {
|
|
4
|
+
return typeof value === "object" && value !== null && "tag_name" in value && typeof value.tag_name === "string" && "name" in value && (value.name === null || typeof value.name === "string") && "body" in value && (value.body === null || typeof value.body === "string") && "updated_at" in value && typeof value.updated_at === "string" && "published_at" in value && typeof value.published_at === "string" && "html_url" in value && typeof value.html_url === "string" && "draft" in value && typeof value.draft === "boolean" && "prerelease" in value && typeof value.prerelease === "boolean";
|
|
5
|
+
}
|
|
6
|
+
function fs(path) {
|
|
7
|
+
const [owner, repo2, ...filePath] = path.split("/");
|
|
8
|
+
return {
|
|
9
|
+
readFile: async () => {
|
|
10
|
+
const url = `https://api.github.com/repos/${owner}/${repo2}/contents/${filePath.join("/")}`;
|
|
11
|
+
const res = await fetch(url);
|
|
12
|
+
const data = await res.json();
|
|
13
|
+
if (typeof data !== "object" || data === null || !("content" in data) || typeof data.content !== "string") {
|
|
14
|
+
throw new Error(`GitHub returned no file content for ${path}`);
|
|
15
|
+
}
|
|
16
|
+
return atob(data.content);
|
|
17
|
+
},
|
|
18
|
+
release: async (repo3, opts = {}) => {
|
|
19
|
+
const [owner2, repoName] = repo3.split("/");
|
|
20
|
+
const perPage = 100;
|
|
21
|
+
const maxPages = 10;
|
|
22
|
+
const releases = [];
|
|
23
|
+
let stop = false;
|
|
24
|
+
for (let page = 1; page <= maxPages && !stop; page++) {
|
|
25
|
+
const url = `https://api.github.com/repos/${owner2}/${repoName}/releases?per_page=${perPage}&page=${page}`;
|
|
26
|
+
const res = await fetch(url, {
|
|
27
|
+
headers: { Accept: "application/vnd.github+json" }
|
|
28
|
+
});
|
|
29
|
+
if (!res.ok) break;
|
|
30
|
+
const pageData = await res.json();
|
|
31
|
+
if (!Array.isArray(pageData) || !pageData.every(isRelease)) {
|
|
32
|
+
throw new Error(`Unexpected GitHub releases response for ${repo3}`);
|
|
33
|
+
}
|
|
34
|
+
if (!pageData.length) break;
|
|
35
|
+
for (const rel of pageData) {
|
|
36
|
+
if (!opts.includeDrafts && rel.draft) continue;
|
|
37
|
+
if (!opts.includePrerelease && rel.prerelease) continue;
|
|
38
|
+
releases.push(rel);
|
|
39
|
+
if (opts.untilTag && rel.tag_name === opts.untilTag) {
|
|
40
|
+
if (opts.inclusive === false) {
|
|
41
|
+
releases.pop();
|
|
42
|
+
}
|
|
43
|
+
stop = true;
|
|
44
|
+
break;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
return releases;
|
|
49
|
+
}
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
var github = {
|
|
53
|
+
file(filePath) {
|
|
54
|
+
const sourceId = `github:file:${filePath}`;
|
|
55
|
+
const documentId = filePath;
|
|
56
|
+
return {
|
|
57
|
+
sourceId,
|
|
58
|
+
sources: async function* () {
|
|
59
|
+
const { readFile: readFile4 } = fs(filePath);
|
|
60
|
+
yield {
|
|
61
|
+
id: documentId,
|
|
62
|
+
content: () => readFile4()
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
},
|
|
67
|
+
/**
|
|
68
|
+
* Create a connector over GitHub Releases with optional pagination & early-stop.
|
|
69
|
+
* @param repo owner/repo string
|
|
70
|
+
* @param options Release filtering & pagination controls
|
|
71
|
+
*/
|
|
72
|
+
release: (repo2, options = {}) => {
|
|
73
|
+
const sourceId = `github:releases:${repo2}`;
|
|
74
|
+
return {
|
|
75
|
+
sourceId,
|
|
76
|
+
sources: async function* () {
|
|
77
|
+
const releases = await fs(repo2).release(repo2, options);
|
|
78
|
+
for (const rel of releases) {
|
|
79
|
+
yield {
|
|
80
|
+
id: `${repo2}:release:${rel.tag_name}`,
|
|
81
|
+
content: async () => {
|
|
82
|
+
const name = rel.name || rel.tag_name;
|
|
83
|
+
const body = rel.body || "";
|
|
84
|
+
return `Release: ${name}
|
|
85
|
+
Tag: ${rel.tag_name}
|
|
86
|
+
Published at: ${rel.published_at}
|
|
87
|
+
Updated at: ${rel.updated_at}
|
|
88
|
+
URL: ${rel.html_url}
|
|
89
|
+
Draft: ${rel.draft}
|
|
90
|
+
Prerelease: ${rel.prerelease}
|
|
91
|
+
|
|
92
|
+
${body}`;
|
|
93
|
+
}
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
};
|
|
98
|
+
},
|
|
99
|
+
/**
|
|
100
|
+
* Create a connector over a GitHub repository file tree using gitingest to produce a markdown digest.
|
|
101
|
+
* @param repoUrl full https://github.com/OWNER/REPO or .../tree/BRANCH/subdir URL
|
|
102
|
+
*/
|
|
103
|
+
repo: (repoUrl, opts) => {
|
|
104
|
+
const sourceId = `github:repo:${repoUrl}`;
|
|
105
|
+
const {
|
|
106
|
+
excludes = [
|
|
107
|
+
"**/node_modules/**",
|
|
108
|
+
"**/dist/**",
|
|
109
|
+
"**/coverage/**",
|
|
110
|
+
"**/*.test.ts",
|
|
111
|
+
"**/*.test.tsx",
|
|
112
|
+
"**/dist/**",
|
|
113
|
+
"**/.git/**",
|
|
114
|
+
"**/.github/**",
|
|
115
|
+
"**/.vscode/**",
|
|
116
|
+
"**/.idea/**",
|
|
117
|
+
"**/build/**",
|
|
118
|
+
"**/out/**",
|
|
119
|
+
"**/vendor/**",
|
|
120
|
+
"**/__tests__/**",
|
|
121
|
+
"**/*.d.ts"
|
|
122
|
+
],
|
|
123
|
+
branch,
|
|
124
|
+
includeGitignored = false,
|
|
125
|
+
includeSubmodules = false,
|
|
126
|
+
githubToken,
|
|
127
|
+
ingestWhen,
|
|
128
|
+
includes
|
|
129
|
+
} = opts;
|
|
130
|
+
async function gitingestDigest() {
|
|
131
|
+
const args = [
|
|
132
|
+
"gitingest",
|
|
133
|
+
...branch ? ["-b", branch] : [],
|
|
134
|
+
...includes.flatMap((p) => ["-i", p]),
|
|
135
|
+
...excludes.flatMap((p) => ["-e", p]),
|
|
136
|
+
...includeGitignored ? ["--include-gitignored"] : [],
|
|
137
|
+
...includeSubmodules ? ["--include-submodules"] : [],
|
|
138
|
+
repoUrl,
|
|
139
|
+
"-o",
|
|
140
|
+
"-"
|
|
141
|
+
];
|
|
142
|
+
const { stdout } = await execa("uvx", args, {
|
|
143
|
+
env: githubToken ? { ...process.env, GITHUB_TOKEN: githubToken } : process.env,
|
|
144
|
+
stdout: "pipe",
|
|
145
|
+
stderr: "inherit"
|
|
146
|
+
});
|
|
147
|
+
return stdout;
|
|
148
|
+
}
|
|
149
|
+
return {
|
|
150
|
+
sourceId,
|
|
151
|
+
ingestWhen,
|
|
152
|
+
sources: async function* () {
|
|
153
|
+
const content = await gitingestDigest();
|
|
154
|
+
yield {
|
|
155
|
+
id: "digest.md",
|
|
156
|
+
content: async () => content
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
// packages/retrieval/src/lib/connectors/linear.ts
|
|
164
|
+
import { LinearClient } from "@linear/sdk";
|
|
165
|
+
function linear(apiKey) {
|
|
166
|
+
const linearClient = new LinearClient({ apiKey });
|
|
167
|
+
const sourceId = "linear:workspace";
|
|
168
|
+
return {
|
|
169
|
+
sourceId,
|
|
170
|
+
instructions: "You answer questions about Linear issues assigned to the user.",
|
|
171
|
+
sources: async () => {
|
|
172
|
+
const user = await linearClient.viewer;
|
|
173
|
+
const issues = await user.assignedIssues();
|
|
174
|
+
return Promise.all(
|
|
175
|
+
issues.nodes.map(async (it) => {
|
|
176
|
+
const state = await it.state;
|
|
177
|
+
return {
|
|
178
|
+
id: it.id,
|
|
179
|
+
content: async () => `Issue: ${it.title}
|
|
180
|
+
Description: ${it.description || "No description"}
|
|
181
|
+
Status: ${state?.name || "Unknown"}
|
|
182
|
+
ID: ${it.id}`
|
|
183
|
+
};
|
|
184
|
+
})
|
|
185
|
+
);
|
|
186
|
+
}
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
// packages/retrieval/src/lib/connectors/local.ts
|
|
191
|
+
import fg from "fast-glob";
|
|
192
|
+
import ignore from "ignore";
|
|
193
|
+
import { readFile, stat } from "node:fs/promises";
|
|
194
|
+
import { dirname, join, relative, resolve } from "node:path";
|
|
195
|
+
var gitignoreCache = /* @__PURE__ */ new Map();
|
|
196
|
+
async function collectGitignorePatterns(targetPath) {
|
|
197
|
+
const absolutePath = resolve(targetPath);
|
|
198
|
+
if (gitignoreCache.has(absolutePath)) {
|
|
199
|
+
return gitignoreCache.get(absolutePath);
|
|
200
|
+
}
|
|
201
|
+
const ig = ignore();
|
|
202
|
+
const pathSegments = absolutePath.split("/");
|
|
203
|
+
const gitignoreFiles = [];
|
|
204
|
+
for (let i = 1; i <= pathSegments.length; i++) {
|
|
205
|
+
const currentPath = pathSegments.slice(0, i).join("/");
|
|
206
|
+
const gitignorePath = join(currentPath, ".gitignore");
|
|
207
|
+
try {
|
|
208
|
+
await stat(gitignorePath);
|
|
209
|
+
gitignoreFiles.push(gitignorePath);
|
|
210
|
+
} catch {
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
for (const gitignoreFile of gitignoreFiles) {
|
|
214
|
+
try {
|
|
215
|
+
const content = await readFile(gitignoreFile, "utf8");
|
|
216
|
+
const patterns = content.split("\n").map((line) => line.trim()).filter((line) => line && !line.startsWith("#"));
|
|
217
|
+
ig.add(patterns);
|
|
218
|
+
} catch {
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
gitignoreCache.set(absolutePath, ig);
|
|
222
|
+
return ig;
|
|
223
|
+
}
|
|
224
|
+
async function getFilteredFiles(cwd, pattern) {
|
|
225
|
+
const allFiles = await fg(pattern, {
|
|
226
|
+
dot: false,
|
|
227
|
+
absolute: true,
|
|
228
|
+
onlyFiles: true,
|
|
229
|
+
cwd,
|
|
230
|
+
followSymbolicLinks: false,
|
|
231
|
+
ignore: [
|
|
232
|
+
"**/node_modules/**",
|
|
233
|
+
"**/.git/**",
|
|
234
|
+
"**/.DS_Store",
|
|
235
|
+
"**/Thumbs.db",
|
|
236
|
+
"**/*.tmp",
|
|
237
|
+
"**/*.temp",
|
|
238
|
+
"**/coverage/**",
|
|
239
|
+
"**/dist/**",
|
|
240
|
+
"**/build/**"
|
|
241
|
+
]
|
|
242
|
+
});
|
|
243
|
+
if (allFiles.length === 0) {
|
|
244
|
+
return [];
|
|
245
|
+
}
|
|
246
|
+
const filesByDir = /* @__PURE__ */ new Map();
|
|
247
|
+
for (const file of allFiles) {
|
|
248
|
+
const dir = dirname(file);
|
|
249
|
+
if (!filesByDir.has(dir)) {
|
|
250
|
+
filesByDir.set(dir, []);
|
|
251
|
+
}
|
|
252
|
+
filesByDir.get(dir).push(file);
|
|
253
|
+
}
|
|
254
|
+
const filteredFiles = [];
|
|
255
|
+
for (const [dir, files] of filesByDir) {
|
|
256
|
+
const ignoreFilter = await collectGitignorePatterns(dir);
|
|
257
|
+
for (const file of files) {
|
|
258
|
+
if (!ignoreFilter.ignores(relative(cwd, file))) {
|
|
259
|
+
filteredFiles.push(file);
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
return filteredFiles.sort();
|
|
264
|
+
}
|
|
265
|
+
function local(pattern, options) {
|
|
266
|
+
const sourceId = `glob:${pattern}`;
|
|
267
|
+
return {
|
|
268
|
+
sourceId,
|
|
269
|
+
ingestWhen: options?.ingestWhen,
|
|
270
|
+
expiresAfter: options?.expiresAfter,
|
|
271
|
+
sources: async function* () {
|
|
272
|
+
const paths = await getFilteredFiles(
|
|
273
|
+
options?.cwd ?? process.cwd(),
|
|
274
|
+
pattern
|
|
275
|
+
);
|
|
276
|
+
for (const filePath of paths) {
|
|
277
|
+
yield {
|
|
278
|
+
id: filePath,
|
|
279
|
+
content: () => readFile(filePath, "utf8").catch(() => "")
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
};
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
// packages/retrieval/src/lib/connectors/pdf.ts
|
|
287
|
+
import { glob, readFile as readFile2 } from "node:fs/promises";
|
|
288
|
+
import { extractText, getDocumentProxy } from "unpdf";
|
|
289
|
+
function pdf(pattern) {
|
|
290
|
+
const sourceId = `pdf:${pattern}`;
|
|
291
|
+
return {
|
|
292
|
+
sourceId,
|
|
293
|
+
sources: async function* () {
|
|
294
|
+
const paths = await Array.fromAsync(
|
|
295
|
+
glob(pattern, { exclude: ["**/node_modules/**", "**/.git/**"] })
|
|
296
|
+
);
|
|
297
|
+
for (const path of paths) {
|
|
298
|
+
if (!path.toLowerCase().endsWith(".pdf")) continue;
|
|
299
|
+
yield {
|
|
300
|
+
id: path,
|
|
301
|
+
content: async () => {
|
|
302
|
+
const buffer = await readFile2(path);
|
|
303
|
+
const pdf2 = await getDocumentProxy(new Uint8Array(buffer));
|
|
304
|
+
const { text } = await extractText(pdf2, { mergePages: true });
|
|
305
|
+
return text;
|
|
306
|
+
}
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
};
|
|
311
|
+
}
|
|
312
|
+
function pdfFile(source) {
|
|
313
|
+
const isUrl = /^https?:\/\//.test(source);
|
|
314
|
+
const sourceId = `pdf:${isUrl ? "url" : "file"}:${source}`;
|
|
315
|
+
return {
|
|
316
|
+
sourceId,
|
|
317
|
+
sources: async function* () {
|
|
318
|
+
yield {
|
|
319
|
+
id: source,
|
|
320
|
+
content: async () => {
|
|
321
|
+
let buffer;
|
|
322
|
+
if (isUrl) {
|
|
323
|
+
const response = await fetch(source);
|
|
324
|
+
if (!response.ok) {
|
|
325
|
+
throw new Error(
|
|
326
|
+
`HTTP ${response.status}: ${response.statusText}`
|
|
327
|
+
);
|
|
328
|
+
}
|
|
329
|
+
buffer = new Uint8Array(await response.arrayBuffer());
|
|
330
|
+
} else {
|
|
331
|
+
const fileBuffer = await readFile2(source);
|
|
332
|
+
buffer = new Uint8Array(fileBuffer.buffer);
|
|
333
|
+
}
|
|
334
|
+
const pdf2 = await getDocumentProxy(new Uint8Array(buffer));
|
|
335
|
+
const { text } = await extractText(pdf2, { mergePages: true });
|
|
336
|
+
return text;
|
|
337
|
+
}
|
|
338
|
+
};
|
|
339
|
+
}
|
|
340
|
+
};
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
// packages/retrieval/src/lib/connectors/rss.ts
|
|
344
|
+
import { experimental_createWebFetchTool } from "@ai-sdk/tool-web-fetch";
|
|
345
|
+
import assert from "node:assert/strict";
|
|
346
|
+
import Parser from "rss-parser";
|
|
347
|
+
var webFetch = experimental_createWebFetchTool({
|
|
348
|
+
// Ingestion stores the complete article in one result.
|
|
349
|
+
maxLength: Number.MAX_SAFE_INTEGER
|
|
350
|
+
});
|
|
351
|
+
var rssParser = new Parser({
|
|
352
|
+
customFields: {
|
|
353
|
+
feed: ["copyright", "language", "managingEditor", "webMaster"],
|
|
354
|
+
item: ["category", "creator", "enclosure", "guid"]
|
|
355
|
+
}
|
|
356
|
+
});
|
|
357
|
+
async function parseRSSFeed(feedUrl) {
|
|
358
|
+
try {
|
|
359
|
+
console.log(`Fetching RSS feed: ${feedUrl}`);
|
|
360
|
+
const feed = await rssParser.parseURL(feedUrl);
|
|
361
|
+
return {
|
|
362
|
+
title: feed.title || "",
|
|
363
|
+
description: feed.description || "",
|
|
364
|
+
link: feed.link || "",
|
|
365
|
+
language: feed.language || "en",
|
|
366
|
+
lastBuildDate: feed.lastBuildDate || (/* @__PURE__ */ new Date()).toISOString(),
|
|
367
|
+
items: feed.items.map((item) => ({
|
|
368
|
+
title: item.title || "",
|
|
369
|
+
description: item.content || item.summary || item.contentSnippet || "",
|
|
370
|
+
link: item.link || "",
|
|
371
|
+
pubDate: item.pubDate || item.isoDate || "",
|
|
372
|
+
author: item.creator || item.author || "",
|
|
373
|
+
categories: Array.isArray(item.categories) ? item.categories : item.category ? [item.category] : [],
|
|
374
|
+
guid: item.guid || item.guid || "",
|
|
375
|
+
contentEncoded: item["content:encoded"] || item.content || ""
|
|
376
|
+
}))
|
|
377
|
+
};
|
|
378
|
+
} catch (error) {
|
|
379
|
+
console.error(`Failed to parse RSS feed ${feedUrl}:`, error);
|
|
380
|
+
throw new Error(
|
|
381
|
+
`RSS parsing failed: ${error instanceof Error ? error.message : "Unknown error"}`
|
|
382
|
+
);
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
async function fetchArticleContent(url) {
|
|
386
|
+
try {
|
|
387
|
+
console.log(`Fetching full article content from: ${url}`);
|
|
388
|
+
const article = await webFetch.execute(
|
|
389
|
+
{ url },
|
|
390
|
+
{ toolCallId: url, messages: [], context: {} }
|
|
391
|
+
);
|
|
392
|
+
assert.ok("type" in article, "Web fetch must return a complete result");
|
|
393
|
+
if (article.type === "error") {
|
|
394
|
+
throw new Error(article.message);
|
|
395
|
+
}
|
|
396
|
+
const fullContent = article.title ? `${article.title}
|
|
397
|
+
|
|
398
|
+
${article.content}` : article.content;
|
|
399
|
+
if (fullContent.length < 200) {
|
|
400
|
+
throw new Error(
|
|
401
|
+
`Extracted content too short (${fullContent.length} chars) - likely failed to find main content`
|
|
402
|
+
);
|
|
403
|
+
}
|
|
404
|
+
console.log(
|
|
405
|
+
`\u2713 Successfully extracted article: "${article.title ?? ""}" (${fullContent.length} chars, ~${Math.round(fullContent.split(" ").length)} words)`
|
|
406
|
+
);
|
|
407
|
+
return fullContent;
|
|
408
|
+
} catch (error) {
|
|
409
|
+
console.warn(
|
|
410
|
+
`Failed to fetch article content from ${url}:`,
|
|
411
|
+
error instanceof Error ? error.message : error
|
|
412
|
+
);
|
|
413
|
+
return "";
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
function rss(feedUrl, options = {}) {
|
|
417
|
+
const sourceId = `rss:${feedUrl}`;
|
|
418
|
+
const { maxItems = 50, fetchFullArticles = false } = options;
|
|
419
|
+
return {
|
|
420
|
+
sourceId,
|
|
421
|
+
instructions: `You answer questions about articles and content from the RSS feed: ${feedUrl}.
|
|
422
|
+
Always cite the article title and link when referencing specific content.
|
|
423
|
+
The feed contains recent articles, blog posts, and news items.
|
|
424
|
+
When referencing content, include the publication date and author when available.
|
|
425
|
+
${fetchFullArticles ? "Full article content has been extracted from the original links for comprehensive analysis." : "Content includes RSS summaries and descriptions."}`,
|
|
426
|
+
sources: async function* () {
|
|
427
|
+
const feed = await parseRSSFeed(feedUrl);
|
|
428
|
+
const feedSummary = `RSS Feed: ${feed.title}
|
|
429
|
+
Description: ${feed.description}
|
|
430
|
+
Website: ${feed.link}
|
|
431
|
+
Language: ${feed.language}
|
|
432
|
+
Last Updated: ${feed.lastBuildDate}
|
|
433
|
+
Total Items: ${feed.items.length}
|
|
434
|
+
|
|
435
|
+
This feed provides: ${feed.description}`;
|
|
436
|
+
yield {
|
|
437
|
+
id: "feed-info",
|
|
438
|
+
content: async () => feedSummary
|
|
439
|
+
};
|
|
440
|
+
const itemsToProcess = feed.items.slice(0, maxItems);
|
|
441
|
+
for (const item of itemsToProcess) {
|
|
442
|
+
const documentId = item.guid || item.link || `${item.title}-${item.pubDate}`;
|
|
443
|
+
yield {
|
|
444
|
+
id: documentId,
|
|
445
|
+
content: async () => {
|
|
446
|
+
let articleContent = item.contentEncoded || item.description;
|
|
447
|
+
if (fetchFullArticles && item.link) {
|
|
448
|
+
const fullContent = await fetchArticleContent(item.link);
|
|
449
|
+
if (fullContent && fullContent.length > articleContent.length) {
|
|
450
|
+
articleContent = fullContent;
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
return `Title: ${item.title}
|
|
454
|
+
Author: ${item.author}
|
|
455
|
+
Published: ${item.pubDate}
|
|
456
|
+
Categories: ${item.categories.join(", ")}
|
|
457
|
+
Link: ${item.link}
|
|
458
|
+
${fetchFullArticles ? "Full Article Content:" : "Content:"}
|
|
459
|
+
${articleContent}
|
|
460
|
+
|
|
461
|
+
Summary: ${item.title} - ${item.description}`;
|
|
462
|
+
}
|
|
463
|
+
};
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
};
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
// packages/retrieval/src/lib/connectors/repo.ts
|
|
470
|
+
import fg2 from "fast-glob";
|
|
471
|
+
import { opendir, readFile as readFile3, stat as stat2 } from "node:fs/promises";
|
|
472
|
+
import { join as join2 } from "node:path";
|
|
473
|
+
function repo(dir, extensions, ingestWhen) {
|
|
474
|
+
const sourceId = `repo:${dir}`;
|
|
475
|
+
return {
|
|
476
|
+
sourceId,
|
|
477
|
+
ingestWhen,
|
|
478
|
+
sources: async function* () {
|
|
479
|
+
const paths = await collectFiles(dir, extensions);
|
|
480
|
+
for await (const path of paths) {
|
|
481
|
+
const maxSize = 3 * 1024;
|
|
482
|
+
const st = await stat2(path);
|
|
483
|
+
if (st.size > maxSize) {
|
|
484
|
+
continue;
|
|
485
|
+
}
|
|
486
|
+
yield {
|
|
487
|
+
id: path,
|
|
488
|
+
metadata: { repo: dir },
|
|
489
|
+
content: () => readFile3(path, "utf8").catch(() => "")
|
|
490
|
+
};
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
};
|
|
494
|
+
}
|
|
495
|
+
async function* findAllGitRepos(root) {
|
|
496
|
+
const stack = [root];
|
|
497
|
+
const skip = [
|
|
498
|
+
"node_modules",
|
|
499
|
+
"Library",
|
|
500
|
+
"Applications",
|
|
501
|
+
"Pictures",
|
|
502
|
+
"Movies",
|
|
503
|
+
"Music",
|
|
504
|
+
"Downloads",
|
|
505
|
+
".cache",
|
|
506
|
+
".npm",
|
|
507
|
+
".pnpm",
|
|
508
|
+
"development"
|
|
509
|
+
];
|
|
510
|
+
while (stack.length) {
|
|
511
|
+
const folder = stack.pop();
|
|
512
|
+
const isGitRepo = await stat2(join2(folder, ".git")).then((st) => st.isDirectory() || st.isFile()).catch(() => false);
|
|
513
|
+
if (isGitRepo) {
|
|
514
|
+
yield folder;
|
|
515
|
+
continue;
|
|
516
|
+
}
|
|
517
|
+
const dir = await opendir(folder);
|
|
518
|
+
for await (const dirent of dir) {
|
|
519
|
+
if (dirent.isDirectory() && !dirent.isSymbolicLink() && !skip.includes(dirent.name) && !dirent.name.startsWith(".")) {
|
|
520
|
+
stack.push(join2(folder, dirent.name));
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
}
|
|
525
|
+
async function gitignore(gitignoreFile) {
|
|
526
|
+
const content = await readFile3(gitignoreFile, "utf8").catch(() => "");
|
|
527
|
+
if (!content) {
|
|
528
|
+
return [];
|
|
529
|
+
}
|
|
530
|
+
const patterns = content.split("\n").map((line) => line.trim()).filter((line) => line && !line.startsWith("#"));
|
|
531
|
+
return patterns;
|
|
532
|
+
}
|
|
533
|
+
async function ignorePatterns(repo2) {
|
|
534
|
+
return [
|
|
535
|
+
// Inherit repo-specific ignore patterns
|
|
536
|
+
...await gitignore(join2(repo2, ".gitignore")),
|
|
537
|
+
// Package managers & dependency dirs
|
|
538
|
+
"node_modules/**",
|
|
539
|
+
"**/node_modules/**",
|
|
540
|
+
"**/.pnpm/**",
|
|
541
|
+
"**/.npm/**",
|
|
542
|
+
"**/.yarn/**",
|
|
543
|
+
"**/vendor/**",
|
|
544
|
+
// PHP / Go modules (when vendored)
|
|
545
|
+
"**/3rdparty/**",
|
|
546
|
+
// Version control + VCS metadata
|
|
547
|
+
"**/.git/**",
|
|
548
|
+
"**/.svn/**",
|
|
549
|
+
"**/.hg/**",
|
|
550
|
+
// OS / system junk
|
|
551
|
+
"**/.DS_Store",
|
|
552
|
+
"**/Thumbs.db",
|
|
553
|
+
"**/Library/**",
|
|
554
|
+
"**/Applications/**",
|
|
555
|
+
"**/Pictures/**",
|
|
556
|
+
"**/Movies/**",
|
|
557
|
+
"**/Music/**",
|
|
558
|
+
"**/Downloads/**",
|
|
559
|
+
"**/.cache/**",
|
|
560
|
+
// Environment / secrets (explicit)
|
|
561
|
+
"**/.env",
|
|
562
|
+
"**/.env.*",
|
|
563
|
+
// Lockfiles & generated dependency state
|
|
564
|
+
"**/*.lock",
|
|
565
|
+
"**/yarn.lock",
|
|
566
|
+
"**/package-lock.json",
|
|
567
|
+
"**/pnpm-lock.yaml",
|
|
568
|
+
// Build / compilation outputs
|
|
569
|
+
"**/dist/**",
|
|
570
|
+
"**/debug/**",
|
|
571
|
+
"**/build/**",
|
|
572
|
+
"**/out/**",
|
|
573
|
+
"**/target/**",
|
|
574
|
+
// Rust / JVM
|
|
575
|
+
"**/bin/**",
|
|
576
|
+
"**/obj/**",
|
|
577
|
+
"**/classes/**",
|
|
578
|
+
// Framework / tool specific build artifacts
|
|
579
|
+
"**/.next/**",
|
|
580
|
+
"**/.vercel/**",
|
|
581
|
+
"**/.turbo/**",
|
|
582
|
+
"**/.docusaurus/**",
|
|
583
|
+
"**/.vite/**",
|
|
584
|
+
"**/.parcel-cache/**",
|
|
585
|
+
"**/.rollup.cache/**",
|
|
586
|
+
"**/.vuepress/**",
|
|
587
|
+
"cdk.out/**",
|
|
588
|
+
// Infra & deployment tooling
|
|
589
|
+
"**/.serverless/**",
|
|
590
|
+
"**/.terraform/**",
|
|
591
|
+
"**/.terragrunt-cache/**",
|
|
592
|
+
"**/.pulumi/**",
|
|
593
|
+
// Coverage & testing caches
|
|
594
|
+
"**/coverage/**",
|
|
595
|
+
"**/.nyc_output/**",
|
|
596
|
+
"**/jest-cache/**",
|
|
597
|
+
"**/.pytest_cache/**",
|
|
598
|
+
// Language / tooling caches
|
|
599
|
+
"**/__pycache__/**",
|
|
600
|
+
"**/.mypy_cache/**",
|
|
601
|
+
"**/.tox/**",
|
|
602
|
+
"**/.gradle/**",
|
|
603
|
+
"**/.mvn/**",
|
|
604
|
+
"**/.eslintcache",
|
|
605
|
+
"**/.stylelintcache",
|
|
606
|
+
// IDE / editor configs + history (we don't want to embed these)
|
|
607
|
+
"**/.idea/**",
|
|
608
|
+
"**/.vscode/**",
|
|
609
|
+
"**/.fleet/**",
|
|
610
|
+
"**/.history/**",
|
|
611
|
+
// Virtual environments
|
|
612
|
+
"**/.venv/**",
|
|
613
|
+
"**/venv/**"
|
|
614
|
+
];
|
|
615
|
+
}
|
|
616
|
+
async function collectFiles(repo2, extensions) {
|
|
617
|
+
const exts = extensions.map((ext) => ext.replace(/^\./, ""));
|
|
618
|
+
const entries = fg2.stream(
|
|
619
|
+
extensions.length > 1 ? `**/*.{${exts.join(",")}}` : `**/*.${exts[0]}`,
|
|
620
|
+
{
|
|
621
|
+
dot: false,
|
|
622
|
+
onlyFiles: true,
|
|
623
|
+
unique: true,
|
|
624
|
+
absolute: true,
|
|
625
|
+
cwd: repo2,
|
|
626
|
+
ignore: await ignorePatterns(repo2)
|
|
627
|
+
}
|
|
628
|
+
);
|
|
629
|
+
return (async function* () {
|
|
630
|
+
for await (const entry of entries) {
|
|
631
|
+
if (typeof entry !== "string") {
|
|
632
|
+
throw new TypeError("fast-glob yielded a non-path entry");
|
|
633
|
+
}
|
|
634
|
+
yield entry;
|
|
635
|
+
}
|
|
636
|
+
})();
|
|
637
|
+
}
|
|
638
|
+
export {
|
|
639
|
+
collectFiles,
|
|
640
|
+
findAllGitRepos,
|
|
641
|
+
github,
|
|
642
|
+
ignorePatterns,
|
|
643
|
+
linear,
|
|
644
|
+
local,
|
|
645
|
+
pdf,
|
|
646
|
+
pdfFile,
|
|
647
|
+
repo,
|
|
648
|
+
rss
|
|
649
|
+
};
|
|
8
650
|
//# sourceMappingURL=index.js.map
|