@deepagents/retrieval 7.2.0 → 7.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/dist/index.d.ts +7 -7
  2. package/dist/index.js +561 -7
  3. package/dist/index.js.map +4 -4
  4. package/dist/lib/connectors/index.d.ts +7 -7
  5. package/dist/lib/connectors/index.js +649 -7
  6. package/dist/lib/connectors/index.js.map +4 -4
  7. package/dist/lib/connectors/local.d.ts +1 -1
  8. package/dist/lib/connectors/local.d.ts.map +1 -1
  9. package/dist/lib/connectors/pdf.d.ts +1 -1
  10. package/dist/lib/connectors/pdf.d.ts.map +1 -1
  11. package/dist/lib/connectors/repo.d.ts +1 -1
  12. package/dist/lib/embedders/fastembed.d.ts +1 -1
  13. package/dist/lib/embedders/huggingface.d.ts +1 -1
  14. package/dist/lib/ingest.d.ts +3 -3
  15. package/dist/lib/similiarty-search.d.ts +1 -1
  16. package/dist/lib/stores/sqlite/ddl.sqlite.d.ts +1 -1
  17. package/dist/lib/stores/sqlite/ddl.sqlite.d.ts.map +1 -1
  18. package/dist/lib/stores/sqlite/node-sqlite.d.ts +1 -1
  19. package/dist/lib/stores/sqlite/open.d.ts +1 -2
  20. package/dist/lib/stores/sqlite/open.d.ts.map +1 -1
  21. package/dist/lib/stores/sqlite/sqlite.d.ts +1 -1
  22. package/package.json +4 -3
  23. package/dist/lib/connectors/connector.js +0 -1
  24. package/dist/lib/connectors/connector.js.map +0 -7
  25. package/dist/lib/connectors/github.js +0 -164
  26. package/dist/lib/connectors/github.js.map +0 -7
  27. package/dist/lib/connectors/linear.js +0 -29
  28. package/dist/lib/connectors/linear.js.map +0 -7
  29. package/dist/lib/connectors/local.js +0 -98
  30. package/dist/lib/connectors/local.js.map +0 -7
  31. package/dist/lib/connectors/pdf.js +0 -58
  32. package/dist/lib/connectors/pdf.js.map +0 -7
  33. package/dist/lib/connectors/repo.js +0 -179
  34. package/dist/lib/connectors/repo.js.map +0 -7
  35. package/dist/lib/connectors/rss.js +0 -128
  36. package/dist/lib/connectors/rss.js.map +0 -7
  37. package/dist/lib/embedders/fastembed.js +0 -63
  38. package/dist/lib/embedders/fastembed.js.map +0 -7
  39. package/dist/lib/embedders/huggingface.js +0 -42
  40. package/dist/lib/embedders/huggingface.js.map +0 -7
  41. package/dist/lib/ingest.js +0 -117
  42. package/dist/lib/ingest.js.map +0 -7
  43. package/dist/lib/pipeline.js +0 -1
  44. package/dist/lib/pipeline.js.map +0 -7
  45. package/dist/lib/sidecar.js +0 -1
  46. package/dist/lib/sidecar.js.map +0 -7
  47. package/dist/lib/similiarty-search.js +0 -42
  48. package/dist/lib/similiarty-search.js.map +0 -7
  49. package/dist/lib/stores/cid.js +0 -8
  50. package/dist/lib/stores/cid.js.map +0 -7
  51. package/dist/lib/stores/sqlite/ddl.sqlite.js +0 -53
  52. package/dist/lib/stores/sqlite/ddl.sqlite.js.map +0 -7
  53. package/dist/lib/stores/sqlite/node-sqlite.js +0 -9
  54. package/dist/lib/stores/sqlite/node-sqlite.js.map +0 -7
  55. package/dist/lib/stores/sqlite/open.js +0 -11
  56. package/dist/lib/stores/sqlite/open.js.map +0 -7
  57. package/dist/lib/stores/sqlite/sqlite.js +0 -245
  58. package/dist/lib/stores/sqlite/sqlite.js.map +0 -7
  59. package/dist/lib/stores/store.js +0 -1
  60. package/dist/lib/stores/store.js.map +0 -7
@@ -1,8 +1,8 @@
1
- export * from './connector.js';
2
- export * from './github.js';
3
- export * from './linear.js';
4
- export * from './local.js';
5
- export * from './pdf.js';
6
- export * from './rss.js';
7
- export * from './repo.js';
1
+ export * from './connector.ts';
2
+ export * from './github.ts';
3
+ export * from './linear.ts';
4
+ export * from './local.ts';
5
+ export * from './pdf.ts';
6
+ export * from './rss.ts';
7
+ export * from './repo.ts';
8
8
  //# sourceMappingURL=index.d.ts.map
@@ -1,8 +1,650 @@
1
- export * from "./connector.js";
2
- export * from "./github.js";
3
- export * from "./linear.js";
4
- export * from "./local.js";
5
- export * from "./pdf.js";
6
- export * from "./rss.js";
7
- export * from "./repo.js";
1
+ // packages/retrieval/src/lib/connectors/github.ts
2
+ import { execa } from "execa";
3
+ function isRelease(value) {
4
+ return typeof value === "object" && value !== null && "tag_name" in value && typeof value.tag_name === "string" && "name" in value && (value.name === null || typeof value.name === "string") && "body" in value && (value.body === null || typeof value.body === "string") && "updated_at" in value && typeof value.updated_at === "string" && "published_at" in value && typeof value.published_at === "string" && "html_url" in value && typeof value.html_url === "string" && "draft" in value && typeof value.draft === "boolean" && "prerelease" in value && typeof value.prerelease === "boolean";
5
+ }
6
+ function fs(path) {
7
+ const [owner, repo2, ...filePath] = path.split("/");
8
+ return {
9
+ readFile: async () => {
10
+ const url = `https://api.github.com/repos/${owner}/${repo2}/contents/${filePath.join("/")}`;
11
+ const res = await fetch(url);
12
+ const data = await res.json();
13
+ if (typeof data !== "object" || data === null || !("content" in data) || typeof data.content !== "string") {
14
+ throw new Error(`GitHub returned no file content for ${path}`);
15
+ }
16
+ return atob(data.content);
17
+ },
18
+ release: async (repo3, opts = {}) => {
19
+ const [owner2, repoName] = repo3.split("/");
20
+ const perPage = 100;
21
+ const maxPages = 10;
22
+ const releases = [];
23
+ let stop = false;
24
+ for (let page = 1; page <= maxPages && !stop; page++) {
25
+ const url = `https://api.github.com/repos/${owner2}/${repoName}/releases?per_page=${perPage}&page=${page}`;
26
+ const res = await fetch(url, {
27
+ headers: { Accept: "application/vnd.github+json" }
28
+ });
29
+ if (!res.ok) break;
30
+ const pageData = await res.json();
31
+ if (!Array.isArray(pageData) || !pageData.every(isRelease)) {
32
+ throw new Error(`Unexpected GitHub releases response for ${repo3}`);
33
+ }
34
+ if (!pageData.length) break;
35
+ for (const rel of pageData) {
36
+ if (!opts.includeDrafts && rel.draft) continue;
37
+ if (!opts.includePrerelease && rel.prerelease) continue;
38
+ releases.push(rel);
39
+ if (opts.untilTag && rel.tag_name === opts.untilTag) {
40
+ if (opts.inclusive === false) {
41
+ releases.pop();
42
+ }
43
+ stop = true;
44
+ break;
45
+ }
46
+ }
47
+ }
48
+ return releases;
49
+ }
50
+ };
51
+ }
52
+ var github = {
53
+ file(filePath) {
54
+ const sourceId = `github:file:${filePath}`;
55
+ const documentId = filePath;
56
+ return {
57
+ sourceId,
58
+ sources: async function* () {
59
+ const { readFile: readFile4 } = fs(filePath);
60
+ yield {
61
+ id: documentId,
62
+ content: () => readFile4()
63
+ };
64
+ }
65
+ };
66
+ },
67
+ /**
68
+ * Create a connector over GitHub Releases with optional pagination & early-stop.
69
+ * @param repo owner/repo string
70
+ * @param options Release filtering & pagination controls
71
+ */
72
+ release: (repo2, options = {}) => {
73
+ const sourceId = `github:releases:${repo2}`;
74
+ return {
75
+ sourceId,
76
+ sources: async function* () {
77
+ const releases = await fs(repo2).release(repo2, options);
78
+ for (const rel of releases) {
79
+ yield {
80
+ id: `${repo2}:release:${rel.tag_name}`,
81
+ content: async () => {
82
+ const name = rel.name || rel.tag_name;
83
+ const body = rel.body || "";
84
+ return `Release: ${name}
85
+ Tag: ${rel.tag_name}
86
+ Published at: ${rel.published_at}
87
+ Updated at: ${rel.updated_at}
88
+ URL: ${rel.html_url}
89
+ Draft: ${rel.draft}
90
+ Prerelease: ${rel.prerelease}
91
+
92
+ ${body}`;
93
+ }
94
+ };
95
+ }
96
+ }
97
+ };
98
+ },
99
+ /**
100
+ * Create a connector over a GitHub repository file tree using gitingest to produce a markdown digest.
101
+ * @param repoUrl full https://github.com/OWNER/REPO or .../tree/BRANCH/subdir URL
102
+ */
103
+ repo: (repoUrl, opts) => {
104
+ const sourceId = `github:repo:${repoUrl}`;
105
+ const {
106
+ excludes = [
107
+ "**/node_modules/**",
108
+ "**/dist/**",
109
+ "**/coverage/**",
110
+ "**/*.test.ts",
111
+ "**/*.test.tsx",
112
+ "**/dist/**",
113
+ "**/.git/**",
114
+ "**/.github/**",
115
+ "**/.vscode/**",
116
+ "**/.idea/**",
117
+ "**/build/**",
118
+ "**/out/**",
119
+ "**/vendor/**",
120
+ "**/__tests__/**",
121
+ "**/*.d.ts"
122
+ ],
123
+ branch,
124
+ includeGitignored = false,
125
+ includeSubmodules = false,
126
+ githubToken,
127
+ ingestWhen,
128
+ includes
129
+ } = opts;
130
+ async function gitingestDigest() {
131
+ const args = [
132
+ "gitingest",
133
+ ...branch ? ["-b", branch] : [],
134
+ ...includes.flatMap((p) => ["-i", p]),
135
+ ...excludes.flatMap((p) => ["-e", p]),
136
+ ...includeGitignored ? ["--include-gitignored"] : [],
137
+ ...includeSubmodules ? ["--include-submodules"] : [],
138
+ repoUrl,
139
+ "-o",
140
+ "-"
141
+ ];
142
+ const { stdout } = await execa("uvx", args, {
143
+ env: githubToken ? { ...process.env, GITHUB_TOKEN: githubToken } : process.env,
144
+ stdout: "pipe",
145
+ stderr: "inherit"
146
+ });
147
+ return stdout;
148
+ }
149
+ return {
150
+ sourceId,
151
+ ingestWhen,
152
+ sources: async function* () {
153
+ const content = await gitingestDigest();
154
+ yield {
155
+ id: "digest.md",
156
+ content: async () => content
157
+ };
158
+ }
159
+ };
160
+ }
161
+ };
162
+
163
+ // packages/retrieval/src/lib/connectors/linear.ts
164
+ import { LinearClient } from "@linear/sdk";
165
+ function linear(apiKey) {
166
+ const linearClient = new LinearClient({ apiKey });
167
+ const sourceId = "linear:workspace";
168
+ return {
169
+ sourceId,
170
+ instructions: "You answer questions about Linear issues assigned to the user.",
171
+ sources: async () => {
172
+ const user = await linearClient.viewer;
173
+ const issues = await user.assignedIssues();
174
+ return Promise.all(
175
+ issues.nodes.map(async (it) => {
176
+ const state = await it.state;
177
+ return {
178
+ id: it.id,
179
+ content: async () => `Issue: ${it.title}
180
+ Description: ${it.description || "No description"}
181
+ Status: ${state?.name || "Unknown"}
182
+ ID: ${it.id}`
183
+ };
184
+ })
185
+ );
186
+ }
187
+ };
188
+ }
189
+
190
+ // packages/retrieval/src/lib/connectors/local.ts
191
+ import fg from "fast-glob";
192
+ import ignore from "ignore";
193
+ import { readFile, stat } from "node:fs/promises";
194
+ import { dirname, join, relative, resolve } from "node:path";
195
+ var gitignoreCache = /* @__PURE__ */ new Map();
196
+ async function collectGitignorePatterns(targetPath) {
197
+ const absolutePath = resolve(targetPath);
198
+ if (gitignoreCache.has(absolutePath)) {
199
+ return gitignoreCache.get(absolutePath);
200
+ }
201
+ const ig = ignore();
202
+ const pathSegments = absolutePath.split("/");
203
+ const gitignoreFiles = [];
204
+ for (let i = 1; i <= pathSegments.length; i++) {
205
+ const currentPath = pathSegments.slice(0, i).join("/");
206
+ const gitignorePath = join(currentPath, ".gitignore");
207
+ try {
208
+ await stat(gitignorePath);
209
+ gitignoreFiles.push(gitignorePath);
210
+ } catch {
211
+ }
212
+ }
213
+ for (const gitignoreFile of gitignoreFiles) {
214
+ try {
215
+ const content = await readFile(gitignoreFile, "utf8");
216
+ const patterns = content.split("\n").map((line) => line.trim()).filter((line) => line && !line.startsWith("#"));
217
+ ig.add(patterns);
218
+ } catch {
219
+ }
220
+ }
221
+ gitignoreCache.set(absolutePath, ig);
222
+ return ig;
223
+ }
224
+ async function getFilteredFiles(cwd, pattern) {
225
+ const allFiles = await fg(pattern, {
226
+ dot: false,
227
+ absolute: true,
228
+ onlyFiles: true,
229
+ cwd,
230
+ followSymbolicLinks: false,
231
+ ignore: [
232
+ "**/node_modules/**",
233
+ "**/.git/**",
234
+ "**/.DS_Store",
235
+ "**/Thumbs.db",
236
+ "**/*.tmp",
237
+ "**/*.temp",
238
+ "**/coverage/**",
239
+ "**/dist/**",
240
+ "**/build/**"
241
+ ]
242
+ });
243
+ if (allFiles.length === 0) {
244
+ return [];
245
+ }
246
+ const filesByDir = /* @__PURE__ */ new Map();
247
+ for (const file of allFiles) {
248
+ const dir = dirname(file);
249
+ if (!filesByDir.has(dir)) {
250
+ filesByDir.set(dir, []);
251
+ }
252
+ filesByDir.get(dir).push(file);
253
+ }
254
+ const filteredFiles = [];
255
+ for (const [dir, files] of filesByDir) {
256
+ const ignoreFilter = await collectGitignorePatterns(dir);
257
+ for (const file of files) {
258
+ if (!ignoreFilter.ignores(relative(cwd, file))) {
259
+ filteredFiles.push(file);
260
+ }
261
+ }
262
+ }
263
+ return filteredFiles.sort();
264
+ }
265
+ function local(pattern, options) {
266
+ const sourceId = `glob:${pattern}`;
267
+ return {
268
+ sourceId,
269
+ ingestWhen: options?.ingestWhen,
270
+ expiresAfter: options?.expiresAfter,
271
+ sources: async function* () {
272
+ const paths = await getFilteredFiles(
273
+ options?.cwd ?? process.cwd(),
274
+ pattern
275
+ );
276
+ for (const filePath of paths) {
277
+ yield {
278
+ id: filePath,
279
+ content: () => readFile(filePath, "utf8").catch(() => "")
280
+ };
281
+ }
282
+ }
283
+ };
284
+ }
285
+
286
+ // packages/retrieval/src/lib/connectors/pdf.ts
287
+ import { glob, readFile as readFile2 } from "node:fs/promises";
288
+ import { extractText, getDocumentProxy } from "unpdf";
289
+ function pdf(pattern) {
290
+ const sourceId = `pdf:${pattern}`;
291
+ return {
292
+ sourceId,
293
+ sources: async function* () {
294
+ const paths = await Array.fromAsync(
295
+ glob(pattern, { exclude: ["**/node_modules/**", "**/.git/**"] })
296
+ );
297
+ for (const path of paths) {
298
+ if (!path.toLowerCase().endsWith(".pdf")) continue;
299
+ yield {
300
+ id: path,
301
+ content: async () => {
302
+ const buffer = await readFile2(path);
303
+ const pdf2 = await getDocumentProxy(new Uint8Array(buffer));
304
+ const { text } = await extractText(pdf2, { mergePages: true });
305
+ return text;
306
+ }
307
+ };
308
+ }
309
+ }
310
+ };
311
+ }
312
+ function pdfFile(source) {
313
+ const isUrl = /^https?:\/\//.test(source);
314
+ const sourceId = `pdf:${isUrl ? "url" : "file"}:${source}`;
315
+ return {
316
+ sourceId,
317
+ sources: async function* () {
318
+ yield {
319
+ id: source,
320
+ content: async () => {
321
+ let buffer;
322
+ if (isUrl) {
323
+ const response = await fetch(source);
324
+ if (!response.ok) {
325
+ throw new Error(
326
+ `HTTP ${response.status}: ${response.statusText}`
327
+ );
328
+ }
329
+ buffer = new Uint8Array(await response.arrayBuffer());
330
+ } else {
331
+ const fileBuffer = await readFile2(source);
332
+ buffer = new Uint8Array(fileBuffer.buffer);
333
+ }
334
+ const pdf2 = await getDocumentProxy(new Uint8Array(buffer));
335
+ const { text } = await extractText(pdf2, { mergePages: true });
336
+ return text;
337
+ }
338
+ };
339
+ }
340
+ };
341
+ }
342
+
343
+ // packages/retrieval/src/lib/connectors/rss.ts
344
+ import { experimental_createWebFetchTool } from "@ai-sdk/tool-web-fetch";
345
+ import assert from "node:assert/strict";
346
+ import Parser from "rss-parser";
347
+ var webFetch = experimental_createWebFetchTool({
348
+ // Ingestion stores the complete article in one result.
349
+ maxLength: Number.MAX_SAFE_INTEGER
350
+ });
351
+ var rssParser = new Parser({
352
+ customFields: {
353
+ feed: ["copyright", "language", "managingEditor", "webMaster"],
354
+ item: ["category", "creator", "enclosure", "guid"]
355
+ }
356
+ });
357
+ async function parseRSSFeed(feedUrl) {
358
+ try {
359
+ console.log(`Fetching RSS feed: ${feedUrl}`);
360
+ const feed = await rssParser.parseURL(feedUrl);
361
+ return {
362
+ title: feed.title || "",
363
+ description: feed.description || "",
364
+ link: feed.link || "",
365
+ language: feed.language || "en",
366
+ lastBuildDate: feed.lastBuildDate || (/* @__PURE__ */ new Date()).toISOString(),
367
+ items: feed.items.map((item) => ({
368
+ title: item.title || "",
369
+ description: item.content || item.summary || item.contentSnippet || "",
370
+ link: item.link || "",
371
+ pubDate: item.pubDate || item.isoDate || "",
372
+ author: item.creator || item.author || "",
373
+ categories: Array.isArray(item.categories) ? item.categories : item.category ? [item.category] : [],
374
+ guid: item.guid || item.guid || "",
375
+ contentEncoded: item["content:encoded"] || item.content || ""
376
+ }))
377
+ };
378
+ } catch (error) {
379
+ console.error(`Failed to parse RSS feed ${feedUrl}:`, error);
380
+ throw new Error(
381
+ `RSS parsing failed: ${error instanceof Error ? error.message : "Unknown error"}`
382
+ );
383
+ }
384
+ }
385
+ async function fetchArticleContent(url) {
386
+ try {
387
+ console.log(`Fetching full article content from: ${url}`);
388
+ const article = await webFetch.execute(
389
+ { url },
390
+ { toolCallId: url, messages: [], context: {} }
391
+ );
392
+ assert.ok("type" in article, "Web fetch must return a complete result");
393
+ if (article.type === "error") {
394
+ throw new Error(article.message);
395
+ }
396
+ const fullContent = article.title ? `${article.title}
397
+
398
+ ${article.content}` : article.content;
399
+ if (fullContent.length < 200) {
400
+ throw new Error(
401
+ `Extracted content too short (${fullContent.length} chars) - likely failed to find main content`
402
+ );
403
+ }
404
+ console.log(
405
+ `\u2713 Successfully extracted article: "${article.title ?? ""}" (${fullContent.length} chars, ~${Math.round(fullContent.split(" ").length)} words)`
406
+ );
407
+ return fullContent;
408
+ } catch (error) {
409
+ console.warn(
410
+ `Failed to fetch article content from ${url}:`,
411
+ error instanceof Error ? error.message : error
412
+ );
413
+ return "";
414
+ }
415
+ }
416
+ function rss(feedUrl, options = {}) {
417
+ const sourceId = `rss:${feedUrl}`;
418
+ const { maxItems = 50, fetchFullArticles = false } = options;
419
+ return {
420
+ sourceId,
421
+ instructions: `You answer questions about articles and content from the RSS feed: ${feedUrl}.
422
+ Always cite the article title and link when referencing specific content.
423
+ The feed contains recent articles, blog posts, and news items.
424
+ When referencing content, include the publication date and author when available.
425
+ ${fetchFullArticles ? "Full article content has been extracted from the original links for comprehensive analysis." : "Content includes RSS summaries and descriptions."}`,
426
+ sources: async function* () {
427
+ const feed = await parseRSSFeed(feedUrl);
428
+ const feedSummary = `RSS Feed: ${feed.title}
429
+ Description: ${feed.description}
430
+ Website: ${feed.link}
431
+ Language: ${feed.language}
432
+ Last Updated: ${feed.lastBuildDate}
433
+ Total Items: ${feed.items.length}
434
+
435
+ This feed provides: ${feed.description}`;
436
+ yield {
437
+ id: "feed-info",
438
+ content: async () => feedSummary
439
+ };
440
+ const itemsToProcess = feed.items.slice(0, maxItems);
441
+ for (const item of itemsToProcess) {
442
+ const documentId = item.guid || item.link || `${item.title}-${item.pubDate}`;
443
+ yield {
444
+ id: documentId,
445
+ content: async () => {
446
+ let articleContent = item.contentEncoded || item.description;
447
+ if (fetchFullArticles && item.link) {
448
+ const fullContent = await fetchArticleContent(item.link);
449
+ if (fullContent && fullContent.length > articleContent.length) {
450
+ articleContent = fullContent;
451
+ }
452
+ }
453
+ return `Title: ${item.title}
454
+ Author: ${item.author}
455
+ Published: ${item.pubDate}
456
+ Categories: ${item.categories.join(", ")}
457
+ Link: ${item.link}
458
+ ${fetchFullArticles ? "Full Article Content:" : "Content:"}
459
+ ${articleContent}
460
+
461
+ Summary: ${item.title} - ${item.description}`;
462
+ }
463
+ };
464
+ }
465
+ }
466
+ };
467
+ }
468
+
469
+ // packages/retrieval/src/lib/connectors/repo.ts
470
+ import fg2 from "fast-glob";
471
+ import { opendir, readFile as readFile3, stat as stat2 } from "node:fs/promises";
472
+ import { join as join2 } from "node:path";
473
+ function repo(dir, extensions, ingestWhen) {
474
+ const sourceId = `repo:${dir}`;
475
+ return {
476
+ sourceId,
477
+ ingestWhen,
478
+ sources: async function* () {
479
+ const paths = await collectFiles(dir, extensions);
480
+ for await (const path of paths) {
481
+ const maxSize = 3 * 1024;
482
+ const st = await stat2(path);
483
+ if (st.size > maxSize) {
484
+ continue;
485
+ }
486
+ yield {
487
+ id: path,
488
+ metadata: { repo: dir },
489
+ content: () => readFile3(path, "utf8").catch(() => "")
490
+ };
491
+ }
492
+ }
493
+ };
494
+ }
495
+ async function* findAllGitRepos(root) {
496
+ const stack = [root];
497
+ const skip = [
498
+ "node_modules",
499
+ "Library",
500
+ "Applications",
501
+ "Pictures",
502
+ "Movies",
503
+ "Music",
504
+ "Downloads",
505
+ ".cache",
506
+ ".npm",
507
+ ".pnpm",
508
+ "development"
509
+ ];
510
+ while (stack.length) {
511
+ const folder = stack.pop();
512
+ const isGitRepo = await stat2(join2(folder, ".git")).then((st) => st.isDirectory() || st.isFile()).catch(() => false);
513
+ if (isGitRepo) {
514
+ yield folder;
515
+ continue;
516
+ }
517
+ const dir = await opendir(folder);
518
+ for await (const dirent of dir) {
519
+ if (dirent.isDirectory() && !dirent.isSymbolicLink() && !skip.includes(dirent.name) && !dirent.name.startsWith(".")) {
520
+ stack.push(join2(folder, dirent.name));
521
+ }
522
+ }
523
+ }
524
+ }
525
+ async function gitignore(gitignoreFile) {
526
+ const content = await readFile3(gitignoreFile, "utf8").catch(() => "");
527
+ if (!content) {
528
+ return [];
529
+ }
530
+ const patterns = content.split("\n").map((line) => line.trim()).filter((line) => line && !line.startsWith("#"));
531
+ return patterns;
532
+ }
533
+ async function ignorePatterns(repo2) {
534
+ return [
535
+ // Inherit repo-specific ignore patterns
536
+ ...await gitignore(join2(repo2, ".gitignore")),
537
+ // Package managers & dependency dirs
538
+ "node_modules/**",
539
+ "**/node_modules/**",
540
+ "**/.pnpm/**",
541
+ "**/.npm/**",
542
+ "**/.yarn/**",
543
+ "**/vendor/**",
544
+ // PHP / Go modules (when vendored)
545
+ "**/3rdparty/**",
546
+ // Version control + VCS metadata
547
+ "**/.git/**",
548
+ "**/.svn/**",
549
+ "**/.hg/**",
550
+ // OS / system junk
551
+ "**/.DS_Store",
552
+ "**/Thumbs.db",
553
+ "**/Library/**",
554
+ "**/Applications/**",
555
+ "**/Pictures/**",
556
+ "**/Movies/**",
557
+ "**/Music/**",
558
+ "**/Downloads/**",
559
+ "**/.cache/**",
560
+ // Environment / secrets (explicit)
561
+ "**/.env",
562
+ "**/.env.*",
563
+ // Lockfiles & generated dependency state
564
+ "**/*.lock",
565
+ "**/yarn.lock",
566
+ "**/package-lock.json",
567
+ "**/pnpm-lock.yaml",
568
+ // Build / compilation outputs
569
+ "**/dist/**",
570
+ "**/debug/**",
571
+ "**/build/**",
572
+ "**/out/**",
573
+ "**/target/**",
574
+ // Rust / JVM
575
+ "**/bin/**",
576
+ "**/obj/**",
577
+ "**/classes/**",
578
+ // Framework / tool specific build artifacts
579
+ "**/.next/**",
580
+ "**/.vercel/**",
581
+ "**/.turbo/**",
582
+ "**/.docusaurus/**",
583
+ "**/.vite/**",
584
+ "**/.parcel-cache/**",
585
+ "**/.rollup.cache/**",
586
+ "**/.vuepress/**",
587
+ "cdk.out/**",
588
+ // Infra & deployment tooling
589
+ "**/.serverless/**",
590
+ "**/.terraform/**",
591
+ "**/.terragrunt-cache/**",
592
+ "**/.pulumi/**",
593
+ // Coverage & testing caches
594
+ "**/coverage/**",
595
+ "**/.nyc_output/**",
596
+ "**/jest-cache/**",
597
+ "**/.pytest_cache/**",
598
+ // Language / tooling caches
599
+ "**/__pycache__/**",
600
+ "**/.mypy_cache/**",
601
+ "**/.tox/**",
602
+ "**/.gradle/**",
603
+ "**/.mvn/**",
604
+ "**/.eslintcache",
605
+ "**/.stylelintcache",
606
+ // IDE / editor configs + history (we don't want to embed these)
607
+ "**/.idea/**",
608
+ "**/.vscode/**",
609
+ "**/.fleet/**",
610
+ "**/.history/**",
611
+ // Virtual environments
612
+ "**/.venv/**",
613
+ "**/venv/**"
614
+ ];
615
+ }
616
+ async function collectFiles(repo2, extensions) {
617
+ const exts = extensions.map((ext) => ext.replace(/^\./, ""));
618
+ const entries = fg2.stream(
619
+ extensions.length > 1 ? `**/*.{${exts.join(",")}}` : `**/*.${exts[0]}`,
620
+ {
621
+ dot: false,
622
+ onlyFiles: true,
623
+ unique: true,
624
+ absolute: true,
625
+ cwd: repo2,
626
+ ignore: await ignorePatterns(repo2)
627
+ }
628
+ );
629
+ return (async function* () {
630
+ for await (const entry of entries) {
631
+ if (typeof entry !== "string") {
632
+ throw new TypeError("fast-glob yielded a non-path entry");
633
+ }
634
+ yield entry;
635
+ }
636
+ })();
637
+ }
638
+ export {
639
+ collectFiles,
640
+ findAllGitRepos,
641
+ github,
642
+ ignorePatterns,
643
+ linear,
644
+ local,
645
+ pdf,
646
+ pdfFile,
647
+ repo,
648
+ rss
649
+ };
8
650
  //# sourceMappingURL=index.js.map