@deepagents/retrieval 0.1.0

package/dist/lib/connectors/repo.js

@@ -0,0 +1,171 @@
+import fg from "fast-glob";
+import { opendir, readFile, stat } from "node:fs/promises";
+import { join } from "node:path";
+function repo(dir, extensions, ingestWhen) {
+  const sourceId = `repo:${dir}`;
+  return {
+    sourceId,
+    ingestWhen,
+    sources: async function* () {
+      const paths = await collectFiles(dir, extensions);
+      for await (const path of paths) {
+        const maxSize = 3 * 1024;
+        const st = await stat(path);
+        if (st.size > maxSize) {
+          continue;
+        }
+        yield {
+          id: path,
+          metadata: { repo: dir },
+          content: () => readFile(path, "utf8").catch(() => "")
+        };
+      }
+    }
+  };
+}
+async function* findAllGitRepos(root) {
+  const stack = [root];
+  const skip = [
+    "node_modules",
+    "Library",
+    "Applications",
+    "Pictures",
+    "Movies",
+    "Music",
+    "Downloads",
+    ".cache",
+    ".npm",
+    ".pnpm",
+    "development"
+  ];
+  while (stack.length) {
+    const folder = stack.pop();
+    const isGitRepo = await stat(join(folder, ".git")).then((st) => st.isDirectory() || st.isFile()).catch(() => false);
+    if (isGitRepo) {
+      yield folder;
+      continue;
+    }
+    const dir = await opendir(folder);
+    for await (const dirent of dir) {
+      if (dirent.isDirectory() && !dirent.isSymbolicLink() && !skip.includes(dirent.name) && !dirent.name.startsWith(".")) {
+        stack.push(join(folder, dirent.name));
+      }
+    }
+  }
+}
+function detectRepoTooling(repo2) {
+  const tools = [];
+  return tools;
+}
+async function gitignore(gitignoreFile) {
+  const content = await readFile(gitignoreFile, "utf8").catch(() => "");
+  if (!content) {
+    return [];
+  }
+  const patterns = content.split("\n").map((line) => line.trim()).filter((line) => line && !line.startsWith("#"));
+  return patterns;
+}
+async function ignorePatterns(repo2) {
+  return [
+    // Inherit repo-specific ignore patterns
+    ...await gitignore(join(repo2, ".gitignore")),
+    // Package managers & dependency dirs
+    "node_modules/**",
+    "**/node_modules/**",
+    "**/.pnpm/**",
+    "**/.npm/**",
+    "**/.yarn/**",
+    "**/vendor/**",
+    // PHP / Go modules (when vendored)
+    "**/3rdparty/**",
+    // Version control + VCS metadata
+    "**/.git/**",
+    "**/.svn/**",
+    "**/.hg/**",
+    // OS / system junk
+    "**/.DS_Store",
+    "**/Thumbs.db",
+    "**/Library/**",
+    "**/Applications/**",
+    "**/Pictures/**",
+    "**/Movies/**",
+    "**/Music/**",
+    "**/Downloads/**",
+    "**/.cache/**",
+    // Environment / secrets (explicit)
+    "**/.env",
+    "**/.env.*",
+    // Lockfiles & generated dependency state
+    "**/*.lock",
+    "**/yarn.lock",
+    "**/package-lock.json",
+    "**/pnpm-lock.yaml",
+    // Build / compilation outputs
+    "**/dist/**",
+    "**/debug/**",
+    "**/build/**",
+    "**/out/**",
+    "**/target/**",
+    // Rust / JVM
+    "**/bin/**",
+    "**/obj/**",
+    "**/classes/**",
+    // Framework / tool specific build artifacts
+    "**/.next/**",
+    "**/.vercel/**",
+    "**/.turbo/**",
+    "**/.docusaurus/**",
+    "**/.vite/**",
+    "**/.parcel-cache/**",
+    "**/.rollup.cache/**",
+    "**/.vuepress/**",
+    "cdk.out/**",
+    // Infra & deployment tooling
+    "**/.serverless/**",
+    "**/.terraform/**",
+    "**/.terragrunt-cache/**",
+    "**/.pulumi/**",
+    // Coverage & testing caches
+    "**/coverage/**",
+    "**/.nyc_output/**",
+    "**/jest-cache/**",
+    "**/.pytest_cache/**",
+    // Language / tooling caches
+    "**/__pycache__/**",
+    "**/.mypy_cache/**",
+    "**/.tox/**",
+    "**/.gradle/**",
+    "**/.mvn/**",
+    "**/.eslintcache",
+    "**/.stylelintcache",
+    // IDE / editor configs + history (we don't want to embed these)
+    "**/.idea/**",
+    "**/.vscode/**",
+    "**/.fleet/**",
+    "**/.history/**",
+    // Virtual environments
+    "**/.venv/**",
+    "**/venv/**"
+  ];
+}
+async function collectFiles(repo2, extensions) {
+  const exts = extensions.map((ext) => ext.replace(/^\./, ""));
+  return fg.stream(
+    extensions.length > 1 ? `**/*.{${exts.join(",")}}` : `**/*.${exts[0]}`,
+    {
+      dot: false,
+      onlyFiles: true,
+      unique: true,
+      absolute: true,
+      cwd: repo2,
+      ignore: await ignorePatterns(repo2)
+    }
+  );
+}
+export {
+  collectFiles,
+  findAllGitRepos,
+  ignorePatterns,
+  repo
+};
+//# sourceMappingURL=repo.js.map

package/dist/lib/connectors/repo.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../src/lib/connectors/repo.ts"],
+  "sourcesContent": ["import fg from 'fast-glob';\nimport { opendir, readFile, stat } from 'node:fs/promises';\nimport { join } from 'node:path';\n\nimport type { Connector } from './connector.js';\n\nexport function repo(\n  dir: string,\n  extensions: string[],\n  ingestWhen: Connector['ingestWhen'],\n): Connector {\n  const sourceId = `repo:${dir}`;\n  return {\n    sourceId,\n    ingestWhen,\n    sources: async function* () {\n      const paths = await collectFiles(dir, extensions);\n      for await (const path of paths) {\n        const maxSize = 3 * 1024; // 3KB\n        const st = await stat(path);\n        if (st.size > maxSize) {\n          continue;\n        }\n        yield {\n          id: path,\n          metadata: { repo: dir },\n          content: () => readFile(path, 'utf8').catch(() => ''),\n        };\n      }\n    },\n  };\n}\n\nexport async function* findAllGitRepos(root: string) {\n  const stack = [root];\n  const skip = [\n    'node_modules',\n    'Library',\n    'Applications',\n    'Pictures',\n    'Movies',\n    'Music',\n    'Downloads',\n    '.cache',\n    '.npm',\n    '.pnpm',\n    'development',\n  ];\n  while (stack.length) {\n    const folder = stack.pop()!;\n    const isGitRepo = await stat(join(folder, '.git'))\n      .then((st) => st.isDirectory() || st.isFile())\n      .catch(() => false);\n    if (isGitRepo) {\n      yield folder;\n      continue;\n    }\n    const dir = await opendir(folder);\n    for await (const dirent of dir) {\n      if (\n        dirent.isDirectory() &&\n        !dirent.isSymbolicLink() &&\n        !skip.includes(dirent.name) &&\n        !dirent.name.startsWith('.')\n      ) {\n        stack.push(join(folder, dirent.name));\n      }\n    }\n  }\n}\n\nfunction detectRepoTooling(repo: string) {\n  const tools: string[] = [];\n  return tools;\n}\n\nasync function gitignore(gitignoreFile: string) {\n  const content = await readFile(gitignoreFile, 'utf8').catch(() => '');\n  if (!content) {\n    return [];\n  }\n  const patterns = content\n    .split('\\n')\n    .map((line) => line.trim())\n    .filter((line) => line && !line.startsWith('#'));\n  return patterns;\n}\n\nexport async function ignorePatterns(repo: string) {\n  return [\n    // Inherit repo-specific ignore patterns\n    ...(await gitignore(join(repo, '.gitignore'))),\n\n    // Package managers & dependency dirs\n    'node_modules/**',\n    '**/node_modules/**',\n    '**/.pnpm/**',\n    '**/.npm/**',\n    '**/.yarn/**',\n    '**/vendor/**', // PHP / Go modules (when vendored)\n    '**/3rdparty/**',\n\n    // Version control + VCS metadata\n    '**/.git/**',\n    '**/.svn/**',\n    '**/.hg/**',\n\n    // OS / system junk\n    '**/.DS_Store',\n    '**/Thumbs.db',\n    '**/Library/**',\n    '**/Applications/**',\n    '**/Pictures/**',\n    '**/Movies/**',\n    '**/Music/**',\n    '**/Downloads/**',\n    '**/.cache/**',\n\n    // Environment / secrets (explicit)\n    '**/.env',\n    '**/.env.*',\n\n    // Lockfiles & generated dependency state\n    '**/*.lock',\n    '**/yarn.lock',\n    '**/package-lock.json',\n    '**/pnpm-lock.yaml',\n\n    // Build / compilation outputs\n    '**/dist/**',\n    '**/debug/**',\n    '**/build/**',\n    '**/out/**',\n    '**/target/**', // Rust / JVM\n    '**/bin/**',\n    '**/obj/**',\n    '**/classes/**',\n\n    // Framework / tool specific build artifacts\n    '**/.next/**',\n    '**/.vercel/**',\n    '**/.turbo/**',\n    '**/.docusaurus/**',\n    '**/.vite/**',\n    '**/.parcel-cache/**',\n    '**/.rollup.cache/**',\n    '**/.vuepress/**',\n    'cdk.out/**',\n\n    // Infra & deployment tooling\n    '**/.serverless/**',\n    '**/.terraform/**',\n    '**/.terragrunt-cache/**',\n    '**/.pulumi/**',\n\n    // Coverage & testing caches\n    '**/coverage/**',\n    '**/.nyc_output/**',\n    '**/jest-cache/**',\n    '**/.pytest_cache/**',\n\n    // Language / tooling caches\n    '**/__pycache__/**',\n    '**/.mypy_cache/**',\n    '**/.tox/**',\n    '**/.gradle/**',\n    '**/.mvn/**',\n    '**/.eslintcache',\n    '**/.stylelintcache',\n\n    // IDE / editor configs + history (we don't want to embed these)\n    '**/.idea/**',\n    '**/.vscode/**',\n    '**/.fleet/**',\n    '**/.history/**',\n\n    // Virtual environments\n    '**/.venv/**',\n    '**/venv/**',\n  ];\n}\n\nexport async function collectFiles(\n  repo: string,\n  extensions: string[],\n): Promise<AsyncIterable<string>> {\n  const exts = extensions.map((ext) => ext.replace(/^\\./, ''));\n  return fg.stream(\n    extensions.length > 1 ? `**/*.{${exts.join(',')}}` : `**/*.${exts[0]}`,\n    {\n      dot: false,\n      onlyFiles: true,\n      unique: true,\n      absolute: true,\n      cwd: repo,\n      ignore: await ignorePatterns(repo),\n    },\n  ) as AsyncIterable<string>;\n}\n"],
+  "mappings": "AAAA,OAAO,QAAQ;AACf,SAAS,SAAS,UAAU,YAAY;AACxC,SAAS,YAAY;AAId,SAAS,KACd,KACA,YACA,YACW;AACX,QAAM,WAAW,QAAQ,GAAG;AAC5B,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,SAAS,mBAAmB;AAC1B,YAAM,QAAQ,MAAM,aAAa,KAAK,UAAU;AAChD,uBAAiB,QAAQ,OAAO;AAC9B,cAAM,UAAU,IAAI;AACpB,cAAM,KAAK,MAAM,KAAK,IAAI;AAC1B,YAAI,GAAG,OAAO,SAAS;AACrB;AAAA,QACF;AACA,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,UAAU,EAAE,MAAM,IAAI;AAAA,UACtB,SAAS,MAAM,SAAS,MAAM,MAAM,EAAE,MAAM,MAAM,EAAE;AAAA,QACtD;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAEA,gBAAuB,gBAAgB,MAAc;AACnD,QAAM,QAAQ,CAAC,IAAI;AACnB,QAAM,OAAO;AAAA,IACX;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACA,SAAO,MAAM,QAAQ;AACnB,UAAM,SAAS,MAAM,IAAI;AACzB,UAAM,YAAY,MAAM,KAAK,KAAK,QAAQ,MAAM,CAAC,EAC9C,KAAK,CAAC,OAAO,GAAG,YAAY,KAAK,GAAG,OAAO,CAAC,EAC5C,MAAM,MAAM,KAAK;AACpB,QAAI,WAAW;AACb,YAAM;AACN;AAAA,IACF;AACA,UAAM,MAAM,MAAM,QAAQ,MAAM;AAChC,qBAAiB,UAAU,KAAK;AAC9B,UACE,OAAO,YAAY,KACnB,CAAC,OAAO,eAAe,KACvB,CAAC,KAAK,SAAS,OAAO,IAAI,KAC1B,CAAC,OAAO,KAAK,WAAW,GAAG,GAC3B;AACA,cAAM,KAAK,KAAK,QAAQ,OAAO,IAAI,CAAC;AAAA,MACtC;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,kBAAkBA,OAAc;AACvC,QAAM,QAAkB,CAAC;AACzB,SAAO;AACT;AAEA,eAAe,UAAU,eAAuB;AAC9C,QAAM,UAAU,MAAM,SAAS,eAAe,MAAM,EAAE,MAAM,MAAM,EAAE;AACpE,MAAI,CAAC,SAAS;AACZ,WAAO,CAAC;AAAA,EACV;AACA,QAAM,WAAW,QACd,MAAM,IAAI,EACV,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,OAAO,CAAC,SAAS,QAAQ,CAAC,KAAK,WAAW,GAAG,CAAC;AACjD,SAAO;AACT;AAEA,eAAsB,eAAeA,OAAc;AACjD,SAAO;AAAA;AAAA,IAEL,GAAI,MAAM,UAAU,KAAKA,OAAM,YAAY,CAAC;AAAA;AAAA,IAG5C;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,IAGA;AAAA,IACA;AAAA,EACF;AACF;AAEA,eAAsB,aACpBA,OACA,YACgC;AAChC,QAAM,OAAO,WAAW,IAAI,CAAC,QAAQ,IAAI,QAAQ,OAAO,EAAE,CAAC;AAC3D,SAAO,GAAG;AAAA,IACR,WAAW,SAAS,IAAI,SAAS,KAAK,KAAK,GAAG,CAAC,MAAM,QAAQ,KAAK,CAAC,CAAC;AAAA,IACpE;AAAA,MACE,KAAK;AAAA,MACL,WAAW;AAAA,MACX,QAAQ;AAAA,MACR,UAAU;AAAA,MACV,KAAKA;AAAA,MACL,QAAQ,MAAM,eAAeA,KAAI;AAAA,IACnC;AAAA,EACF;AACF;",
+  "names": ["repo"]
+}

package/dist/lib/connectors/rss.d.ts

@@ -0,0 +1,12 @@
+export declare function rss(feedUrl: string, options?: {
+    maxItems?: number;
+    fetchFullArticles?: boolean;
+}): {
+    sourceId: string;
+    instructions: string;
+    sources: () => AsyncGenerator<{
+        id: any;
+        content: () => Promise<string>;
+    }, void, unknown>;
+};
+//# sourceMappingURL=rss.d.ts.map

package/dist/lib/connectors/rss.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rss.d.ts","sourceRoot":"","sources":["../../../src/lib/connectors/rss.ts"],"names":[],"mappings":"AAsGA,wBAAgB,GAAG,CACjB,OAAO,EAAE,MAAM,EACf,OAAO,GAAE;IACP,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,iBAAiB,CAAC,EAAE,OAAO,CAAC;CACxB;;;;;;;EA0DP"}

package/dist/lib/connectors/rss.js

@@ -0,0 +1,136 @@
+import { Readability } from "@mozilla/readability";
+import Parser from "rss-parser";
+const rssParser = new Parser({
+  customFields: {
+    feed: ["copyright", "language", "managingEditor", "webMaster"],
+    item: ["category", "creator", "enclosure", "guid"]
+  }
+});
+async function parseRSSFeed(feedUrl) {
+  try {
+    console.log(`Fetching RSS feed: ${feedUrl}`);
+    const feed = await rssParser.parseURL(feedUrl);
+    return {
+      title: feed.title || "",
+      description: feed.description || "",
+      link: feed.link || "",
+      language: feed.language || "en",
+      lastBuildDate: feed.lastBuildDate || (/* @__PURE__ */ new Date()).toISOString(),
+      items: feed.items.map((item) => ({
+        title: item.title || "",
+        description: item.content || item.summary || item.contentSnippet || "",
+        link: item.link || "",
+        pubDate: item.pubDate || item.isoDate || "",
+        author: item.creator || item.author || "",
+        categories: Array.isArray(item.categories) ? item.categories : item.category ? [item.category] : [],
+        guid: item.guid || item.guid || "",
+        contentEncoded: item["content:encoded"] || item.content || ""
+      }))
+    };
+  } catch (error) {
+    console.error(`Failed to parse RSS feed ${feedUrl}:`, error);
+    throw new Error(
+      `RSS parsing failed: ${error instanceof Error ? error.message : "Unknown error"}`
+    );
+  }
+}
+async function fetchArticleContent(url) {
+  try {
+    console.log(`Fetching full article content from: ${url}`);
+    const response = await fetch(url, {
+      headers: {
+        "User-Agent": "Mozilla/5.0 (compatible; RSS-RAG-Bot/1.0)",
+        Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+        "Accept-Language": "en-US,en;q=0.5",
+        "Accept-Encoding": "gzip, deflate",
+        Connection: "keep-alive"
+      },
+      signal: AbortSignal.timeout(1e4)
+      // 10 second timeout
+    });
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+    }
+    const html = await response.text();
+    const { JSDOM } = await import("jsdom");
+    const dom = new JSDOM(html, { url });
+    const document = dom.window.document;
+    const reader = new Readability(document);
+    const article = reader.parse();
+    if (!article) {
+      throw new Error("Readability failed to extract article content");
+    }
+    const fullContent = `${article.title ? article.title + "\n\n" : ""}${article.textContent || article.content}`;
+    if (fullContent.length < 200) {
+      throw new Error(
+        `Extracted content too short (${fullContent.length} chars) - likely failed to find main content`
+      );
+    }
+    console.log(
+      `\u2713 Successfully extracted article: "${article.title}" (${fullContent.length} chars, ~${Math.round(fullContent.split(" ").length)} words)`
+    );
+    return fullContent;
+  } catch (error) {
+    console.warn(
+      `Failed to fetch article content from ${url}:`,
+      error instanceof Error ? error.message : error
+    );
+    return "";
+  }
+}
+function rss(feedUrl, options = {}) {
+  const sourceId = `rss:${feedUrl}`;
+  const { maxItems = 50, fetchFullArticles = false } = options;
+  return {
+    sourceId,
+    instructions: `You answer questions about articles and content from the RSS feed: ${feedUrl}.
+      Always cite the article title and link when referencing specific content.
+      The feed contains recent articles, blog posts, and news items.
+      When referencing content, include the publication date and author when available.
+      ${fetchFullArticles ? "Full article content has been extracted from the original links for comprehensive analysis." : "Content includes RSS summaries and descriptions."}`,
+    sources: async function* () {
+      const feed = await parseRSSFeed(feedUrl);
+      const feedSummary = `RSS Feed: ${feed.title}
+Description: ${feed.description}
+Website: ${feed.link}
+Language: ${feed.language}
+Last Updated: ${feed.lastBuildDate}
+Total Items: ${feed.items.length}
+
+This feed provides: ${feed.description}`;
+      yield {
+        id: "feed-info",
+        content: async () => feedSummary
+      };
+      const itemsToProcess = feed.items.slice(0, maxItems);
+      for (const item of itemsToProcess) {
+        const documentId = item.guid || item.link || `${item.title}-${item.pubDate}`;
+        yield {
+          id: documentId,
+          content: async () => {
+            let articleContent = item.contentEncoded || item.description;
+            if (fetchFullArticles && item.link) {
+              const fullContent = await fetchArticleContent(item.link);
+              if (fullContent && fullContent.length > articleContent.length) {
+                articleContent = fullContent;
+              }
+            }
+            return `Title: ${item.title}
+Author: ${item.author}
+Published: ${item.pubDate}
+Categories: ${item.categories.join(", ")}
+Link: ${item.link}
+${fetchFullArticles ? "Full Article Content:" : "Content:"}
+${articleContent}
+
+Summary: ${item.title} - ${item.description}`;
+          }
+        };
+      }
+    }
+  };
+}
+export {
+  rss
+};
+//# sourceMappingURL=rss.js.map

package/dist/lib/connectors/rss.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../src/lib/connectors/rss.ts"],
+  "sourcesContent": ["import { Readability } from '@mozilla/readability';\nimport Parser from 'rss-parser';\n\nconst rssParser = new Parser({\n  customFields: {\n    feed: ['copyright', 'language', 'managingEditor', 'webMaster'],\n    item: ['category', 'creator', 'enclosure', 'guid'],\n  },\n});\n\nasync function parseRSSFeed(feedUrl: string) {\n  try {\n    console.log(`Fetching RSS feed: ${feedUrl}`);\n    const feed = await rssParser.parseURL(feedUrl);\n\n    return {\n      title: feed.title || '',\n      description: feed.description || '',\n      link: feed.link || '',\n      language: feed.language || 'en',\n      lastBuildDate: (feed as any).lastBuildDate || new Date().toISOString(),\n      items: feed.items.map((item) => ({\n        title: item.title || '',\n        description: item.content || item.summary || item.contentSnippet || '',\n        link: item.link || '',\n        pubDate: item.pubDate || item.isoDate || '',\n        author: item.creator || (item as any).author || '',\n        categories: Array.isArray(item.categories)\n          ? item.categories\n          : item.category\n            ? [item.category]\n            : [],\n        guid: item.guid || item.guid || '',\n        contentEncoded:\n          (item as any)['content:encoded'] || (item as any).content || '',\n      })),\n    };\n  } catch (error) {\n    console.error(`Failed to parse RSS feed ${feedUrl}:`, error);\n    throw new Error(\n      `RSS parsing failed: ${error instanceof Error ? error.message : 'Unknown error'}`,\n    );\n  }\n}\n\nasync function fetchArticleContent(url: string): Promise<string> {\n  try {\n    console.log(`Fetching full article content from: ${url}`);\n\n    const response = await fetch(url, {\n      headers: {\n        'User-Agent': 'Mozilla/5.0 (compatible; RSS-RAG-Bot/1.0)',\n        Accept:\n          'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n        'Accept-Language': 'en-US,en;q=0.5',\n        'Accept-Encoding': 'gzip, deflate',\n        Connection: 'keep-alive',\n      },\n      signal: AbortSignal.timeout(10000), // 10 second timeout\n    });\n\n    if (!response.ok) {\n      throw new Error(`HTTP ${response.status}: ${response.statusText}`);\n    }\n\n    const html = await response.text();\n\n    const { JSDOM } = await import('jsdom');\n    const dom = new JSDOM(html, { url });\n    const document = dom.window.document;\n\n    // Use Mozilla's Readability to extract the main content\n    const reader = new Readability(document);\n    const article = reader.parse();\n\n    if (!article) {\n      throw new Error('Readability failed to extract article content');\n    }\n\n    // Combine title and content for comprehensive text\n    const fullContent = `${article.title ? article.title + '\\n\\n' : ''}${article.textContent || article.content}`;\n\n    // Ensure we have meaningful content (at least 200 characters for full articles)\n    if (fullContent.length < 200) {\n      throw new Error(\n        `Extracted content too short (${fullContent.length} chars) - likely failed to find main content`,\n      );\n    }\n\n    console.log(\n      `\u2713 Successfully extracted article: \"${article.title}\" (${fullContent.length} chars, ~${Math.round(fullContent.split(' ').length)} words)`,\n    );\n\n    return fullContent;\n  } catch (error) {\n    console.warn(\n      `Failed to fetch article content from ${url}:`,\n      error instanceof Error ? error.message : error,\n    );\n    return ''; // Return empty string on failure rather than throwing\n  }\n}\nexport function rss(\n  feedUrl: string,\n  options: {\n    maxItems?: number;\n    fetchFullArticles?: boolean;\n  } = {},\n) {\n  const sourceId = `rss:${feedUrl}`;\n  const { maxItems = 50, fetchFullArticles = false } = options;\n\n  return {\n    sourceId,\n    instructions: `You answer questions about articles and content from the RSS feed: ${feedUrl}.\n      Always cite the article title and link when referencing specific content.\n      The feed contains recent articles, blog posts, and news items.\n      When referencing content, include the publication date and author when available.\n      ${fetchFullArticles ? 'Full article content has been extracted from the original links for comprehensive analysis.' : 'Content includes RSS summaries and descriptions.'}`,\n    sources: async function* () {\n      const feed = await parseRSSFeed(feedUrl);\n      // Add feed summary source\n      const feedSummary = `RSS Feed: ${feed.title}\nDescription: ${feed.description}\nWebsite: ${feed.link}\nLanguage: ${feed.language}\nLast Updated: ${feed.lastBuildDate}\nTotal Items: ${feed.items.length}\n\nThis feed provides: ${feed.description}`;\n      yield {\n        id: 'feed-info',\n        content: async () => feedSummary,\n      };\n\n      // Individual article sources (limit to maxItems)\n      const itemsToProcess = feed.items.slice(0, maxItems);\n      for (const item of itemsToProcess) {\n        const documentId =\n          item.guid || item.link || `${item.title}-${item.pubDate}`;\n        yield {\n          id: documentId,\n          content: async () => {\n            // Try full article fetch if enabled, fallback to RSS content\n            let articleContent = item.contentEncoded || item.description;\n            if (fetchFullArticles && item.link) {\n              const fullContent = await fetchArticleContent(item.link);\n              if (fullContent && fullContent.length > articleContent.length) {\n                articleContent = fullContent;\n              }\n            }\n            return `Title: ${item.title}\nAuthor: ${item.author}\nPublished: ${item.pubDate}\nCategories: ${item.categories.join(', ')}\nLink: ${item.link}\n${fetchFullArticles ? 'Full Article Content:' : 'Content:'}\n${articleContent}\n\nSummary: ${item.title} - ${item.description}`;\n          },\n        };\n      }\n    },\n  };\n}\n"],
+  "mappings": "AAAA,SAAS,mBAAmB;AAC5B,OAAO,YAAY;AAEnB,MAAM,YAAY,IAAI,OAAO;AAAA,EAC3B,cAAc;AAAA,IACZ,MAAM,CAAC,aAAa,YAAY,kBAAkB,WAAW;AAAA,IAC7D,MAAM,CAAC,YAAY,WAAW,aAAa,MAAM;AAAA,EACnD;AACF,CAAC;AAED,eAAe,aAAa,SAAiB;AAC3C,MAAI;AACF,YAAQ,IAAI,sBAAsB,OAAO,EAAE;AAC3C,UAAM,OAAO,MAAM,UAAU,SAAS,OAAO;AAE7C,WAAO;AAAA,MACL,OAAO,KAAK,SAAS;AAAA,MACrB,aAAa,KAAK,eAAe;AAAA,MACjC,MAAM,KAAK,QAAQ;AAAA,MACnB,UAAU,KAAK,YAAY;AAAA,MAC3B,eAAgB,KAAa,kBAAiB,oBAAI,KAAK,GAAE,YAAY;AAAA,MACrE,OAAO,KAAK,MAAM,IAAI,CAAC,UAAU;AAAA,QAC/B,OAAO,KAAK,SAAS;AAAA,QACrB,aAAa,KAAK,WAAW,KAAK,WAAW,KAAK,kBAAkB;AAAA,QACpE,MAAM,KAAK,QAAQ;AAAA,QACnB,SAAS,KAAK,WAAW,KAAK,WAAW;AAAA,QACzC,QAAQ,KAAK,WAAY,KAAa,UAAU;AAAA,QAChD,YAAY,MAAM,QAAQ,KAAK,UAAU,IACrC,KAAK,aACL,KAAK,WACH,CAAC,KAAK,QAAQ,IACd,CAAC;AAAA,QACP,MAAM,KAAK,QAAQ,KAAK,QAAQ;AAAA,QAChC,gBACG,KAAa,iBAAiB,KAAM,KAAa,WAAW;AAAA,MACjE,EAAE;AAAA,IACJ;AAAA,EACF,SAAS,OAAO;AACd,YAAQ,MAAM,4BAA4B,OAAO,KAAK,KAAK;AAC3D,UAAM,IAAI;AAAA,MACR,uBAAuB,iBAAiB,QAAQ,MAAM,UAAU,eAAe;AAAA,IACjF;AAAA,EACF;AACF;AAEA,eAAe,oBAAoB,KAA8B;AAC/D,MAAI;AACF,YAAQ,IAAI,uCAAuC,GAAG,EAAE;AAExD,UAAM,WAAW,MAAM,MAAM,KAAK;AAAA,MAChC,SAAS;AAAA,QACP,cAAc;AAAA,QACd,QACE;AAAA,QACF,mBAAmB;AAAA,QACnB,mBAAmB;AAAA,QACnB,YAAY;AAAA,MACd;AAAA,MACA,QAAQ,YAAY,QAAQ,GAAK;AAAA;AAAA,IACnC,CAAC;AAED,QAAI,CAAC,SAAS,IAAI;AAChB,YAAM,IAAI,MAAM,QAAQ,SAAS,MAAM,KAAK,SAAS,UAAU,EAAE;AAAA,IACnE;AAEA,UAAM,OAAO,MAAM,SAAS,KAAK;AAEjC,UAAM,EAAE,MAAM,IAAI,MAAM,OAAO,OAAO;AACtC,UAAM,MAAM,IAAI,MAAM,MAAM,EAAE,IAAI,CAAC;AACnC,UAAM,WAAW,IAAI,OAAO;AAG5B,UAAM,SAAS,IAAI,YAAY,QAAQ;AACvC,UAAM,UAAU,OAAO,MAAM;AAE7B,QAAI,CAAC,SAAS;AACZ,YAAM,IAAI,MAAM,+CAA+C;AAAA,IACjE;AAGA,UAAM,cAAc,GAAG,QAAQ,QAAQ,QAAQ,QAAQ,SAAS,EAAE,GAAG,QAAQ,eAAe,QAAQ,OAAO;AAG3G,QAAI,YAAY,SAAS,KAAK;AAC5B,YAAM,IAAI;AAAA,QACR,gCAAgC,YAAY,MAAM;AAAA,MACpD;AAAA,IACF;AAEA,YAAQ;AAAA,MACN,2CAAsC,QAAQ,KAAK,MAAM,YAAY,MAAM,YAAY,KAAK,MAAM,YAAY,MAAM,GAAG,EAAE,MAAM,CAAC;AAAA,IAClI;AAEA,WAAO;AAAA,EACT,SAAS,OAAO;AACd,YAAQ;AAAA,MACN,wCAAwC,GAAG;AAAA,MAC3C,iBAAiB,QAAQ,MAAM,UAAU;AAAA,IAC3C;AACA,WAAO;AAAA,EACT;AACF;AACO,SAAS,IACd,SACA,UAGI,CAAC,GACL;AACA,QAAM,WAAW,OAAO,OAAO;AAC/B,QAAM,EAAE,WAAW,IAAI,oBAAoB,MAAM,IAAI;AAErD,SAAO;AAAA,IACL;AAAA,IACA,cAAc,sEAAsE,OAAO;AAAA;AAAA;AAAA;AAAA,QAIvF,oBAAoB,gGAAgG,kDAAkD;AAAA,IAC1K,SAAS,mBAAmB;AAC1B,YAAM,OAAO,MAAM,aAAa,OAAO;AAEvC,YAAM,cAAc,aAAa,KAAK,KAAK;AAAA,eAClC,KAAK,WAAW;AAAA,WACpB,KAAK,IAAI;AAAA,YACR,KAAK,QAAQ;AAAA,gBACT,KAAK,aAAa;AAAA,eACnB,KAAK,MAAM,MAAM;AAAA;AAAA,sBAEV,KAAK,WAAW;AAChC,YAAM;AAAA,QACJ,IAAI;AAAA,QACJ,SAAS,YAAY;AAAA,MACvB;AAGA,YAAM,iBAAiB,KAAK,MAAM,MAAM,GAAG,QAAQ;AACnD,iBAAW,QAAQ,gBAAgB;AACjC,cAAM,aACJ,KAAK,QAAQ,KAAK,QAAQ,GAAG,KAAK,KAAK,IAAI,KAAK,OAAO;AACzD,cAAM;AAAA,UACJ,IAAI;AAAA,UACJ,SAAS,YAAY;AAEnB,gBAAI,iBAAiB,KAAK,kBAAkB,KAAK;AACjD,gBAAI,qBAAqB,KAAK,MAAM;AAClC,oBAAM,cAAc,MAAM,oBAAoB,KAAK,IAAI;AACvD,kBAAI,eAAe,YAAY,SAAS,eAAe,QAAQ;AAC7D,iCAAiB;AAAA,cACnB;AAAA,YACF;AACA,mBAAO,UAAU,KAAK,KAAK;AAAA,UAC7B,KAAK,MAAM;AAAA,aACR,KAAK,OAAO;AAAA,cACX,KAAK,WAAW,KAAK,IAAI,CAAC;AAAA,QAChC,KAAK,IAAI;AAAA,EACf,oBAAoB,0BAA0B,UAAU;AAAA,EACxD,cAAc;AAAA;AAAA,WAEL,KAAK,KAAK,MAAM,KAAK,WAAW;AAAA,UACjC;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;",
+  "names": []
+}

package/dist/lib/embedders/fastembed.d.ts

@@ -0,0 +1,11 @@
+import { EmbeddingModel } from 'fastembed';
+import type { Embedder } from '../stores/store.js';
+type StandardModel = EmbeddingModel.AllMiniLML6V2 | EmbeddingModel.BGEBaseEN | EmbeddingModel.BGEBaseENV15 | EmbeddingModel.BGESmallEN | EmbeddingModel.BGESmallENV15 | EmbeddingModel.BGESmallZH | EmbeddingModel.MLE5Large;
+export interface FastEmbedOptions {
+    model?: StandardModel;
+    batchSize?: number;
+    cacheDir?: string;
+}
+export declare function fastembed(options?: FastEmbedOptions): Embedder;
+export {};
+//# sourceMappingURL=fastembed.d.ts.map

package/dist/lib/embedders/fastembed.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fastembed.d.ts","sourceRoot":"","sources":["../../../src/lib/embedders/fastembed.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAiB,MAAM,WAAW,CAAC;AAE1D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAEnD,KAAK,aAAa,GACd,cAAc,CAAC,aAAa,GAC5B,cAAc,CAAC,SAAS,GACxB,cAAc,CAAC,YAAY,GAC3B,cAAc,CAAC,UAAU,GACzB,cAAc,CAAC,aAAa,GAC5B,cAAc,CAAC,UAAU,GACzB,cAAc,CAAC,SAAS,CAAC;AAE7B,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,aAAa,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,wBAAgB,SAAS,CAAC,OAAO,GAAE,gBAAqB,GAAG,QAAQ,CAoClE"}

package/dist/lib/embedders/fastembed.js

@@ -0,0 +1,35 @@
+import { EmbeddingModel, FlagEmbedding } from "fastembed";
+function fastembed(options = {}) {
+  const {
+    model: modelId = EmbeddingModel.BGESmallENV15,
+    batchSize,
+    cacheDir
+  } = options;
+  let modelPromise = null;
+  const getModel = () => {
+    if (!modelPromise) {
+      modelPromise = FlagEmbedding.init({
+        model: modelId,
+        cacheDir
+      });
+    }
+    return modelPromise;
+  };
+  return async (documents) => {
+    const model = await getModel();
+    const batches = model.embed(documents, batchSize);
+    const embeddings = [];
+    let dimensions = 0;
+    for await (const batch of batches) {
+      for (const vec of batch) {
+        if (dimensions === 0) dimensions = vec.length;
+        embeddings.push(vec);
+      }
+    }
+    return { embeddings, dimensions };
+  };
+}
+export {
+  fastembed
+};
+//# sourceMappingURL=fastembed.js.map

package/dist/lib/embedders/fastembed.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../src/lib/embedders/fastembed.ts"],
+  "sourcesContent": ["import { EmbeddingModel, FlagEmbedding } from 'fastembed';\n\nimport type { Embedder } from '../stores/store.js';\n\ntype StandardModel =\n  | EmbeddingModel.AllMiniLML6V2\n  | EmbeddingModel.BGEBaseEN\n  | EmbeddingModel.BGEBaseENV15\n  | EmbeddingModel.BGESmallEN\n  | EmbeddingModel.BGESmallENV15\n  | EmbeddingModel.BGESmallZH\n  | EmbeddingModel.MLE5Large;\n\nexport interface FastEmbedOptions {\n  model?: StandardModel;\n  batchSize?: number;\n  cacheDir?: string;\n}\n\nexport function fastembed(options: FastEmbedOptions = {}): Embedder {\n  const {\n    model: modelId = EmbeddingModel.BGESmallENV15,\n    batchSize,\n    cacheDir,\n  } = options;\n\n  let modelPromise: Promise<\n    Awaited<ReturnType<typeof FlagEmbedding.init>>\n  > | null = null;\n  const getModel = () => {\n    if (!modelPromise) {\n      modelPromise = FlagEmbedding.init({\n        model: modelId,\n        cacheDir,\n      });\n    }\n    return modelPromise;\n  };\n\n  return async (documents: string[]) => {\n    const model = await getModel();\n    const batches = model.embed(documents, batchSize);\n\n    const embeddings: number[][] = [];\n    let dimensions = 0;\n\n    for await (const batch of batches) {\n      for (const vec of batch) {\n        if (dimensions === 0) dimensions = vec.length;\n        embeddings.push(vec);\n      }\n    }\n\n    return { embeddings, dimensions };\n  };\n}\n"],
+  "mappings": "AAAA,SAAS,gBAAgB,qBAAqB;AAmBvC,SAAS,UAAU,UAA4B,CAAC,GAAa;AAClE,QAAM;AAAA,IACJ,OAAO,UAAU,eAAe;AAAA,IAChC;AAAA,IACA;AAAA,EACF,IAAI;AAEJ,MAAI,eAEO;AACX,QAAM,WAAW,MAAM;AACrB,QAAI,CAAC,cAAc;AACjB,qBAAe,cAAc,KAAK;AAAA,QAChC,OAAO;AAAA,QACP;AAAA,MACF,CAAC;AAAA,IACH;AACA,WAAO;AAAA,EACT;AAEA,SAAO,OAAO,cAAwB;AACpC,UAAM,QAAQ,MAAM,SAAS;AAC7B,UAAM,UAAU,MAAM,MAAM,WAAW,SAAS;AAEhD,UAAM,aAAyB,CAAC;AAChC,QAAI,aAAa;AAEjB,qBAAiB,SAAS,SAAS;AACjC,iBAAW,OAAO,OAAO;AACvB,YAAI,eAAe,EAAG,cAAa,IAAI;AACvC,mBAAW,KAAK,GAAG;AAAA,MACrB;AAAA,IACF;AAEA,WAAO,EAAE,YAAY,WAAW;AAAA,EAClC;AACF;",
+  "names": []
+}

package/dist/lib/embedders/huggingface.d.ts

@@ -0,0 +1,14 @@
+import type { FeatureExtractionPipeline, Tensor } from '@huggingface/transformers';
+import type { Embedder } from '../stores/store.js';
+export type FeatureExtractionFn = FeatureExtractionPipeline;
+export interface EmbeddingOptions {
+    extractorFn: () => Promise<FeatureExtractionPipeline> | FeatureExtractionPipeline;
+    pooling?: 'mean' | 'cls';
+    normalize?: boolean;
+}
+export declare function tensorToEmbeddings(tensor: Tensor): {
+    embeddings: Float32Array<ArrayBufferLike>[];
+    dimensions: number;
+};
+export declare function huggingface(extractorFn: () => Promise<FeatureExtractionPipeline> | FeatureExtractionPipeline): Embedder;
+//# sourceMappingURL=huggingface.d.ts.map

package/dist/lib/embedders/huggingface.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"huggingface.d.ts","sourceRoot":"","sources":["../../../src/lib/embedders/huggingface.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,yBAAyB,EACzB,MAAM,EACP,MAAM,2BAA2B,CAAC;AAEnC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAEnD,MAAM,MAAM,mBAAmB,GAAG,yBAAyB,CAAC;AAE5D,MAAM,WAAW,gBAAgB;IAC/B,WAAW,EAAE,MACT,OAAO,CAAC,yBAAyB,CAAC,GAClC,yBAAyB,CAAC;IAC9B,OAAO,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC;IACzB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAeD,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM;;;EAuBhD;AAED,wBAAgB,WAAW,CACzB,WAAW,EAAE,MACT,OAAO,CAAC,yBAAyB,CAAC,GAClC,yBAAyB,GAC5B,QAAQ,CAEV"}

package/dist/lib/embedders/huggingface.js

@@ -0,0 +1,40 @@
+async function textEmbeddings(inputs, { extractorFn, pooling, normalize }) {
+  inputs = (Array.isArray(inputs) ? inputs : [inputs]).map((it) => it.trim());
+  const extractor = await extractorFn();
+  const tensor = await extractor(inputs, {
+    pooling: pooling ?? "mean",
+    normalize: normalize ?? true
+  });
+  return tensorToEmbeddings(tensor);
+}
+function tensorToEmbeddings(tensor) {
+  const dims = tensor.dims;
+  if (!Array.isArray(dims) || dims.length < 2) {
+    throw new Error(`Unexpected tensor dims: ${JSON.stringify(dims)}`);
+  }
+  const batchSize = dims[0];
+  const hiddenSize = dims[dims.length - 1];
+  const expectedLen = batchSize * hiddenSize;
+  if (tensor.data.length !== expectedLen) {
+    throw new Error(
+      `Data length mismatch: got ${tensor.data.length}, expected ${expectedLen} (batch=${batchSize}, hidden=${hiddenSize})`
+    );
+  }
+  const embeddings = [];
+  const dimensions = tensor.dims[tensor.dims.length - 1];
+  for (let i = 0; i < batchSize; i++) {
+    const start = i * hiddenSize;
+    embeddings.push(
+      tensor.data.subarray(start, start + hiddenSize)
+    );
+  }
+  return { embeddings, dimensions };
+}
+function huggingface(extractorFn) {
+  return (documents) => textEmbeddings(documents, { extractorFn });
+}
+export {
+  huggingface,
+  tensorToEmbeddings
+};
+//# sourceMappingURL=huggingface.js.map

package/dist/lib/embedders/huggingface.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../src/lib/embedders/huggingface.ts"],
+  "sourcesContent": ["import type {\n  FeatureExtractionPipeline,\n  Tensor,\n} from '@huggingface/transformers';\n\nimport type { Embedder } from '../stores/store.js';\n\nexport type FeatureExtractionFn = FeatureExtractionPipeline;\n\nexport interface EmbeddingOptions {\n  extractorFn: () =>\n    | Promise<FeatureExtractionPipeline>\n    | FeatureExtractionPipeline;\n  pooling?: 'mean' | 'cls';\n  normalize?: boolean;\n}\n\nasync function textEmbeddings(\n  inputs: string | string[],\n  { extractorFn, pooling, normalize }: EmbeddingOptions,\n) {\n  inputs = (Array.isArray(inputs) ? inputs : [inputs]).map((it) => it.trim());\n  const extractor = await extractorFn();\n  const tensor = await extractor(inputs, {\n    pooling: pooling ?? 'mean',\n    normalize: normalize ?? true,\n  });\n  return tensorToEmbeddings(tensor);\n}\n\nexport function tensorToEmbeddings(tensor: Tensor) {\n  const dims = tensor.dims;\n  if (!Array.isArray(dims) || dims.length < 2) {\n    throw new Error(`Unexpected tensor dims: ${JSON.stringify(dims)}`);\n  }\n  const batchSize = dims[0];\n  const hiddenSize = dims[dims.length - 1];\n  const expectedLen = batchSize * hiddenSize;\n  if (tensor.data.length !== expectedLen) {\n    throw new Error(\n      `Data length mismatch: got ${tensor.data.length}, expected ${expectedLen} (batch=${batchSize}, hidden=${hiddenSize})`,\n    );\n  }\n  // Reuse the underlying typed array without copying; we'll copy when creating Buffer for SQLite\n  const embeddings: Float32Array[] = [];\n  const dimensions = tensor.dims[tensor.dims.length - 1];\n  for (let i = 0; i < batchSize; i++) {\n    const start = i * hiddenSize;\n    embeddings.push(\n      (tensor.data as Float32Array).subarray(start, start + hiddenSize),\n    );\n  }\n  return { embeddings, dimensions };\n}\n\nexport function huggingface(\n  extractorFn: () =>\n    | Promise<FeatureExtractionPipeline>\n    | FeatureExtractionPipeline,\n): Embedder {\n  return (documents) => textEmbeddings(documents, { extractorFn });\n}\n"],
+  "mappings": "AAiBA,eAAe,eACb,QACA,EAAE,aAAa,SAAS,UAAU,GAClC;AACA,YAAU,MAAM,QAAQ,MAAM,IAAI,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;AAC1E,QAAM,YAAY,MAAM,YAAY;AACpC,QAAM,SAAS,MAAM,UAAU,QAAQ;AAAA,IACrC,SAAS,WAAW;AAAA,IACpB,WAAW,aAAa;AAAA,EAC1B,CAAC;AACD,SAAO,mBAAmB,MAAM;AAClC;AAEO,SAAS,mBAAmB,QAAgB;AACjD,QAAM,OAAO,OAAO;AACpB,MAAI,CAAC,MAAM,QAAQ,IAAI,KAAK,KAAK,SAAS,GAAG;AAC3C,UAAM,IAAI,MAAM,2BAA2B,KAAK,UAAU,IAAI,CAAC,EAAE;AAAA,EACnE;AACA,QAAM,YAAY,KAAK,CAAC;AACxB,QAAM,aAAa,KAAK,KAAK,SAAS,CAAC;AACvC,QAAM,cAAc,YAAY;AAChC,MAAI,OAAO,KAAK,WAAW,aAAa;AACtC,UAAM,IAAI;AAAA,MACR,6BAA6B,OAAO,KAAK,MAAM,cAAc,WAAW,WAAW,SAAS,YAAY,UAAU;AAAA,IACpH;AAAA,EACF;AAEA,QAAM,aAA6B,CAAC;AACpC,QAAM,aAAa,OAAO,KAAK,OAAO,KAAK,SAAS,CAAC;AACrD,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,UAAM,QAAQ,IAAI;AAClB,eAAW;AAAA,MACR,OAAO,KAAsB,SAAS,OAAO,QAAQ,UAAU;AAAA,IAClE;AAAA,EACF;AACA,SAAO,EAAE,YAAY,WAAW;AAClC;AAEO,SAAS,YACd,aAGU;AACV,SAAO,CAAC,cAAc,eAAe,WAAW,EAAE,YAAY,CAAC;AACjE;",
+  "names": []
+}

package/dist/lib/ingest.d.ts

@@ -0,0 +1,24 @@
+import type { Connector } from './connectors/connector.js';
+import type { Splitter } from './pipeline.js';
+import type { Embedder, Store } from './stores/store.js';
+export interface IngestionConfig {
+    connector: Connector;
+    store: Store;
+    splitter?: Splitter;
+    embedder: Embedder;
+}
+export declare function ingest(config: IngestionConfig, callback?: (it: string) => void): Promise<void>;
+export type ChunkPosition = {
+    startLine: number;
+    startColumn: number;
+    endLine: number;
+    endColumn: number;
+};
+export type SplitChunkWithPosition = {
+    content: string;
+    position: ChunkPosition | null;
+    index: number;
+};
+export declare function splitTypeScriptWithPositions(id: string, content: string): Promise<SplitChunkWithPosition[]>;
+export declare function splitTypeScript(id: string, content: string): Promise<string[]>;
+//# sourceMappingURL=ingest.d.ts.map

package/dist/lib/ingest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../../src/lib/ingest.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAE9C,OAAO,KAAK,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAEzD,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,SAAS,CAAC;IACrB,KAAK,EAAE,KAAK,CAAC;IACb,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;CACpB;AAED,wBAAsB,MAAM,CAC1B,MAAM,EAAE,eAAe,EACvB,QAAQ,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,IAAI,iBAkChC;AAOD,MAAM,MAAM,aAAa,GAAG;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,sBAAsB,GAAG;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,aAAa,GAAG,IAAI,CAAC;IAC/B,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AA+EF,wBAAsB,4BAA4B,CAChD,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,sBAAsB,EAAE,CAAC,CAQnC;AAED,wBAAsB,eAAe,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,qBAGhE"}

package/dist/lib/ingest.js

@@ -0,0 +1,111 @@
+import {
+  MarkdownTextSplitter,
+  RecursiveCharacterTextSplitter
+} from "langchain/text_splitter";
+import { cid } from "./stores/cid.js";
+async function ingest(config, callback) {
+  const splitter = config.splitter ?? split;
+  const embedder = config.embedder;
+  const corpuses = config.connector.sources();
+  for await (const it of corpuses) {
+    callback?.(it.id);
+    const content = await it.content();
+    if (!content.trim()) {
+      continue;
+    }
+    await config.store.index(config.connector.sourceId, {
+      id: it.id,
+      cid: cid(content),
+      metadata: it.metadata,
+      chunker: async function* () {
+        const values = await splitter(it.id, content);
+        const batchSize = 40;
+        for (let i = 0; i < values.length; i += batchSize) {
+          const batch = values.slice(i, i + batchSize);
+          const { embeddings } = await embedder(batch);
+          for (let j = 0; j < embeddings.length; j++) {
+            yield {
+              content: batch[j],
+              embedding: embeddings[j]
+            };
+          }
+        }
+      }
+    });
+  }
+}
+function split(id, content) {
+  const splitter = new MarkdownTextSplitter();
+  return splitter.splitText(content);
+}
+function normalizeNewlines(value) {
+  return value.replace(/\r\n/g, "\n");
+}
+function computePositions(originalContent, chunks) {
+  if (!chunks.length) {
+    return [];
+  }
+  const normalizedContent = normalizeNewlines(originalContent);
+  const positions = [];
+  let searchOffset = 0;
+  for (const chunk of chunks) {
+    const normalizedChunk = normalizeNewlines(chunk);
+    const trimmedChunk = normalizedChunk.trim();
+    const seek = (needle, fromIndex) => needle ? normalizedContent.indexOf(needle, fromIndex) : -1;
+    let matchIndex = seek(normalizedChunk, searchOffset);
+    let matchValue = normalizedChunk;
+    if (matchIndex === -1 && trimmedChunk) {
+      matchIndex = seek(trimmedChunk, searchOffset);
+      matchValue = trimmedChunk;
+    }
+    if (matchIndex === -1) {
+      matchIndex = seek(normalizedChunk, 0);
+      matchValue = normalizedChunk;
+    }
+    if (matchIndex === -1 && trimmedChunk) {
+      matchIndex = seek(trimmedChunk, 0);
+      matchValue = trimmedChunk;
+    }
+    if (matchIndex === -1) {
+      positions.push(null);
+      continue;
+    }
+    const before = normalizedContent.slice(0, matchIndex);
+    const beforeLines = before.split("\n");
+    const startLine = beforeLines.length;
+    const startColumn = beforeLines[beforeLines.length - 1].length + 1;
+    const lines = matchValue.split("\n");
+    const endLine = startLine + lines.length - 1;
+    const endColumn = lines.length === 1 ? startColumn + lines[0].length : lines[lines.length - 1].length + 1;
+    positions.push({ startLine, startColumn, endLine, endColumn });
+    searchOffset = matchIndex + matchValue.length;
+  }
+  return positions;
+}
+function buildChunksWithPositions(originalContent, chunks) {
+  const positions = computePositions(originalContent, chunks);
+  return chunks.map((content, index) => ({
+    content,
+    index,
+    position: positions[index] ?? null
+  }));
+}
+async function splitTypeScriptWithPositions(id, content) {
+  const splitter = RecursiveCharacterTextSplitter.fromLanguage("js", {
+    chunkSize: 512,
+    chunkOverlap: 100
+  });
+  const docs = await splitter.createDocuments([content]);
+  const chunks = docs.map((d) => d.pageContent);
+  return buildChunksWithPositions(content, chunks);
+}
+async function splitTypeScript(id, content) {
+  const chunks = await splitTypeScriptWithPositions(id, content);
+  return chunks.map((chunk) => chunk.content);
+}
+export {
+  ingest,
+  splitTypeScript,
+  splitTypeScriptWithPositions
+};
+//# sourceMappingURL=ingest.js.map