agdex 0.5.1 → 0.6.1

package/dist/cli/index.js

@@ -1,8 +1,5 @@
 #!/usr/bin/env node
 import {
-  __commonJS,
-  __require,
-  __toESM,
   basedpyrightProvider,
   buildDocTree,
   bunProvider,
@@ -42,7 +39,12 @@ import {
   tailwindProvider,
   tauriProvider,
   tyProvider
-} from "../index-thmt54kg.js";
+} from "../index-cm3qmz9v.js";
+import {
+  __commonJS,
+  __require,
+  __toESM
+} from "../index-dtcewfnz.js";
 
 // node_modules/commander/lib/error.js
 var require_error = __commonJS((exports) => {
@@ -2107,7 +2109,7 @@ var require_clear = __commonJS((exports, module) => {
         if (it)
           o = it;
         var i = 0;
-        var F = function F() {};
+        var F = function F2() {};
         return { s: F, n: function n() {
           if (i >= o.length)
             return { done: true };
@@ -4439,7 +4441,7 @@ var require_dist = __commonJS((exports, module) => {
         if (it)
           o = it;
         var i = 0;
-        var F = function F() {};
+        var F = function F2() {};
         return { s: F, n: function n() {
           if (i >= o.length)
             return { done: true };
@@ -4542,7 +4544,7 @@ In order to be iterable, non-array objects must have a [Symbol.iterator]() metho
           }
           return question2.format ? yield question2.format(answer2, answers) : answer2;
         });
-        return function getFormattedAnswer(_x, _x2) {
+        return function getFormattedAnswer2(_x, _x2) {
           return _ref.apply(this, arguments);
         };
       }();
@@ -7048,6 +7050,13 @@ Cancelled.`));
 }
 async function runEmbed(options) {
   const cwd = process.cwd();
+  if (options.url) {
+    await runUrl(options.url, {
+      output: options.output,
+      name: options.description
+    });
+    return;
+  }
   let provider;
   let version = options.fwVersion;
   let output;
@@ -7090,7 +7099,7 @@ async function runEmbed(options) {
     return;
   }
   output = options.output || getDefaultOutput();
-  if (!version && !provider.detectVersion) {
+  if (!version && !provider.detectVersion && !provider.urlConfig) {
     console.error(import_picocolors2.default.red(`Provider ${provider.displayName} requires --version flag since auto-detection is not supported.`));
     process.exit(1);
   }
@@ -7100,7 +7109,10 @@ async function runEmbed(options) {
 async function executeEmbed(cwd, provider, version, output, globalCache, description) {
   let resolvedVersion = version;
   let usingDefaultBranch = false;
-  if (!resolvedVersion && provider.detectVersion) {
+  if (!resolvedVersion && provider.urlConfig) {
+    resolvedVersion = "latest";
+    usingDefaultBranch = true;
+  } else if (!resolvedVersion && provider.detectVersion) {
     const detected = provider.detectVersion(cwd);
     if (!detected.version) {
       const fallbackBranch = provider.defaultBranch || "main";
@@ -7555,6 +7567,68 @@ Building index from ${import_picocolors2.default.cyan(docsPath)}...`);
   console.log(`${import_picocolors2.default.green("✓")} ${action} ${import_picocolors2.default.bold(output)} (${sizeInfo})`);
   console.log("");
 }
+async function runUrl(url, options) {
+  const cwd = process.cwd();
+  const { createUrlProvider } = await import("../url-scraper-5sj8c56t.js");
+  const { pullDocsFromUrl } = await import("../url-scraper-5sj8c56t.js");
+  const name = options.name || new URL(url).hostname.replace(/^docs\./, "").replace(/\.\w+$/, "");
+  const providerName = name.toLowerCase().replace(/\s+/g, "-");
+  const output = options.output || getDefaultOutput();
+  const cacheBase = path.join(os.homedir(), ".cache", "agdex");
+  const docsPath = path.join(cacheBase, providerName);
+  console.log(`
+Scraping documentation from ${import_picocolors2.default.cyan(url)}...`);
+  const cacheHit = fs.existsSync(docsPath) && fs.readdirSync(docsPath).length > 0;
+  if (cacheHit) {
+    console.log(`${import_picocolors2.default.green("✓")} Using cached docs from ${import_picocolors2.default.bold(docsPath)}`);
+  } else {
+    const urlConfig = {
+      baseUrl: url,
+      contentSelector: options.selector || "main#main-content, main, article, .body",
+      removeSelectors: [],
+      concurrency: options.concurrency ? parseInt(options.concurrency, 10) : 5,
+      fetchDelay: options.delay ? parseInt(options.delay, 10) : 200
+    };
+    const pullResult = await pullDocsFromUrl(urlConfig, docsPath, {
+      onProgress: (current, total, page) => {
+        process.stdout.write(`\r  Fetching pages... ${current}/${total} (${page})`);
+      }
+    });
+    if (!pullResult.success) {
+      console.error(import_picocolors2.default.red(`
+Failed: ${pullResult.error}`));
+      process.exit(1);
+    }
+    console.log(`
+${import_picocolors2.default.green("✓")} Downloaded docs to ${import_picocolors2.default.bold(docsPath)}`);
+  }
+  const targetPath = path.join(cwd, output);
+  let existingContent = "";
+  let sizeBefore = 0;
+  let isNewFile = true;
+  if (fs.existsSync(targetPath)) {
+    existingContent = fs.readFileSync(targetPath, "utf-8");
+    sizeBefore = Buffer.byteLength(existingContent, "utf-8");
+    isNewFile = false;
+  }
+  const docFiles = collectDocFiles(docsPath, { extensions: [".md"] });
+  const sections = buildDocTree(docFiles);
+  const indexContent = generateIndex({
+    docsPath,
+    sections,
+    outputFile: output,
+    providerName: name,
+    instruction: `IMPORTANT: Prefer retrieval-led reasoning over pre-training-led reasoning for any ${name} tasks.`,
+    regenerateCommand: `npx agdex url "${url}" --name "${name}" --output ${output}`
+  });
+  const newContent = injectIndex(existingContent, indexContent, providerName);
+  fs.writeFileSync(targetPath, newContent, "utf-8");
+  const sizeAfter = Buffer.byteLength(newContent, "utf-8");
+  const action = isNewFile ? "Created" : "Updated";
+  const sizeInfo = isNewFile ? formatSize(sizeAfter) : `${formatSize(sizeBefore)} → ${formatSize(sizeAfter)}`;
+  console.log(`${import_picocolors2.default.green("✓")} ${action} ${import_picocolors2.default.bold(output)} (${sizeInfo})`);
+  console.log("");
+}
 function runList() {
   console.log(import_picocolors2.default.cyan(`
 \uD83D\uDCDA Built-in Documentation Providers
@@ -7563,8 +7637,8 @@ function runList() {
     const provider = getProvider(preset);
     const status = provider ? import_picocolors2.default.green("✓") : import_picocolors2.default.gray("○");
     const name = provider?.displayName || preset;
-    const repo = provider?.repo || "not implemented";
-    console.log(`  ${status} ${import_picocolors2.default.bold(preset)} - ${name} (${import_picocolors2.default.gray(repo)})`);
+    const source = provider?.urlConfig ? provider.urlConfig.baseUrl : provider?.repo || "not implemented";
+    console.log(`  ${status} ${import_picocolors2.default.bold(preset)} - ${name} (${import_picocolors2.default.gray(source)})`);
   }
   console.log("");
   console.log(import_picocolors2.default.cyan(`\uD83D\uDCE6 Usage Examples
@@ -7579,6 +7653,9 @@ function runList() {
   console.log(`    ${import_picocolors2.default.white("agdex")} ${import_picocolors2.default.gray("(interactive)")}`);
   console.log(`    ${import_picocolors2.default.gray("→ then enter:")} ${import_picocolors2.default.white("https://github.com/anthropics/skills/tree/main/skills")}`);
   console.log("");
+  console.log(import_picocolors2.default.gray("  Website URL:"));
+  console.log(`    ${import_picocolors2.default.white('agdex url https://docs.example.com/latest/index.html --name "My Docs"')}`);
+  console.log("");
   console.log(import_picocolors2.default.gray("  Local documentation:"));
   console.log(`    ${import_picocolors2.default.white('agdex local ./my-docs --name "My Docs"')}`);
   console.log("");
@@ -7591,14 +7668,16 @@ function runList() {
 program2.name("agdex").description(`Create compressed documentation indexes for AI coding agents.
 
 Sources you can index:
-  • Built-in providers (Next.js, React, Bun, Pixi, Tauri, etc.)
+  • Built-in providers (Next.js, React, Bun, Pixi, Tauri, TensorRT, etc.)
   • Any GitHub repository URL or owner/repo
+  • Any documentation website URL
   • Local documentation directories
   • Claude Code skills
 
 Run 'agdex' without arguments for interactive mode.`).version("0.4.2");
-program2.command("embed", { isDefault: true }).description("Embed documentation index into AGENTS.md/CLAUDE.md").option("-p, --provider <name>", "Documentation provider (nextjs, react, etc.)").option("--fw-version <version>", "Framework version (auto-detected if not provided)").option("-o, --output <file>", "Target file (default: from config or CLAUDE.md)").option("--repo <owner/repo>", "Custom GitHub repository").option("--docs-path <path>", "Path to docs folder in repository").option("-g, --global", "Store docs in global cache (~/.cache/agdex/) (default)").option("-l, --local", "Store docs in local .agdex/ instead of global cache").option("-d, --description <text>", "Additional description to include in the index").action(runEmbed);
+program2.command("embed", { isDefault: true }).description("Embed documentation index into AGENTS.md/CLAUDE.md").option("-p, --provider <name>", "Documentation provider (nextjs, react, etc.)").option("--fw-version <version>", "Framework version (auto-detected if not provided)").option("-o, --output <file>", "Target file (default: from config or CLAUDE.md)").option("--repo <owner/repo>", "Custom GitHub repository").option("--docs-path <path>", "Path to docs folder in repository").option("-g, --global", "Store docs in global cache (~/.cache/agdex/) (default)").option("-l, --local", "Store docs in local .agdex/ instead of global cache").option("-d, --description <text>", "Additional description to include in the index").option("-u, --url <url>", "Scrape documentation from a website URL").action(runEmbed);
 program2.command("local <docs-path>").description("Build index from local documentation directory").option("-n, --name <name>", "Display name for the documentation").option("-o, --output <file>", "Target file (default: from config or CLAUDE.md)").option("-e, --extensions <exts>", "File extensions to include (comma-separated, default: .md,.mdx)").action(runLocal);
+program2.command("url <url>").description("Scrape documentation from a website URL and build index").option("-n, --name <name>", "Display name for the documentation (default: derived from URL)").option("-o, --output <file>", "Target file (default: from config or CLAUDE.md)").option("-s, --selector <css>", "CSS selector for main content (default: main#main-content, main, article)").option("-c, --concurrency <n>", "Max concurrent fetches (default: 5)").option("--delay <ms>", "Delay between fetch batches in ms (default: 200)").action(runUrl);
 program2.command("list").description("List available documentation providers").action(runList);
 async function runRemove(options) {
   const cwd = process.cwd();

package/dist/{index-thmt54kg.js → index-1v32v5tn.js}

@@ -1,22 +1,6 @@
-import { createRequire } from "node:module";
-var __create = Object.create;
-var __getProtoOf = Object.getPrototypeOf;
-var __defProp = Object.defineProperty;
-var __getOwnPropNames = Object.getOwnPropertyNames;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __toESM = (mod, isNodeMode, target) => {
-  target = mod != null ? __create(__getProtoOf(mod)) : {};
-  const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
-  for (let key of __getOwnPropNames(mod))
-    if (!__hasOwnProp.call(to, key))
-      __defProp(to, key, {
-        get: () => mod[key],
-        enumerable: true
-      });
-  return to;
-};
-var __commonJS = (cb, mod) => () => (mod || cb((mod = { exports: {} }).exports, mod), mod.exports);
-var __require = /* @__PURE__ */ createRequire(import.meta.url);
+import {
+  __require
+} from "./index-sh9kr6eq.js";
 
 // src/lib/agents-md.ts
 import { execSync } from "child_process";
@@ -33,6 +17,11 @@ function getEndMarker(providerName) {
   return providerName ? `${END_MARKER_PREFIX}:${providerName}${MARKER_SUFFIX}` : `${END_MARKER_PREFIX}${MARKER_SUFFIX}`;
 }
 async function pullDocs(provider, options) {
+  if (provider.urlConfig) {
+    const { pullDocsFromUrl } = await import("./url-scraper-9xq4d56t.js");
+    const docsPath2 = options.docsDir ?? fs.mkdtempSync(path.join(os.tmpdir(), "agdex-"));
+    return pullDocsFromUrl(provider.urlConfig, docsPath2, { onProgress: options.onProgress });
+  }
   const { cwd, version: versionOverride, docsDir } = options;
   let version;
   if (versionOverride) {
@@ -1617,6 +1606,69 @@ var manimProvider = {
   instruction: "IMPORTANT: Prefer retrieval-led reasoning over pre-training-led reasoning for any Manim tasks. Manim is a Python library for mathematical animations."
 };
 
+// src/lib/providers/tensorrt.ts
+var tensorrtProvider = {
+  name: "tensorrt",
+  displayName: "NVIDIA TensorRT",
+  repo: "",
+  docsPath: "",
+  extensions: [".md"],
+  excludePatterns: [],
+  defaultBranch: "latest",
+  instruction: "IMPORTANT: Prefer retrieval-led reasoning over pre-training-led reasoning for any NVIDIA TensorRT tasks. These docs cover TensorRT SDK for optimizing deep learning models for high-performance inference.",
+  urlConfig: {
+    baseUrl: "https://docs.nvidia.com/deeplearning/tensorrt/latest/index.html",
+    contentSelector: "main#main-content, main, article, .body",
+    removeSelectors: [
+      ".headerlink",
+      ".toctree-wrapper",
+      ".breadcrumb",
+      ".page-navigation",
+      ".related-pages"
+    ],
+    pages: [
+      "getting-started/release-notes.html",
+      "getting-started/quick-start-guide.html",
+      "getting-started/support-matrix.html",
+      "installing-tensorrt/overview.html",
+      "installing-tensorrt/prerequisites.html",
+      "installing-tensorrt/installing.html",
+      "installing-tensorrt/upgrading.html",
+      "installing-tensorrt/uninstalling.html",
+      "architecture/architecture-overview.html",
+      "architecture/capabilities.html",
+      "architecture/how-trt-works.html",
+      "inference-library/c-api-docs.html",
+      "inference-library/python-api-docs.html",
+      "inference-library/sample-support-guide.html",
+      "inference-library/advanced.html",
+      "inference-library/work-quantized-types.html",
+      "inference-library/accuracy-considerations.html",
+      "inference-library/work-dynamic-shapes.html",
+      "inference-library/extending-custom-layers.html",
+      "inference-library/work-with-loops.html",
+      "inference-library/work-with-conditionals.html",
+      "inference-library/work-with-dla.html",
+      "inference-library/capture-replay.html",
+      "inference-library/work-with-transformers.html",
+      "performance/best-practices.html",
+      "api/c-api.html",
+      "api/python-api.html",
+      "api/migration-guide.html",
+      "api/onnx-graphsurgeon-api.html",
+      "api/polygraphy-api.html",
+      "reference/troubleshooting.html",
+      "reference/data-format-desc.html",
+      "reference/command-line-programs.html",
+      "reference/operators.html",
+      "reference/additional-resources.html",
+      "reference/glossary.html"
+    ],
+    concurrency: 3,
+    fetchDelay: 300
+  }
+};
+
 // src/lib/providers/generic.ts
 import fs19 from "fs";
 import path19 from "path";
@@ -1836,6 +1888,8 @@ function getProvider(preset) {
       return manimProvider;
     case "cuda-feedstock":
       return cudaFeedstockProvider;
+    case "tensorrt":
+      return tensorrtProvider;
     case "vue":
     case "astro":
       return null;
@@ -1844,7 +1898,7 @@ function getProvider(preset) {
   }
 }
 function listProviders() {
-  return ["nextjs", "react", "pixi", "rattler-build", "tauri", "conda-forge", "bun", "vue", "svelte", "sveltekit", "shadcn-svelte", "astro", "tailwind", "ruff", "ty", "basedpyright", "convex", "polars", "delta-rs", "obsidian", "obsidian-excalidraw", "ffmpeg", "manim", "cuda-feedstock"];
+  return ["nextjs", "react", "pixi", "rattler-build", "tauri", "conda-forge", "bun", "vue", "svelte", "sveltekit", "shadcn-svelte", "astro", "tailwind", "ruff", "ty", "basedpyright", "convex", "polars", "delta-rs", "obsidian", "obsidian-excalidraw", "ffmpeg", "manim", "cuda-feedstock", "tensorrt"];
 }
 function isProviderAvailable(preset) {
   return getProvider(preset) !== null;
@@ -2299,4 +2353,4 @@ function getDefaultOutput(cwd = process.cwd()) {
   return config.output || "AGENTS.md";
 }
 
-export { __toESM, __commonJS, __require, pullDocs, collectDocFiles, buildDocTree, generateIndex, hasExistingIndex, getEmbeddedProviders, removeDocsIndex, injectIndex, ensureGitignoreEntry, getGlobalCacheDir, getLocalCacheDir, embed, nextjsProvider, reactProvider, pixiProvider, rattlerBuildProvider, tauriProvider, condaForgeProvider, bunProvider, svelteProvider, tailwindProvider, ruffProvider, tyProvider, basedpyrightProvider, convexProvider, polarsProvider, deltaRsProvider, obsidianProvider, obsidianExcalidrawProvider, ffmpegProvider, manimProvider, createProvider, createLocalProvider, getProvider, listProviders, isProviderAvailable, fetchSkillsShSearch, getEnabledPluginSources, parseSkillFrontmatter, discoverPluginSkills, discoverFlatSkills, discoverSkillsShRepo, collectAllSkills, generateSkillsIndex, hasExistingSkillsIndex, removeSkillsIndex, injectSkillsIndex, getDefaultSkillSources, embedSkills, loadConfig, getDefaultOutput };
+export { pullDocs, collectDocFiles, buildDocTree, generateIndex, hasExistingIndex, getEmbeddedProviders, removeDocsIndex, injectIndex, ensureGitignoreEntry, getGlobalCacheDir, getLocalCacheDir, embed, nextjsProvider, reactProvider, pixiProvider, rattlerBuildProvider, tauriProvider, condaForgeProvider, bunProvider, svelteProvider, tailwindProvider, ruffProvider, tyProvider, basedpyrightProvider, convexProvider, polarsProvider, deltaRsProvider, obsidianProvider, obsidianExcalidrawProvider, ffmpegProvider, manimProvider, tensorrtProvider, createProvider, createLocalProvider, getProvider, listProviders, isProviderAvailable, fetchSkillsShSearch, getEnabledPluginSources, parseSkillFrontmatter, discoverPluginSkills, discoverFlatSkills, discoverSkillsShRepo, collectAllSkills, generateSkillsIndex, hasExistingSkillsIndex, removeSkillsIndex, injectSkillsIndex, getDefaultSkillSources, embedSkills, loadConfig, getDefaultOutput };