agdex 0.5.1 → 0.6.0

package/dist/index.d.ts

@@ -36,10 +36,11 @@
  * ```
  */
 export { embed, pullDocs, collectDocFiles, buildDocTree, generateIndex, injectIndex, hasExistingIndex, removeDocsIndex, getEmbeddedProviders, ensureGitignoreEntry, getGlobalCacheDir, getLocalCacheDir, } from './lib/agents-md';
-export { nextjsProvider, reactProvider, pixiProvider, rattlerBuildProvider, tauriProvider, condaForgeProvider, bunProvider, createProvider, createLocalProvider, getProvider, listProviders, isProviderAvailable, } from './lib/providers';
+export { nextjsProvider, reactProvider, pixiProvider, rattlerBuildProvider, tauriProvider, condaForgeProvider, bunProvider, tensorrtProvider, createProvider, createLocalProvider, createUrlProvider, getProvider, listProviders, isProviderAvailable, } from './lib/providers';
+export { pullDocsFromUrl } from './lib/url-scraper';
 export { embedSkills, collectAllSkills, parseSkillFrontmatter, discoverPluginSkills, discoverFlatSkills, generateSkillsIndex, injectSkillsIndex, hasExistingSkillsIndex, removeSkillsIndex, getDefaultSkillSources, getEnabledPluginSources, } from './lib/skills';
 export { loadConfig, getDefaultOutput } from './lib/config';
 export type { AgdexConfig } from './lib/config';
-export type { DocProvider, DocFile, DocSection, VersionResult, PullResult, GitignoreStatus, IndexOptions, EmbedOptions, EmbedResult, ProviderPreset, SkillFrontmatter, SkillEntry, SkillSource, SkillSourceConfig, SkillsEmbedOptions, SkillsEmbedResult, } from './lib/types';
+export type { DocProvider, DocFile, DocSection, VersionResult, PullResult, GitignoreStatus, IndexOptions, EmbedOptions, EmbedResult, ProviderPreset, UrlDocConfig, SkillFrontmatter, SkillEntry, SkillSource, SkillSourceConfig, SkillsEmbedOptions, SkillsEmbedResult, } from './lib/types';
 export type { GenericProviderOptions } from './lib/providers/generic';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAGH,OAAO,EACL,KAAK,EACL,QAAQ,EACR,eAAe,EACf,YAAY,EACZ,aAAa,EACb,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,oBAAoB,EACpB,oBAAoB,EACpB,iBAAiB,EACjB,gBAAgB,GACjB,MAAM,iBAAiB,CAAA;AAGxB,OAAO,EACL,cAAc,EACd,aAAa,EACb,YAAY,EACZ,oBAAoB,EACpB,aAAa,EACb,kBAAkB,EAClB,WAAW,EACX,cAAc,EACd,mBAAmB,EACnB,WAAW,EACX,aAAa,EACb,mBAAmB,GACpB,MAAM,iBAAiB,CAAA;AAGxB,OAAO,EACL,WAAW,EACX,gBAAgB,EAChB,qBAAqB,EACrB,oBAAoB,EACpB,kBAAkB,EAClB,mBAAmB,EACnB,iBAAiB,EACjB,sBAAsB,EACtB,iBAAiB,EACjB,sBAAsB,EACtB,uBAAuB,GACxB,MAAM,cAAc,CAAA;AAGrB,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AAC3D,YAAY,EAAE,WAAW,EAAE,MAAM,cAAc,CAAA;AAG/C,YAAY,EACV,WAAW,EACX,OAAO,EACP,UAAU,EACV,aAAa,EACb,UAAU,EACV,eAAe,EACf,YAAY,EACZ,YAAY,EACZ,WAAW,EACX,cAAc,EACd,gBAAgB,EAChB,UAAU,EACV,WAAW,EACX,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,GAClB,MAAM,aAAa,CAAA;AAEpB,YAAY,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAGH,OAAO,EACL,KAAK,EACL,QAAQ,EACR,eAAe,EACf,YAAY,EACZ,aAAa,EACb,WAAW,EACX,gBAAgB,EAChB,eAAe,EACf,oBAAoB,EACpB,oBAAoB,EACpB,iBAAiB,EACjB,gBAAgB,GACjB,MAAM,iBAAiB,CAAA;AAGxB,OAAO,EACL,cAAc,EACd,aAAa,EACb,YAAY,EACZ,oBAAoB,EACpB,aAAa,EACb,kBAAkB,EAClB,WAAW,EACX,gBAAgB,EAChB,cAAc,EACd,mBAAmB,EACnB,iBAAiB,EACjB,WAAW,EACX,aAAa,EACb,mBAAmB,GACpB,MAAM,iBAAiB,CAAA;AAGxB,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AAGnD,OAAO,EACL,WAAW,EACX,gBAAgB,EAChB,qBAAqB,EACrB,oBAAoB,EACpB,kBAAkB,EAClB,mBAAmB,EACnB,iBAAiB,EACjB,sBAAsB,EACtB,iBAAiB,EACjB,sBAAsB,EACtB,uBAAuB,GACxB,MAAM,cAAc,CAAA;AAGrB,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AAC3D,YAAY,EAAE,WAAW,EAAE,MAAM,cAAc,CAAA;AAG/C,YAAY,EACV,WAAW,EACX,OAAO,EACP,UAAU,EACV,aAAa,EACb,UAAU,EACV,eAAe,EACf,YAAY,EACZ,YAAY,EACZ,WAAW,EACX,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,UAAU,EACV,WAAW,EACX,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,GAClB,MAAM,aAAa,CAAA;AAEpB,YAAY,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAA"}

package/dist/index.js

@@ -35,14 +35,21 @@ import {
   reactProvider,
   removeDocsIndex,
   removeSkillsIndex,
-  tauriProvider
-} from "./index-thmt54kg.js";
+  tauriProvider,
+  tensorrtProvider
+} from "./index-cm3qmz9v.js";
+import {
+  createUrlProvider,
+  pullDocsFromUrl
+} from "./index-dtcewfnz.js";
 export {
+  tensorrtProvider,
   tauriProvider,
   removeSkillsIndex,
   removeDocsIndex,
   reactProvider,
   rattlerBuildProvider,
+  pullDocsFromUrl,
   pullDocs,
   pixiProvider,
   parseSkillFrontmatter,
@@ -68,6 +75,7 @@ export {
   embed,
   discoverPluginSkills,
   discoverFlatSkills,
+  createUrlProvider,
   createProvider,
   createLocalProvider,
   condaForgeProvider,

package/dist/lib/agents-md.d.ts

@@ -1,11 +1,12 @@
 import type { DocProvider, DocFile, DocSection, PullResult, GitignoreStatus, IndexOptions, EmbedOptions, EmbedResult } from './types';
 /**
- * Pull documentation from a GitHub repository
+ * Pull documentation from a GitHub repository or URL
  */
 export declare function pullDocs(provider: DocProvider, options: {
     cwd: string;
     version?: string;
     docsDir?: string;
+    onProgress?: (current: number, total: number, page: string) => void;
 }): Promise<PullResult>;
 /**
  * Collect all documentation files from a directory

package/dist/lib/agents-md.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"agents-md.d.ts","sourceRoot":"","sources":["../../src/lib/agents-md.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EACV,WAAW,EACX,OAAO,EACP,UAAU,EACV,UAAU,EACV,eAAe,EACf,YAAY,EACZ,YAAY,EACZ,WAAW,EACZ,MAAM,SAAS,CAAA;AAkBhB;;GAEG;AACH,wBAAsB,QAAQ,CAC5B,QAAQ,EAAE,WAAW,EACrB,OAAO,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,GAC3D,OAAO,CAAC,UAAU,CAAC,CAmDrB;AAiDD;;GAEG;AACH,wBAAgB,eAAe,CAC7B,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;IAAE,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,eAAe,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,GAC9D,OAAO,EAAE,CA4CX;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,UAAU,EAAE,CA8E3D;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM,CAkC3D;AAkCD;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAMhF;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAQ9D;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,MAAM,CAkD9E;AAWD;;;GAGG;AACH,wBAAgB,WAAW,CAAC,eAAe,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,MAAM,CAkBxG;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,eAAe,CAuBlF;AAED;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,MAAM,CAE1C;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAEpD;AAED;;GAEG;AACH,wBAAsB,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAiIvE;AAGD,YAAY,EACV,WAAW,EACX,OAAO,EACP,UAAU,EACV,UAAU,EACV,eAAe,EACf,YAAY,EACZ,YAAY,EACZ,WAAW,GACZ,MAAM,SAAS,CAAA"}
+{"version":3,"file":"agents-md.d.ts","sourceRoot":"","sources":["../../src/lib/agents-md.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EACV,WAAW,EACX,OAAO,EACP,UAAU,EACV,UAAU,EACV,eAAe,EACf,YAAY,EACZ,YAAY,EACZ,WAAW,EACZ,MAAM,SAAS,CAAA;AAkBhB;;GAEG;AACH,wBAAsB,QAAQ,CAC5B,QAAQ,EAAE,WAAW,EACrB,OAAO,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;CAAE,GAChI,OAAO,CAAC,UAAU,CAAC,CA0DrB;AAiDD;;GAEG;AACH,wBAAgB,eAAe,CAC7B,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;IAAE,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,eAAe,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,GAC9D,OAAO,EAAE,CA4CX;AAED;;GAEG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,UAAU,EAAE,CA8E3D;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,YAAY,GAAG,MAAM,CAkC3D;AAkCD;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,OAAO,CAMhF;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAQ9D;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,MAAM,CAkD9E;AAWD;;;GAGG;AACH,wBAAgB,WAAW,CAAC,eAAe,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,GAAG,MAAM,CAkBxG;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,eAAe,CAuBlF;AAED;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,MAAM,CAE1C;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAEpD;AAED;;GAEG;AACH,wBAAsB,KAAK,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAiIvE;AAGD,YAAY,EACV,WAAW,EACX,OAAO,EACP,UAAU,EACV,UAAU,EACV,eAAe,EACf,YAAY,EACZ,YAAY,EACZ,WAAW,GACZ,MAAM,SAAS,CAAA"}

package/dist/lib/providers/index.d.ts

@@ -23,7 +23,9 @@ export { obsidianExcalidrawProvider } from './obsidian-excalidraw';
 export { ffmpegProvider } from './ffmpeg';
 export { manimProvider } from './manim';
 export { cudaFeedstockProvider } from './cuda-feedstock';
+export { tensorrtProvider } from './tensorrt';
 export { createProvider, createLocalProvider } from './generic';
+export { createUrlProvider } from '../url-scraper';
 export type { GenericProviderOptions } from './generic';
 import type { DocProvider, ProviderPreset } from '../types';
 /**

package/dist/lib/providers/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/lib/providers/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AACvC,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAA;AACrC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAA;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AACvC,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAA;AAClD,OAAO,EAAE,WAAW,EAAE,MAAM,OAAO,CAAA;AACnC,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAC/C,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAA;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAA;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAA;AACrC,OAAO,EAAE,UAAU,EAAE,MAAM,MAAM,CAAA;AACjC,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAA;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AACzC,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAA;AAC7C,OAAO,EAAE,0BAA0B,EAAE,MAAM,uBAAuB,CAAA;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AACvC,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAA;AACxD,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,WAAW,CAAA;AAC/D,YAAY,EAAE,sBAAsB,EAAE,MAAM,WAAW,CAAA;AAwBvD,OAAO,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AAE3D;;GAEG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,cAAc,GAAG,WAAW,GAAG,IAAI,CAsDtE;AAED;;GAEG;AACH,wBAAgB,aAAa,IAAI,cAAc,EAAE,CAEhD;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAEnE"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/lib/providers/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AACvC,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAA;AACrC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAA;AACtD,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AACvC,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAA;AAClD,OAAO,EAAE,WAAW,EAAE,MAAM,OAAO,CAAA;AACnC,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAC/C,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAA;AACtD,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAA;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAA;AACrC,OAAO,EAAE,UAAU,EAAE,MAAM,MAAM,CAAA;AACjC,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAA;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AACzC,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAC5C,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAA;AAC7C,OAAO,EAAE,0BAA0B,EAAE,MAAM,uBAAuB,CAAA;AAClE,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AACvC,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAA;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAA;AAC7C,OAAO,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,WAAW,CAAA;AAC/D,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;AAClD,YAAY,EAAE,sBAAsB,EAAE,MAAM,WAAW,CAAA;AAyBvD,OAAO,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AAE3D;;GAEG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,cAAc,GAAG,WAAW,GAAG,IAAI,CAwDtE;AAED;;GAEG;AACH,wBAAgB,aAAa,IAAI,cAAc,EAAE,CAEhD;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAEnE"}

package/dist/lib/providers/tensorrt.d.ts

@@ -0,0 +1,9 @@
+/**
+ * NVIDIA TensorRT documentation provider
+ *
+ * Scrapes documentation from the NVIDIA TensorRT docs website
+ * and stores it as markdown files.
+ */
+import type { DocProvider } from '../types';
+export declare const tensorrtProvider: DocProvider;
+//# sourceMappingURL=tensorrt.d.ts.map

package/dist/lib/providers/tensorrt.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tensorrt.d.ts","sourceRoot":"","sources":["../../../src/lib/providers/tensorrt.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,UAAU,CAAA;AAE3C,eAAO,MAAM,gBAAgB,EAAE,WAoE9B,CAAA"}

package/dist/lib/types.d.ts

@@ -24,6 +24,25 @@ export interface GitignoreStatus {
     updated: boolean;
     alreadyPresent: boolean;
 }
+/**
+ * Configuration for scraping documentation from a URL
+ */
+export interface UrlDocConfig {
+    /** Base URL of the documentation site */
+    baseUrl: string;
+    /** CSS selector for the main content container (default: 'main') */
+    contentSelector?: string;
+    /** CSS selectors for elements to remove before extraction */
+    removeSelectors?: string[];
+    /** List of relative page paths to scrape (if not provided, will be discovered from the index page) */
+    pages?: string[];
+    /** Function to discover page URLs from an index page's HTML */
+    discoverPages?: (html: string, baseUrl: string) => string[];
+    /** Maximum number of concurrent fetches (default: 5) */
+    concurrency?: number;
+    /** Delay between fetches in ms (default: 200) */
+    fetchDelay?: number;
+}
 /**
  * Configuration for a documentation provider
  */
@@ -32,7 +51,7 @@ export interface DocProvider {
     name: string;
     /** Display name for CLI output */
     displayName: string;
-    /** GitHub repository in format 'owner/repo' */
+    /** GitHub repository in format 'owner/repo' (empty string for URL-based or local providers) */
     repo: string;
     /** Path to docs folder within the repository */
     docsPath: string;
@@ -48,11 +67,13 @@ export interface DocProvider {
     excludePatterns?: string[];
     /** Custom instruction to include in the index */
     instruction?: string;
+    /** URL-based documentation configuration (alternative to repo-based fetching) */
+    urlConfig?: UrlDocConfig;
 }
 /**
  * Built-in provider presets
  */
-export type ProviderPreset = 'nextjs' | 'react' | 'pixi' | 'rattler-build' | 'tauri' | 'conda-forge' | 'bun' | 'vue' | 'svelte' | 'sveltekit' | 'shadcn-svelte' | 'astro' | 'tailwind' | 'ruff' | 'ty' | 'basedpyright' | 'convex' | 'polars' | 'delta-rs' | 'obsidian' | 'obsidian-excalidraw' | 'ffmpeg' | 'manim' | 'cuda-feedstock';
+export type ProviderPreset = 'nextjs' | 'react' | 'pixi' | 'rattler-build' | 'tauri' | 'conda-forge' | 'bun' | 'vue' | 'svelte' | 'sveltekit' | 'shadcn-svelte' | 'astro' | 'tailwind' | 'ruff' | 'ty' | 'basedpyright' | 'convex' | 'polars' | 'delta-rs' | 'obsidian' | 'obsidian-excalidraw' | 'ffmpeg' | 'manim' | 'cuda-feedstock' | 'tensorrt';
 export interface IndexOptions {
     /** Path where docs are stored */
     docsPath: string;

package/dist/lib/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,OAAO;IACtB,YAAY,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,OAAO,EAAE,CAAA;IAChB,WAAW,EAAE,UAAU,EAAE,CAAA;CAC1B;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,OAAO,CAAA;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,OAAO,CAAA;IAChB,cAAc,EAAE,OAAO,CAAA;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,2EAA2E;IAC3E,IAAI,EAAE,MAAM,CAAA;IAEZ,kCAAkC;IAClC,WAAW,EAAE,MAAM,CAAA;IAEnB,+CAA+C;IAC/C,IAAI,EAAE,MAAM,CAAA;IAEZ,gDAAgD;IAChD,QAAQ,EAAE,MAAM,CAAA;IAEhB,4DAA4D;IAC5D,UAAU,CAAC,EAAE,MAAM,EAAE,CAAA;IAErB,0DAA0D;IAC1D,aAAa,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,aAAa,CAAA;IAE9C,2EAA2E;IAC3E,aAAa,CAAC,EAAE,MAAM,CAAA;IAEtB,iEAAiE;IACjE,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,MAAM,CAAA;IAE1C,kDAAkD;IAClD,eAAe,CAAC,EAAE,MAAM,EAAE,CAAA;IAE1B,iDAAiD;IACjD,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,eAAe,GAAG,OAAO,GAAG,aAAa,GAAG,KAAK,GAAG,KAAK,GAAG,QAAQ,GAAG,WAAW,GAAG,eAAe,GAAG,OAAO,GAAG,UAAU,GAAG,MAAM,GAAG,IAAI,GAAG,cAAc,GAAG,QAAQ,GAAG,QAAQ,GAAG,UAAU,GAAG,UAAU,GAAG,qBAAqB,GAAG,QAAQ,GAAG,OAAO,GAAG,gBAAgB,CAAA;AAEvU,MAAM,WAAW,YAAY;IAC3B,iCAAiC;IACjC,QAAQ,EAAE,MAAM,CAAA;IAEhB,gCAAgC;IAChC,QAAQ,EAAE,UAAU,EAAE,CAAA;IAEtB,8BAA8B;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAA;IAEnB,wCAAwC;IACxC,YAAY,CAAC,EAAE,MAAM,CAAA;IAErB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAA;IAEpB,2CAA2C;IAC3C,WAAW,CAAC,EAAE,MAAM,CAAA;IAEpB,qCAAqC;IACrC,iBAAiB,CAAC,EAAE,MAAM,CAAA;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,wBAAwB;IACxB,GAAG,EAAE,MAAM,CAAA;IAEX,2CAA2C;IAC3C,QAAQ,EAAE,WAAW,CAAA;IAErB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAA;IAEhB,+CAA+C;IAC/C,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAA;IAEhB,oFAAoF;IACpF,WAAW,CAAC,EAAE,OAAO,CAAA;IAErB,mEAAmE;IACnE,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,OAAO,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,SAAS,CAAC,EAAE,OAAO,CAAA;IACnB,gBAAgB,CAAC,EAAE,OAAO,CAAA;IAC1B,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAID,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,EAAE,CAAA;IACtB,MAAM,EAAE,WAAW,CAAA;IACnB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,MAAM,GAAG,SAAS,GAAG,WAAW,CAAA;AAErE,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,WAAW,CAAA;IACjB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,MAAM,CAAA;IACX,OAAO,EAAE,iBAAiB,EAAE,CAAA;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,SAAS,CAAC,EAAE,OAAO,CAAA;IACnB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,eAAe,CAAC,EAAE,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,CAAA;CAC9C;AAED,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAA;IACV,OAAO,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,MAAM,CAAA;IACb,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,oBAAoB,EAAE,CAAA;IAC9B,KAAK,EAAE,MAAM,CAAA;IACb,WAAW,EAAE,MAAM,CAAA;CACpB"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/lib/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,MAAM,WAAW,OAAO;IACtB,YAAY,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,OAAO,EAAE,CAAA;IAChB,WAAW,EAAE,UAAU,EAAE,CAAA;CAC1B;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,OAAO,CAAA;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,OAAO,CAAA;IAChB,cAAc,EAAE,OAAO,CAAA;CACxB;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,yCAAyC;IACzC,OAAO,EAAE,MAAM,CAAA;IAEf,oEAAoE;IACpE,eAAe,CAAC,EAAE,MAAM,CAAA;IAExB,6DAA6D;IAC7D,eAAe,CAAC,EAAE,MAAM,EAAE,CAAA;IAE1B,sGAAsG;IACtG,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;IAEhB,+DAA+D;IAC/D,aAAa,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,KAAK,MAAM,EAAE,CAAA;IAE3D,wDAAwD;IACxD,WAAW,CAAC,EAAE,MAAM,CAAA;IAEpB,iDAAiD;IACjD,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,2EAA2E;IAC3E,IAAI,EAAE,MAAM,CAAA;IAEZ,kCAAkC;IAClC,WAAW,EAAE,MAAM,CAAA;IAEnB,+FAA+F;IAC/F,IAAI,EAAE,MAAM,CAAA;IAEZ,gDAAgD;IAChD,QAAQ,EAAE,MAAM,CAAA;IAEhB,4DAA4D;IAC5D,UAAU,CAAC,EAAE,MAAM,EAAE,CAAA;IAErB,0DAA0D;IAC1D,aAAa,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,aAAa,CAAA;IAE9C,2EAA2E;IAC3E,aAAa,CAAC,EAAE,MAAM,CAAA;IAEtB,iEAAiE;IACjE,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,MAAM,CAAA;IAE1C,kDAAkD;IAClD,eAAe,CAAC,EAAE,MAAM,EAAE,CAAA;IAE1B,iDAAiD;IACjD,WAAW,CAAC,EAAE,MAAM,CAAA;IAEpB,iFAAiF;IACjF,SAAS,CAAC,EAAE,YAAY,CAAA;CACzB;AAED;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,OAAO,GAAG,MAAM,GAAG,eAAe,GAAG,OAAO,GAAG,aAAa,GAAG,KAAK,GAAG,KAAK,GAAG,QAAQ,GAAG,WAAW,GAAG,eAAe,GAAG,OAAO,GAAG,UAAU,GAAG,MAAM,GAAG,IAAI,GAAG,cAAc,GAAG,QAAQ,GAAG,QAAQ,GAAG,UAAU,GAAG,UAAU,GAAG,qBAAqB,GAAG,QAAQ,GAAG,OAAO,GAAG,gBAAgB,GAAG,UAAU,CAAA;AAEpV,MAAM,WAAW,YAAY;IAC3B,iCAAiC;IACjC,QAAQ,EAAE,MAAM,CAAA;IAEhB,gCAAgC;IAChC,QAAQ,EAAE,UAAU,EAAE,CAAA;IAEtB,8BAA8B;IAC9B,UAAU,CAAC,EAAE,MAAM,CAAA;IAEnB,wCAAwC;IACxC,YAAY,CAAC,EAAE,MAAM,CAAA;IAErB,oCAAoC;IACpC,WAAW,CAAC,EAAE,MAAM,CAAA;IAEpB,2CAA2C;IAC3C,WAAW,CAAC,EAAE,MAAM,CAAA;IAEpB,qCAAqC;IACrC,iBAAiB,CAAC,EAAE,MAAM,CAAA;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,wBAAwB;IACxB,GAAG,EAAE,MAAM,CAAA;IAEX,2CAA2C;IAC3C,QAAQ,EAAE,WAAW,CAAA;IAErB,iCAAiC;IACjC,OAAO,CAAC,EAAE,MAAM,CAAA;IAEhB,+CAA+C;IAC/C,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAA;IAEhB,oFAAoF;IACpF,WAAW,CAAC,EAAE,OAAO,CAAA;IAErB,mEAAmE;IACnE,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,OAAO,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,SAAS,CAAC,EAAE,OAAO,CAAA;IACnB,gBAAgB,CAAC,EAAE,OAAO,CAAA;IAC1B,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAID,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,EAAE,CAAA;IACtB,MAAM,EAAE,WAAW,CAAA;IACnB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,MAAM,GAAG,SAAS,GAAG,WAAW,CAAA;AAErE,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,WAAW,CAAA;IACjB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,MAAM,CAAA;IACX,OAAO,EAAE,iBAAiB,EAAE,CAAA;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAA;IAChB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,SAAS,CAAC,EAAE,OAAO,CAAA;IACnB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,eAAe,CAAC,EAAE,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,CAAA;CAC9C;AAED,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAA;IACV,OAAO,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,sBAAsB;IACrC,KAAK,EAAE,MAAM,CAAA;IACb,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,oBAAoB,EAAE,CAAA;IAC9B,KAAK,EAAE,MAAM,CAAA;IACb,WAAW,EAAE,MAAM,CAAA;CACpB"}

package/dist/lib/url-scraper.d.ts

@@ -0,0 +1,19 @@
+import type { UrlDocConfig, PullResult } from './types';
+/**
+ * Scrape documentation from a URL and save as markdown files
+ */
+export declare function pullDocsFromUrl(config: UrlDocConfig, destDir: string, options?: {
+    onProgress?: (current: number, total: number, page: string) => void;
+}): Promise<PullResult>;
+/**
+ * Create a URL-based documentation provider
+ */
+export declare function createUrlProvider(options: {
+    name: string;
+    displayName: string;
+    urlConfig: UrlDocConfig;
+    extensions?: string[];
+    excludePatterns?: string[];
+    instruction?: string;
+}): import('./types').DocProvider;
+//# sourceMappingURL=url-scraper.d.ts.map

package/dist/lib/url-scraper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"url-scraper.d.ts","sourceRoot":"","sources":["../../src/lib/url-scraper.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAwLvD;;GAEG;AACH,wBAAsB,eAAe,CACnC,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,MAAM,EACf,OAAO,CAAC,EAAE;IAAE,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;CAAE,GAChF,OAAO,CAAC,UAAU,CAAC,CAkIrB;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE;IACzC,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;IACnB,SAAS,EAAE,YAAY,CAAA;IACvB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAA;IACrB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAA;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB,GAAG,OAAO,SAAS,EAAE,WAAW,CAahC"}

package/dist/url-scraper-5sj8c56t.js

@@ -0,0 +1,8 @@
+import {
+  createUrlProvider,
+  pullDocsFromUrl
+} from "./index-dtcewfnz.js";
+export {
+  pullDocsFromUrl,
+  createUrlProvider
+};

package/dist/url-scraper-9xq4d56t.js

@@ -0,0 +1,8 @@
+import {
+  createUrlProvider,
+  pullDocsFromUrl
+} from "./index-sh9kr6eq.js";
+export {
+  pullDocsFromUrl,
+  createUrlProvider
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agdex",
-  "version": "0.5.1",
+  "version": "0.6.0",
   "description": "Embed compressed documentation indexes into AGENTS.md/CLAUDE.md for AI coding agents",
   "module": "dist/index.js",
   "main": "dist/index.js",
@@ -27,12 +27,15 @@
   },
   "dependencies": {
     "commander": "^12.1.0",
+    "node-html-parser": "^7.1.0",
     "picocolors": "^1.1.1",
-    "prompts": "^2.4.2"
+    "prompts": "^2.4.2",
+    "turndown": "^7.2.2"
   },
   "devDependencies": {
     "@types/bun": "latest",
-    "@types/prompts": "^2.4.9"
+    "@types/prompts": "^2.4.9",
+    "@types/turndown": "^5.0.6"
   },
   "peerDependencies": {
     "typescript": "^5"