@arabold/docs-mcp-server 1.14.0 → 1.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +84 -28
- package/dist/{EmbeddingFactory-Dz1hdJJe.js → EmbeddingFactory-C6_OpOiy.js} +2 -2
- package/dist/{EmbeddingFactory-Dz1hdJJe.js.map → EmbeddingFactory-C6_OpOiy.js.map} +1 -1
- package/dist/assets/main.css +1 -1
- package/dist/index.js +6047 -0
- package/dist/index.js.map +1 -0
- package/package.json +45 -45
- package/public/assets/main.css +1 -1
- package/dist/DocumentManagementService-BZ_ZZgPI.js +0 -3483
- package/dist/DocumentManagementService-BZ_ZZgPI.js.map +0 -1
- package/dist/FindVersionTool-Dfw5Lbql.js +0 -140
- package/dist/FindVersionTool-Dfw5Lbql.js.map +0 -1
- package/dist/RemoveTool-w8KGOaXw.js +0 -65
- package/dist/RemoveTool-w8KGOaXw.js.map +0 -1
- package/dist/cli.js +0 -236
- package/dist/cli.js.map +0 -1
- package/dist/server.js +0 -769
- package/dist/server.js.map +0 -1
- package/dist/web.js +0 -956
- package/dist/web.js.map +0 -1
|
@@ -1,140 +0,0 @@
|
|
|
1
|
-
import { j as ScrapeMode, T as ToolError, k as HtmlPipeline, M as MarkdownPipeline, l as logger, m as ScraperError, V as VersionNotFoundError } from "./DocumentManagementService-BZ_ZZgPI.js";
|
|
2
|
-
class FetchUrlTool {
|
|
3
|
-
/**
|
|
4
|
-
* Collection of fetchers that will be tried in order for a given URL.
|
|
5
|
-
*/
|
|
6
|
-
fetchers;
|
|
7
|
-
constructor(httpFetcher, fileFetcher) {
|
|
8
|
-
this.fetchers = [httpFetcher, fileFetcher];
|
|
9
|
-
}
|
|
10
|
-
/**
|
|
11
|
-
* Fetches content from a URL and converts it to Markdown.
|
|
12
|
-
* Supports both HTTP/HTTPS URLs and local file URLs (file://).
|
|
13
|
-
* @returns The processed Markdown content
|
|
14
|
-
* @throws {ToolError} If fetching or processing fails
|
|
15
|
-
*/
|
|
16
|
-
async execute(options) {
|
|
17
|
-
const { url, scrapeMode = ScrapeMode.Auto } = options;
|
|
18
|
-
const canFetchResults = this.fetchers.map((f) => f.canFetch(url));
|
|
19
|
-
const fetcherIndex = canFetchResults.findIndex((result) => result === true);
|
|
20
|
-
if (fetcherIndex === -1) {
|
|
21
|
-
throw new ToolError(
|
|
22
|
-
`Invalid URL: ${url}. Must be an HTTP/HTTPS URL or a file:// URL.`,
|
|
23
|
-
this.constructor.name
|
|
24
|
-
);
|
|
25
|
-
}
|
|
26
|
-
const fetcher = this.fetchers[fetcherIndex];
|
|
27
|
-
const htmlPipeline = new HtmlPipeline();
|
|
28
|
-
const markdownPipeline = new MarkdownPipeline();
|
|
29
|
-
const pipelines = [htmlPipeline, markdownPipeline];
|
|
30
|
-
try {
|
|
31
|
-
logger.info(`📡 Fetching ${url}...`);
|
|
32
|
-
const rawContent = await fetcher.fetch(url, {
|
|
33
|
-
followRedirects: options.followRedirects ?? true,
|
|
34
|
-
maxRetries: 3
|
|
35
|
-
});
|
|
36
|
-
logger.info("🔄 Processing content...");
|
|
37
|
-
let processed;
|
|
38
|
-
for (const pipeline of pipelines) {
|
|
39
|
-
if (pipeline.canProcess(rawContent)) {
|
|
40
|
-
processed = await pipeline.process(
|
|
41
|
-
rawContent,
|
|
42
|
-
{
|
|
43
|
-
url,
|
|
44
|
-
library: "",
|
|
45
|
-
version: "",
|
|
46
|
-
maxDepth: 0,
|
|
47
|
-
maxPages: 1,
|
|
48
|
-
maxConcurrency: 1,
|
|
49
|
-
scope: "subpages",
|
|
50
|
-
followRedirects: options.followRedirects ?? true,
|
|
51
|
-
excludeSelectors: void 0,
|
|
52
|
-
ignoreErrors: false,
|
|
53
|
-
scrapeMode
|
|
54
|
-
},
|
|
55
|
-
fetcher
|
|
56
|
-
);
|
|
57
|
-
break;
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
if (!processed) {
|
|
61
|
-
logger.warn(
|
|
62
|
-
`Unsupported content type "${rawContent.mimeType}" for ${url}. Returning raw content.`
|
|
63
|
-
);
|
|
64
|
-
const contentString = typeof rawContent.content === "string" ? rawContent.content : Buffer.from(rawContent.content).toString("utf-8");
|
|
65
|
-
return contentString;
|
|
66
|
-
}
|
|
67
|
-
for (const err of processed.errors) {
|
|
68
|
-
logger.warn(`Processing error for ${url}: ${err.message}`);
|
|
69
|
-
}
|
|
70
|
-
if (typeof processed.textContent !== "string" || !processed.textContent.trim()) {
|
|
71
|
-
throw new ToolError(
|
|
72
|
-
`Processing resulted in empty content for ${url}`,
|
|
73
|
-
this.constructor.name
|
|
74
|
-
);
|
|
75
|
-
}
|
|
76
|
-
logger.info(`✅ Successfully processed ${url}`);
|
|
77
|
-
return processed.textContent;
|
|
78
|
-
} catch (error) {
|
|
79
|
-
if (error instanceof ScraperError || error instanceof ToolError) {
|
|
80
|
-
throw new ToolError(
|
|
81
|
-
`Failed to fetch or process URL: ${error.message}`,
|
|
82
|
-
this.constructor.name
|
|
83
|
-
);
|
|
84
|
-
}
|
|
85
|
-
throw new ToolError(
|
|
86
|
-
`Failed to fetch or process URL: ${error instanceof Error ? error.message : String(error)}`,
|
|
87
|
-
this.constructor.name
|
|
88
|
-
);
|
|
89
|
-
} finally {
|
|
90
|
-
await htmlPipeline.close();
|
|
91
|
-
await markdownPipeline.close();
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
class FindVersionTool {
|
|
96
|
-
docService;
|
|
97
|
-
constructor(docService) {
|
|
98
|
-
this.docService = docService;
|
|
99
|
-
}
|
|
100
|
-
/**
|
|
101
|
-
* Executes the tool to find the best matching version and checks for unversioned docs.
|
|
102
|
-
* @returns A descriptive string indicating the best match and unversioned status, or an error message.
|
|
103
|
-
*/
|
|
104
|
-
async execute(options) {
|
|
105
|
-
const { library, targetVersion } = options;
|
|
106
|
-
const targetVersionString = targetVersion ? `@${targetVersion}` : "";
|
|
107
|
-
try {
|
|
108
|
-
const { bestMatch, hasUnversioned } = await this.docService.findBestVersion(
|
|
109
|
-
library,
|
|
110
|
-
targetVersion
|
|
111
|
-
);
|
|
112
|
-
let message = "";
|
|
113
|
-
if (bestMatch) {
|
|
114
|
-
message = `Best match: ${bestMatch}.`;
|
|
115
|
-
if (hasUnversioned) {
|
|
116
|
-
message += " Unversioned docs also available.";
|
|
117
|
-
}
|
|
118
|
-
} else if (hasUnversioned) {
|
|
119
|
-
message = `No matching version found for ${library}${targetVersionString}, but unversioned docs exist.`;
|
|
120
|
-
} else {
|
|
121
|
-
message = `No matching version or unversioned documents found for ${library}${targetVersionString}.`;
|
|
122
|
-
}
|
|
123
|
-
return message;
|
|
124
|
-
} catch (error) {
|
|
125
|
-
if (error instanceof VersionNotFoundError) {
|
|
126
|
-
logger.info(`ℹ️ Version not found: ${error.message}`);
|
|
127
|
-
return `No matching version or unversioned documents found for ${library}${targetVersionString}. Available: ${error.availableVersions.length > 0 ? error.availableVersions.map((v) => v.version).join(", ") : "None"}.`;
|
|
128
|
-
}
|
|
129
|
-
logger.error(
|
|
130
|
-
`❌ Error finding version for ${library}${targetVersionString}: ${error instanceof Error ? error.message : error}`
|
|
131
|
-
);
|
|
132
|
-
throw error;
|
|
133
|
-
}
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
export {
|
|
137
|
-
FetchUrlTool as F,
|
|
138
|
-
FindVersionTool as a
|
|
139
|
-
};
|
|
140
|
-
//# sourceMappingURL=FindVersionTool-Dfw5Lbql.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"FindVersionTool-Dfw5Lbql.js","sources":["../src/tools/FetchUrlTool.ts","../src/tools/FindVersionTool.ts"],"sourcesContent":["import type {\n ContentFetcher,\n FileFetcher,\n HttpFetcher,\n RawContent,\n} from \"../scraper/fetcher\";\nimport { HtmlPipeline } from \"../scraper/pipelines/HtmlPipeline\";\nimport { MarkdownPipeline } from \"../scraper/pipelines/MarkdownPipeline\";\nimport { ScrapeMode } from \"../scraper/types\";\nimport { ScraperError } from \"../utils/errors\";\nimport { logger } from \"../utils/logger\";\nimport { ToolError } from \"./errors\";\n\nexport interface FetchUrlToolOptions {\n /**\n * The URL to fetch and convert to markdown.\n * Must be a valid HTTP/HTTPS URL or file:// URL.\n */\n url: string;\n\n /**\n * Whether to follow HTTP redirects.\n * @default true\n */\n followRedirects?: boolean;\n\n /**\n * Determines the HTML processing strategy.\n * - 'fetch': Use a simple DOM parser (faster, less JS support).\n * - 'playwright': Use a headless browser (slower, full JS support).\n * - 'auto': Automatically select the best strategy (currently defaults to 'playwright').\n * @default ScrapeMode.Auto\n */\n scrapeMode?: ScrapeMode;\n}\n\n/**\n * Tool for fetching a single URL and converting its content to Markdown.\n * Unlike scrape_docs, this tool only processes one page without crawling\n * or storing the content.\n *\n * Supports both HTTP/HTTPS URLs and local file URLs (file://).\n */\nexport class FetchUrlTool {\n /**\n * Collection of fetchers that will be tried in order for a given URL.\n */\n private readonly fetchers: ContentFetcher[];\n\n constructor(httpFetcher: HttpFetcher, fileFetcher: FileFetcher) {\n this.fetchers = [httpFetcher, fileFetcher];\n }\n\n /**\n * Fetches content from a URL and converts it to Markdown.\n * Supports both HTTP/HTTPS URLs and local file URLs (file://).\n * @returns The processed Markdown content\n * @throws {ToolError} If fetching or processing fails\n */\n async execute(options: FetchUrlToolOptions): Promise<string> {\n const { url, scrapeMode = ScrapeMode.Auto } = options;\n\n const canFetchResults = this.fetchers.map((f) => f.canFetch(url));\n const fetcherIndex = canFetchResults.findIndex((result) => result === true);\n if (fetcherIndex === -1) {\n throw new ToolError(\n `Invalid URL: ${url}. Must be an HTTP/HTTPS URL or a file:// URL.`,\n this.constructor.name,\n );\n }\n\n const fetcher = this.fetchers[fetcherIndex];\n const htmlPipeline = new HtmlPipeline();\n const markdownPipeline = new MarkdownPipeline();\n const pipelines = [htmlPipeline, markdownPipeline];\n\n try {\n logger.info(`📡 Fetching ${url}...`);\n const rawContent: RawContent = await fetcher.fetch(url, {\n followRedirects: options.followRedirects ?? true,\n maxRetries: 3,\n });\n\n logger.info(\"🔄 Processing content...\");\n\n let processed: Awaited<ReturnType<(typeof htmlPipeline)[\"process\"]>> | undefined;\n for (const pipeline of pipelines) {\n if (pipeline.canProcess(rawContent)) {\n processed = await pipeline.process(\n rawContent,\n {\n url,\n library: \"\",\n version: \"\",\n maxDepth: 0,\n maxPages: 1,\n maxConcurrency: 1,\n scope: \"subpages\",\n followRedirects: options.followRedirects ?? true,\n excludeSelectors: undefined,\n ignoreErrors: false,\n scrapeMode,\n },\n fetcher,\n );\n break;\n }\n }\n\n if (!processed) {\n logger.warn(\n `Unsupported content type \"${rawContent.mimeType}\" for ${url}. Returning raw content.`,\n );\n const contentString =\n typeof rawContent.content === \"string\"\n ? rawContent.content\n : Buffer.from(rawContent.content).toString(\"utf-8\");\n return contentString;\n }\n\n for (const err of processed.errors) {\n logger.warn(`Processing error for ${url}: ${err.message}`);\n }\n\n if (typeof processed.textContent !== \"string\" || !processed.textContent.trim()) {\n throw new ToolError(\n `Processing resulted in empty content for ${url}`,\n this.constructor.name,\n );\n }\n\n logger.info(`✅ Successfully processed ${url}`);\n return processed.textContent;\n } catch (error) {\n if (error instanceof ScraperError || error instanceof ToolError) {\n throw new ToolError(\n `Failed to fetch or process URL: ${error.message}`,\n this.constructor.name,\n );\n }\n throw new ToolError(\n `Failed to fetch or process URL: ${error instanceof Error ? error.message : String(error)}`,\n this.constructor.name,\n );\n } finally {\n await htmlPipeline.close();\n await markdownPipeline.close();\n }\n }\n}\n","import type { DocumentManagementService } from \"../store\";\nimport { logger } from \"../utils/logger\";\nimport { VersionNotFoundError } from \"./errors\";\n\nexport interface FindVersionToolOptions {\n library: string;\n targetVersion?: string;\n}\n\n/**\n * Tool for finding the best matching version of a library in the store.\n * Supports exact version matches and X-Range patterns (e.g., '5.x', '5.2.x').\n */\nexport class FindVersionTool {\n private docService: DocumentManagementService;\n\n constructor(docService: DocumentManagementService) {\n this.docService = docService;\n }\n\n /**\n * Executes the tool to find the best matching version and checks for unversioned docs.\n * @returns A descriptive string indicating the best match and unversioned status, or an error message.\n */\n async execute(options: FindVersionToolOptions): Promise<string> {\n const { library, targetVersion } = options;\n const targetVersionString = targetVersion ? `@${targetVersion}` : \"\";\n\n try {\n const { bestMatch, hasUnversioned } = await this.docService.findBestVersion(\n library,\n targetVersion,\n );\n\n let message = \"\";\n if (bestMatch) {\n message = `Best match: ${bestMatch}.`;\n if (hasUnversioned) {\n message += \" Unversioned docs also available.\";\n }\n } else if (hasUnversioned) {\n message = `No matching version found for ${library}${targetVersionString}, but unversioned docs exist.`;\n } else {\n // This case should ideally be caught by VersionNotFoundError below,\n // but added for completeness.\n message = `No matching version or unversioned documents found for ${library}${targetVersionString}.`;\n }\n return message;\n } catch (error) {\n if (error instanceof VersionNotFoundError) {\n // This error is thrown when no semver versions AND no unversioned docs exist.\n logger.info(`ℹ️ Version not found: ${error.message}`);\n return `No matching version or unversioned documents found for ${library}${targetVersionString}. Available: ${\n error.availableVersions.length > 0\n ? error.availableVersions.map((v) => v.version).join(\", \")\n : \"None\"\n }.`;\n }\n // Re-throw unexpected errors\n logger.error(\n `❌ Error finding version for ${library}${targetVersionString}: ${error instanceof Error ? error.message : error}`,\n );\n throw error;\n }\n }\n}\n"],"names":[],"mappings":";AA2CO,MAAM,aAAa;AAAA;AAAA;AAAA;AAAA,EAIP;AAAA,EAEjB,YAAY,aAA0B,aAA0B;AACzD,SAAA,WAAW,CAAC,aAAa,WAAW;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAS3C,MAAM,QAAQ,SAA+C;AAC3D,UAAM,EAAE,KAAK,aAAa,WAAW,KAAS,IAAA;AAExC,UAAA,kBAAkB,KAAK,SAAS,IAAI,CAAC,MAAM,EAAE,SAAS,GAAG,CAAC;AAChE,UAAM,eAAe,gBAAgB,UAAU,CAAC,WAAW,WAAW,IAAI;AAC1E,QAAI,iBAAiB,IAAI;AACvB,YAAM,IAAI;AAAA,QACR,gBAAgB,GAAG;AAAA,QACnB,KAAK,YAAY;AAAA,MACnB;AAAA,IAAA;AAGI,UAAA,UAAU,KAAK,SAAS,YAAY;AACpC,UAAA,eAAe,IAAI,aAAa;AAChC,UAAA,mBAAmB,IAAI,iBAAiB;AACxC,UAAA,YAAY,CAAC,cAAc,gBAAgB;AAE7C,QAAA;AACK,aAAA,KAAK,eAAe,GAAG,KAAK;AACnC,YAAM,aAAyB,MAAM,QAAQ,MAAM,KAAK;AAAA,QACtD,iBAAiB,QAAQ,mBAAmB;AAAA,QAC5C,YAAY;AAAA,MAAA,CACb;AAED,aAAO,KAAK,0BAA0B;AAElC,UAAA;AACJ,iBAAW,YAAY,WAAW;AAC5B,YAAA,SAAS,WAAW,UAAU,GAAG;AACnC,sBAAY,MAAM,SAAS;AAAA,YACzB;AAAA,YACA;AAAA,cACE;AAAA,cACA,SAAS;AAAA,cACT,SAAS;AAAA,cACT,UAAU;AAAA,cACV,UAAU;AAAA,cACV,gBAAgB;AAAA,cAChB,OAAO;AAAA,cACP,iBAAiB,QAAQ,mBAAmB;AAAA,cAC5C,kBAAkB;AAAA,cAClB,cAAc;AAAA,cACd;AAAA,YACF;AAAA,YACA;AAAA,UACF;AACA;AAAA,QAAA;AAAA,MACF;AAGF,UAAI,CAAC,WAAW;AACP,eAAA;AAAA,UACL,6BAA6B,WAAW,QAAQ,SAAS,GAAG;AAAA,QAC9D;AACA,cAAM,gBACJ,OAAO,WAAW,YAAY,WAC1B,WAAW,UACX,OAAO,KAAK,WAAW,OAAO,EAAE,SAAS,OAAO;AAC/C,eAAA;AAAA,MAAA;AAGE,iBAAA,OAAO,UAAU,QAAQ;AAClC,eAAO,KAAK,wBAAwB,GAAG,KAAK,IAAI,OAAO,EAAE;AAAA,MAAA;AAGvD,UAAA,OAAO,UAAU,gBAAgB,YAAY,CAAC,UAAU,YAAY,QAAQ;AAC9E,cAAM,IAAI;AAAA,UACR,4CAA4C,GAAG;AAAA,UAC/C,KAAK,YAAY;AAAA,QACnB;AAAA,MAAA;AAGK,aAAA,KAAK,4BAA4B,GAAG,EAAE;AAC7C,aAAO,UAAU;AAAA,aACV,OAAO;AACV,UAAA,iBAAiB,gBAAgB,iBAAiB,WAAW;AAC/D,cAAM,IAAI;AAAA,UACR,mCAAmC,MAAM,OAAO;AAAA,UAChD,KAAK,YAAY;AAAA,QACnB;AAAA,MAAA;AAEF,YAAM,IAAI;AAAA,QACR,mCAAmC,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,CAAC;AAAA,QACzF,KAAK,YAAY;AAAA,MACnB;AAAA,IAAA,UACA;AACA,YAAM,aAAa,MAAM;AACzB,YAAM,iBAAiB,MAAM;AAAA,IAAA;AAAA,EAC/B;AAEJ;ACxIO,MAAM,gBAAgB;AAAA,EACnB;AAAA,EAER,YAAY,YAAuC;AACjD,SAAK,aAAa;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOpB,MAAM,QAAQ,SAAkD;AACxD,UAAA,EAAE,SAAS,cAAA,IAAkB;AACnC,UAAM,sBAAsB,gBAAgB,IAAI,aAAa,KAAK;AAE9D,QAAA;AACF,YAAM,EAAE,WAAW,eAAA,IAAmB,MAAM,KAAK,WAAW;AAAA,QAC1D;AAAA,QACA;AAAA,MACF;AAEA,UAAI,UAAU;AACd,UAAI,WAAW;AACb,kBAAU,eAAe,SAAS;AAClC,YAAI,gBAAgB;AACP,qBAAA;AAAA,QAAA;AAAA,iBAEJ,gBAAgB;AACf,kBAAA,iCAAiC,OAAO,GAAG,mBAAmB;AAAA,MAAA,OACnE;AAGK,kBAAA,0DAA0D,OAAO,GAAG,mBAAmB;AAAA,MAAA;AAE5F,aAAA;AAAA,aACA,OAAO;AACd,UAAI,iBAAiB,sBAAsB;AAEzC,eAAO,KAAK,yBAAyB,MAAM,OAAO,EAAE;AAC7C,eAAA,0DAA0D,OAAO,GAAG,mBAAmB,gBAC5F,MAAM,kBAAkB,SAAS,IAC7B,MAAM,kBAAkB,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,IAAI,IACvD,MACN;AAAA,MAAA;AAGK,aAAA;AAAA,QACL,+BAA+B,OAAO,GAAG,mBAAmB,KAAK,iBAAiB,QAAQ,MAAM,UAAU,KAAK;AAAA,MACjH;AACM,YAAA;AAAA,IAAA;AAAA,EACR;AAEJ;"}
|
|
@@ -1,65 +0,0 @@
|
|
|
1
|
-
import { l as logger, T as ToolError } from "./DocumentManagementService-BZ_ZZgPI.js";
|
|
2
|
-
class ListJobsTool {
|
|
3
|
-
manager;
|
|
4
|
-
// Change property name and type
|
|
5
|
-
/**
|
|
6
|
-
* Creates an instance of ListJobsTool.
|
|
7
|
-
* @param manager The PipelineManager instance.
|
|
8
|
-
*/
|
|
9
|
-
constructor(manager) {
|
|
10
|
-
this.manager = manager;
|
|
11
|
-
}
|
|
12
|
-
/**
|
|
13
|
-
* Executes the tool to retrieve a list of pipeline jobs.
|
|
14
|
-
* @param input - The input parameters, optionally including a status filter.
|
|
15
|
-
* @returns A promise that resolves with the list of simplified job objects.
|
|
16
|
-
* @throws {PipelineStateError} If the pipeline manager is somehow unavailable.
|
|
17
|
-
*/
|
|
18
|
-
async execute(input) {
|
|
19
|
-
const jobs = await this.manager.getJobs(input.status);
|
|
20
|
-
const simplifiedJobs = jobs.map(
|
|
21
|
-
(job) => ({
|
|
22
|
-
id: job.id,
|
|
23
|
-
library: job.library,
|
|
24
|
-
version: job.version,
|
|
25
|
-
status: job.status,
|
|
26
|
-
createdAt: job.createdAt.toISOString(),
|
|
27
|
-
startedAt: job.startedAt?.toISOString() ?? null,
|
|
28
|
-
finishedAt: job.finishedAt?.toISOString() ?? null,
|
|
29
|
-
error: job.error?.message ?? null
|
|
30
|
-
})
|
|
31
|
-
);
|
|
32
|
-
return { jobs: simplifiedJobs };
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
class RemoveTool {
|
|
36
|
-
constructor(documentManagementService) {
|
|
37
|
-
this.documentManagementService = documentManagementService;
|
|
38
|
-
}
|
|
39
|
-
/**
|
|
40
|
-
* Executes the tool to remove the specified library version documents.
|
|
41
|
-
* Assumes args have been validated by the caller (McpServer) against inputSchema.
|
|
42
|
-
* Returns a simple success message or throws an error.
|
|
43
|
-
*/
|
|
44
|
-
async execute(args) {
|
|
45
|
-
const { library, version } = args;
|
|
46
|
-
logger.info(
|
|
47
|
-
`Removing library: ${library}${version ? `, version: ${version}` : " (unversioned)"}`
|
|
48
|
-
);
|
|
49
|
-
try {
|
|
50
|
-
await this.documentManagementService.removeAllDocuments(library, version);
|
|
51
|
-
const message = `Successfully removed documents for ${library}${version ? `@${version}` : " (unversioned)"}.`;
|
|
52
|
-
logger.info(message);
|
|
53
|
-
return { message };
|
|
54
|
-
} catch (error) {
|
|
55
|
-
const errorMessage = `Failed to remove documents for ${library}${version ? `@${version}` : " (unversioned)"}: ${error instanceof Error ? error.message : String(error)}`;
|
|
56
|
-
logger.error(`Error removing library: ${errorMessage}`);
|
|
57
|
-
throw new ToolError(errorMessage, this.constructor.name);
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
export {
|
|
62
|
-
ListJobsTool as L,
|
|
63
|
-
RemoveTool as R
|
|
64
|
-
};
|
|
65
|
-
//# sourceMappingURL=RemoveTool-w8KGOaXw.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"RemoveTool-w8KGOaXw.js","sources":["../src/tools/ListJobsTool.ts","../src/tools/RemoveTool.ts"],"sourcesContent":["import type { PipelineManager } from \"../pipeline/PipelineManager\";\nimport type { PipelineJob, PipelineJobStatus } from \"../pipeline/types\";\nimport type { JobInfo } from \"./GetJobInfoTool\"; // Import JobInfo\n\n/**\n * Input parameters for the ListJobsTool.\n */\nexport interface ListJobsInput {\n /** Optional status to filter jobs by. */\n status?: PipelineJobStatus;\n}\n\n/**\n * Response structure for the ListJobsTool.\n */\nexport interface ListJobsToolResponse {\n jobs: JobInfo[];\n}\n\n/**\n * Tool for listing pipeline jobs managed by the PipelineManager.\n * Allows filtering jobs by their status.\n */\nexport class ListJobsTool {\n private manager: PipelineManager; // Change property name and type\n\n /**\n * Creates an instance of ListJobsTool.\n * @param manager The PipelineManager instance.\n */\n constructor(manager: PipelineManager) {\n // Change constructor parameter\n this.manager = manager;\n }\n\n /**\n * Executes the tool to retrieve a list of pipeline jobs.\n * @param input - The input parameters, optionally including a status filter.\n * @returns A promise that resolves with the list of simplified job objects.\n * @throws {PipelineStateError} If the pipeline manager is somehow unavailable.\n */\n async execute(input: ListJobsInput): Promise<ListJobsToolResponse> {\n const jobs = await this.manager.getJobs(input.status);\n\n // Transform jobs into simplified objects\n const simplifiedJobs: JobInfo[] = jobs.map(\n (job: PipelineJob): JobInfo => ({\n id: job.id,\n library: job.library,\n version: job.version,\n status: job.status,\n createdAt: job.createdAt.toISOString(),\n startedAt: job.startedAt?.toISOString() ?? null,\n finishedAt: job.finishedAt?.toISOString() ?? null,\n error: job.error?.message ?? null,\n }),\n );\n\n return { jobs: simplifiedJobs };\n }\n}\n","import type { DocumentManagementService } from \"../store\";\nimport { logger } from \"../utils/logger\";\nimport { ToolError } from \"./errors\";\n\n/**\n * Represents the arguments for the remove_docs tool.\n * The MCP server should validate the input against RemoveToolInputSchema before calling execute.\n */\nexport interface RemoveToolArgs {\n library: string;\n version?: string;\n}\n\n/**\n * Tool to remove indexed documentation for a specific library version.\n * This class provides the core logic, intended to be called by the McpServer.\n */\nexport class RemoveTool {\n constructor(private readonly documentManagementService: DocumentManagementService) {}\n\n /**\n * Executes the tool to remove the specified library version documents.\n * Assumes args have been validated by the caller (McpServer) against inputSchema.\n * Returns a simple success message or throws an error.\n */\n async execute(args: RemoveToolArgs): Promise<{ message: string }> {\n const { library, version } = args;\n\n logger.info(\n `Removing library: ${library}${version ? `, version: ${version}` : \" (unversioned)\"}`,\n );\n\n try {\n // Core logic: Call the document management service\n await this.documentManagementService.removeAllDocuments(library, version);\n\n const message = `Successfully removed documents for ${library}${version ? `@${version}` : \" (unversioned)\"}.`;\n logger.info(message);\n // Return a simple success object, the McpServer will format the final response\n return { message };\n } catch (error) {\n const errorMessage = `Failed to remove documents for ${library}${version ? `@${version}` : \" (unversioned)\"}: ${error instanceof Error ? error.message : String(error)}`;\n logger.error(`Error removing library: ${errorMessage}`);\n // Re-throw the error for the McpServer to handle and format\n throw new ToolError(errorMessage, this.constructor.name);\n }\n }\n}\n"],"names":[],"mappings":";AAuBO,MAAM,aAAa;AAAA,EAChB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAMR,YAAY,SAA0B;AAEpC,SAAK,UAAU;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASjB,MAAM,QAAQ,OAAqD;AACjE,UAAM,OAAO,MAAM,KAAK,QAAQ,QAAQ,MAAM,MAAM;AAGpD,UAAM,iBAA4B,KAAK;AAAA,MACrC,CAAC,SAA+B;AAAA,QAC9B,IAAI,IAAI;AAAA,QACR,SAAS,IAAI;AAAA,QACb,SAAS,IAAI;AAAA,QACb,QAAQ,IAAI;AAAA,QACZ,WAAW,IAAI,UAAU,YAAY;AAAA,QACrC,WAAW,IAAI,WAAW,YAAiB,KAAA;AAAA,QAC3C,YAAY,IAAI,YAAY,YAAiB,KAAA;AAAA,QAC7C,OAAO,IAAI,OAAO,WAAW;AAAA,MAC/B;AAAA,IACF;AAEO,WAAA,EAAE,MAAM,eAAe;AAAA,EAAA;AAElC;AC3CO,MAAM,WAAW;AAAA,EACtB,YAA6B,2BAAsD;AAAtD,SAAA,4BAAA;AAAA,EAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAO7B,MAAM,QAAQ,MAAoD;AAC1D,UAAA,EAAE,SAAS,QAAA,IAAY;AAEtB,WAAA;AAAA,MACL,qBAAqB,OAAO,GAAG,UAAU,cAAc,OAAO,KAAK,gBAAgB;AAAA,IACrF;AAEI,QAAA;AAEF,YAAM,KAAK,0BAA0B,mBAAmB,SAAS,OAAO;AAElE,YAAA,UAAU,sCAAsC,OAAO,GAAG,UAAU,IAAI,OAAO,KAAK,gBAAgB;AAC1G,aAAO,KAAK,OAAO;AAEnB,aAAO,EAAE,QAAQ;AAAA,aACV,OAAO;AACd,YAAM,eAAe,kCAAkC,OAAO,GAAG,UAAU,IAAI,OAAO,KAAK,gBAAgB,KAAK,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,CAAC;AAC/J,aAAA,MAAM,2BAA2B,YAAY,EAAE;AAEtD,YAAM,IAAI,UAAU,cAAc,KAAK,YAAY,IAAI;AAAA,IAAA;AAAA,EACzD;AAEJ;"}
|
package/dist/cli.js
DELETED
|
@@ -1,236 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
import "dotenv/config";
|
|
3
|
-
import { Command } from "commander";
|
|
4
|
-
import { D as DocumentManagementService, a as PipelineManager, H as HttpFetcher, F as FileFetcher, S as SearchTool, e as ScrapeTool, f as ListLibrariesTool, c as DEFAULT_MAX_PAGES, b as DEFAULT_MAX_DEPTH, i as DEFAULT_MAX_CONCURRENCY, j as ScrapeMode, s as setLogLevel, d as LogLevel } from "./DocumentManagementService-BZ_ZZgPI.js";
|
|
5
|
-
import "semver";
|
|
6
|
-
import { F as FetchUrlTool, a as FindVersionTool } from "./FindVersionTool-Dfw5Lbql.js";
|
|
7
|
-
const name = "@arabold/docs-mcp-server";
|
|
8
|
-
const version = "1.13.0";
|
|
9
|
-
const description = "MCP server for fetching and searching documentation";
|
|
10
|
-
const type = "module";
|
|
11
|
-
const bin = { "docs-server": "dist/server.js", "docs-cli": "dist/cli.js", "docs-web": "dist/web.js" };
|
|
12
|
-
const license = "MIT";
|
|
13
|
-
const repository = { "type": "git", "url": "git+https://github.com/arabold/docs-mcp-server.git" };
|
|
14
|
-
const files = ["dist", "public", "db", "README.md", "LICENSE", "package.json"];
|
|
15
|
-
const scripts = { "prepare": "husky || true", "build": "vite build --config vite.config.web.ts && vite build", "start": "node --enable-source-maps dist/server.js", "dev:cli": "vite build && node --enable-source-maps dist/cli.js", "dev:server": "vite-node --watch src/server.ts", "dev:server:stdio": "vite-node --watch src/server.ts -- --protocol stdio", "dev:server:http": "vite-node --watch src/server.ts -- --protocol http", "dev:web": "npm-run-all --parallel watch:web web", "watch:web": "vite build --config vite.config.web.ts --watch", "cli": "node --enable-source-maps dist/cli.js", "server": "vite-node --watch src/server.ts", "web": "vite-node --watch src/web.ts", "test": "vitest run", "test:watch": "vitest", "test:coverage": "vitest run --coverage", "lint": "biome check .", "format": "biome format . --write", "postinstall": "npx playwright install --no-shell --with-deps chromium" };
|
|
16
|
-
const dependencies = { "@fastify/formbody": "^8.0.2", "@fastify/static": "^8.1.1", "@joplin/turndown-plugin-gfm": "^1.0.61", "@kitajs/html": "^4.2.7", "@kitajs/ts-html-plugin": "^4.1.1", "@langchain/aws": "^0.1.8", "@langchain/community": "^0.3.34", "@langchain/google-genai": "^0.2.3", "@langchain/google-vertexai": "^0.2.4", "@langchain/openai": "^0.5.0", "@modelcontextprotocol/sdk": "^1.10.2", "alpinejs": "^3.14.9", "axios": "^1.8.3", "axios-retry": "^4.5.0", "better-sqlite3": "^11.9.1", "cheerio": "^1.0.0", "commander": "^13.1.0", "dompurify": "^3.2.5", "dotenv": "^16.4.7", "env-paths": "^3.0.0", "fastify": "^5.3.0", "flowbite": "^3.1.2", "fuse.js": "^7.1.0", "header-generator": "^2.1.66", "htmx.org": "^1.9.12", "jsdom": "^26.0.0", "langchain": "0.3.19", "playwright": "^1.52.0", "psl": "^1.15.0", "remark": "^15.0.1", "remark-gfm": "^4.0.1", "remark-html": "^16.0.1", "semver": "^7.7.1", "sqlite-vec": "^0.1.7-alpha.2", "turndown": "^7.2.0", "zod": "^3.24.2" };
|
|
17
|
-
const devDependencies = { "@biomejs/biome": "1.9.4", "@commitlint/cli": "^19.8.0", "@commitlint/config-conventional": "^19.8.0", "@semantic-release/changelog": "^6.0.3", "@semantic-release/git": "^10.0.1", "@semantic-release/github": "^11.0.1", "@semantic-release/npm": "^12.0.1", "@tailwindcss/postcss": "^4.1.4", "@tailwindcss/vite": "^4.1.4", "@types/alpinejs": "^3.13.11", "@types/better-sqlite3": "^7.6.12", "@types/jsdom": "~21.1.7", "@types/lint-staged": "~13.3.0", "@types/node": "^20.17.23", "@types/node-fetch": "^2.6.12", "@types/psl": "^1.1.3", "@types/semver": "^7.5.8", "@types/turndown": "^5.0.5", "autoprefixer": "^10.4.21", "flowbite-typography": "^1.0.5", "husky": "^9.1.7", "lint-staged": "^15.5.0", "memfs": "^4.17.0", "npm-run-all": "^4.1.5", "postcss": "^8.5.3", "semantic-release": "^24.2.3", "tailwindcss": "^4.1.4", "typescript": "^5.8.2", "vite": "^6.2.1", "vite-node": "^3.1.2", "vite-plugin-dts": "^4.5.3", "vitest": "^3.0.8" };
|
|
18
|
-
const engines = { "node": ">=20.0.0" };
|
|
19
|
-
const packageJson = {
|
|
20
|
-
name,
|
|
21
|
-
version,
|
|
22
|
-
description,
|
|
23
|
-
type,
|
|
24
|
-
bin,
|
|
25
|
-
license,
|
|
26
|
-
repository,
|
|
27
|
-
files,
|
|
28
|
-
scripts,
|
|
29
|
-
dependencies,
|
|
30
|
-
devDependencies,
|
|
31
|
-
engines,
|
|
32
|
-
"lint-staged": { "*.{js,ts,jsx,tsx,json,md}": ["biome check --apply --no-errors-on-unmatched", "biome format --write --no-errors-on-unmatched"] }
|
|
33
|
-
};
|
|
34
|
-
const formatOutput = (data) => JSON.stringify(data, null, 2);
|
|
35
|
-
async function main() {
|
|
36
|
-
let docService;
|
|
37
|
-
let pipelineManager;
|
|
38
|
-
try {
|
|
39
|
-
docService = new DocumentManagementService();
|
|
40
|
-
await docService.initialize();
|
|
41
|
-
pipelineManager = new PipelineManager(docService);
|
|
42
|
-
await pipelineManager.start();
|
|
43
|
-
const tools = {
|
|
44
|
-
listLibraries: new ListLibrariesTool(docService),
|
|
45
|
-
findVersion: new FindVersionTool(docService),
|
|
46
|
-
scrape: new ScrapeTool(docService, pipelineManager),
|
|
47
|
-
// Pass manager
|
|
48
|
-
search: new SearchTool(docService),
|
|
49
|
-
fetchUrl: new FetchUrlTool(new HttpFetcher(), new FileFetcher())
|
|
50
|
-
};
|
|
51
|
-
const program = new Command();
|
|
52
|
-
process.on("SIGINT", async () => {
|
|
53
|
-
if (pipelineManager) await pipelineManager.stop();
|
|
54
|
-
if (docService) await docService.shutdown();
|
|
55
|
-
process.exit(0);
|
|
56
|
-
});
|
|
57
|
-
program.name("docs-mcp").description("CLI for managing documentation vector store").version(packageJson.version).option("--verbose", "Enable verbose (debug) logging", false).option("--silent", "Disable all logging except errors", false);
|
|
58
|
-
program.command("scrape <library> <url>").description("Scrape and index documentation from a URL").option("-v, --version <string>", "Version of the library (optional)").option(
|
|
59
|
-
"-p, --max-pages <number>",
|
|
60
|
-
"Maximum pages to scrape",
|
|
61
|
-
DEFAULT_MAX_PAGES.toString()
|
|
62
|
-
).option(
|
|
63
|
-
"-d, --max-depth <number>",
|
|
64
|
-
"Maximum navigation depth",
|
|
65
|
-
DEFAULT_MAX_DEPTH.toString()
|
|
66
|
-
).option(
|
|
67
|
-
"-c, --max-concurrency <number>",
|
|
68
|
-
"Maximum concurrent page requests",
|
|
69
|
-
DEFAULT_MAX_CONCURRENCY.toString()
|
|
70
|
-
).option("--ignore-errors", "Ignore errors during scraping", true).option(
|
|
71
|
-
"--scope <scope>",
|
|
72
|
-
"Crawling boundary: 'subpages' (default), 'hostname', or 'domain'",
|
|
73
|
-
(value) => {
|
|
74
|
-
const validScopes = ["subpages", "hostname", "domain"];
|
|
75
|
-
if (!validScopes.includes(value)) {
|
|
76
|
-
console.warn(`Warning: Invalid scope '${value}'. Using default 'subpages'.`);
|
|
77
|
-
return "subpages";
|
|
78
|
-
}
|
|
79
|
-
return value;
|
|
80
|
-
},
|
|
81
|
-
"subpages"
|
|
82
|
-
).option(
|
|
83
|
-
"--no-follow-redirects",
|
|
84
|
-
"Disable following HTTP redirects (default: follow redirects)"
|
|
85
|
-
).option(
|
|
86
|
-
"--scrape-mode <mode>",
|
|
87
|
-
`HTML processing strategy: '${ScrapeMode.Fetch}', '${ScrapeMode.Playwright}', '${ScrapeMode.Auto}' (default)`,
|
|
88
|
-
(value) => {
|
|
89
|
-
const validModes = Object.values(ScrapeMode);
|
|
90
|
-
if (!validModes.includes(value)) {
|
|
91
|
-
console.warn(
|
|
92
|
-
`Warning: Invalid scrape mode '${value}'. Using default '${ScrapeMode.Auto}'.`
|
|
93
|
-
);
|
|
94
|
-
return ScrapeMode.Auto;
|
|
95
|
-
}
|
|
96
|
-
return value;
|
|
97
|
-
},
|
|
98
|
-
ScrapeMode.Auto
|
|
99
|
-
// Use enum default
|
|
100
|
-
).action(async (library, url, options) => {
|
|
101
|
-
const result = await tools.scrape.execute({
|
|
102
|
-
url,
|
|
103
|
-
library,
|
|
104
|
-
version: options.version,
|
|
105
|
-
// Get version from options
|
|
106
|
-
options: {
|
|
107
|
-
maxPages: Number.parseInt(options.maxPages),
|
|
108
|
-
maxDepth: Number.parseInt(options.maxDepth),
|
|
109
|
-
maxConcurrency: Number.parseInt(options.maxConcurrency),
|
|
110
|
-
ignoreErrors: options.ignoreErrors,
|
|
111
|
-
scope: options.scope,
|
|
112
|
-
followRedirects: options.followRedirects,
|
|
113
|
-
// This will be `true` by default, or `false` if --no-follow-redirects is used
|
|
114
|
-
scrapeMode: options.scrapeMode
|
|
115
|
-
// Pass the new scrapeMode option
|
|
116
|
-
}
|
|
117
|
-
// CLI always waits for completion (default behavior)
|
|
118
|
-
});
|
|
119
|
-
if ("pagesScraped" in result) {
|
|
120
|
-
console.log(`✅ Successfully scraped ${result.pagesScraped} pages`);
|
|
121
|
-
} else {
|
|
122
|
-
console.log(`🚀 Scraping job started with ID: ${result.jobId}`);
|
|
123
|
-
}
|
|
124
|
-
});
|
|
125
|
-
program.command("search <library> <query>").description(
|
|
126
|
-
"Search documents in a library. Version matching examples:\n - search react --version 18.0.0 'hooks' -> matches docs for React 18.0.0 or earlier versions\n - search react --version 18.0.0 'hooks' --exact-match -> only matches React 18.0.0\n - search typescript --version 5.x 'types' -> matches any TypeScript 5.x.x version\n - search typescript --version 5.2.x 'types' -> matches any TypeScript 5.2.x version"
|
|
127
|
-
).option(
|
|
128
|
-
"-v, --version <string>",
|
|
129
|
-
// Add optional version flag
|
|
130
|
-
"Version of the library (optional, supports ranges)"
|
|
131
|
-
).option("-l, --limit <number>", "Maximum number of results", "5").option(
|
|
132
|
-
"-e, --exact-match",
|
|
133
|
-
"Only use exact version match (e.g., '18.0.0' matches only 18.0.0, not 17.x.x) (default: false)",
|
|
134
|
-
false
|
|
135
|
-
).action(async (library, query, options) => {
|
|
136
|
-
const result = await tools.search.execute({
|
|
137
|
-
library,
|
|
138
|
-
version: options.version,
|
|
139
|
-
// Get version from options
|
|
140
|
-
query,
|
|
141
|
-
limit: Number.parseInt(options.limit),
|
|
142
|
-
exactMatch: options.exactMatch
|
|
143
|
-
});
|
|
144
|
-
console.log(formatOutput(result.results));
|
|
145
|
-
});
|
|
146
|
-
program.command("list").description("List all available libraries and their versions").action(async () => {
|
|
147
|
-
const result = await tools.listLibraries.execute();
|
|
148
|
-
console.log(formatOutput(result.libraries));
|
|
149
|
-
});
|
|
150
|
-
program.command("find-version <library>").description("Find the best matching version for a library").option(
|
|
151
|
-
"-v, --version <string>",
|
|
152
|
-
// Add optional version flag
|
|
153
|
-
"Pattern to match (optional, supports ranges)"
|
|
154
|
-
).action(async (library, options) => {
|
|
155
|
-
const versionInfo = await tools.findVersion.execute({
|
|
156
|
-
library,
|
|
157
|
-
targetVersion: options.version
|
|
158
|
-
// Get version from options
|
|
159
|
-
});
|
|
160
|
-
if (!versionInfo) {
|
|
161
|
-
throw new Error("Failed to get version information");
|
|
162
|
-
}
|
|
163
|
-
console.log(versionInfo);
|
|
164
|
-
});
|
|
165
|
-
program.command("remove <library>").description("Remove documents for a specific library and version").option(
|
|
166
|
-
"-v, --version <string>",
|
|
167
|
-
"Version to remove (optional, removes unversioned if omitted)"
|
|
168
|
-
).action(async (library, options) => {
|
|
169
|
-
if (!docService) {
|
|
170
|
-
throw new Error("Document service not initialized.");
|
|
171
|
-
}
|
|
172
|
-
const { version: version2 } = options;
|
|
173
|
-
try {
|
|
174
|
-
await docService.removeAllDocuments(library, version2);
|
|
175
|
-
console.log(
|
|
176
|
-
`✅ Successfully removed documents for ${library}${version2 ? `@${version2}` : " (unversioned)"}.`
|
|
177
|
-
);
|
|
178
|
-
} catch (error) {
|
|
179
|
-
console.error(
|
|
180
|
-
`❌ Failed to remove documents for ${library}${version2 ? `@${version2}` : " (unversioned)"}:`,
|
|
181
|
-
error instanceof Error ? error.message : String(error)
|
|
182
|
-
);
|
|
183
|
-
throw error;
|
|
184
|
-
}
|
|
185
|
-
});
|
|
186
|
-
program.command("fetch-url <url>").description("Fetch a URL and convert its content to Markdown").option(
|
|
187
|
-
"--no-follow-redirects",
|
|
188
|
-
"Disable following HTTP redirects (default: follow redirects)"
|
|
189
|
-
).option(
|
|
190
|
-
"--scrape-mode <mode>",
|
|
191
|
-
`HTML processing strategy: '${ScrapeMode.Fetch}', '${ScrapeMode.Playwright}', '${ScrapeMode.Auto}' (default)`,
|
|
192
|
-
(value) => {
|
|
193
|
-
const validModes = Object.values(ScrapeMode);
|
|
194
|
-
if (!validModes.includes(value)) {
|
|
195
|
-
console.warn(
|
|
196
|
-
`Warning: Invalid scrape mode '${value}'. Using default '${ScrapeMode.Auto}'.`
|
|
197
|
-
);
|
|
198
|
-
return ScrapeMode.Auto;
|
|
199
|
-
}
|
|
200
|
-
return value;
|
|
201
|
-
},
|
|
202
|
-
ScrapeMode.Auto
|
|
203
|
-
// Use enum default
|
|
204
|
-
).action(async (url, options) => {
|
|
205
|
-
const content = await tools.fetchUrl.execute({
|
|
206
|
-
url,
|
|
207
|
-
followRedirects: options.followRedirects,
|
|
208
|
-
scrapeMode: options.scrapeMode
|
|
209
|
-
// Pass the scrapeMode option
|
|
210
|
-
});
|
|
211
|
-
console.log(content);
|
|
212
|
-
});
|
|
213
|
-
program.hook("preAction", (thisCommand) => {
|
|
214
|
-
const options = thisCommand.opts();
|
|
215
|
-
if (options.silent) {
|
|
216
|
-
setLogLevel(LogLevel.ERROR);
|
|
217
|
-
} else if (options.verbose) {
|
|
218
|
-
setLogLevel(LogLevel.DEBUG);
|
|
219
|
-
}
|
|
220
|
-
});
|
|
221
|
-
await program.parseAsync();
|
|
222
|
-
} catch (error) {
|
|
223
|
-
console.error("Error:", error instanceof Error ? error.message : String(error));
|
|
224
|
-
if (pipelineManager) await pipelineManager.stop();
|
|
225
|
-
if (docService) await docService.shutdown();
|
|
226
|
-
process.exit(1);
|
|
227
|
-
}
|
|
228
|
-
if (pipelineManager) await pipelineManager.stop();
|
|
229
|
-
await docService.shutdown();
|
|
230
|
-
process.exit(0);
|
|
231
|
-
}
|
|
232
|
-
main().catch((error) => {
|
|
233
|
-
console.error("Fatal error:", error);
|
|
234
|
-
process.exit(1);
|
|
235
|
-
});
|
|
236
|
-
//# sourceMappingURL=cli.js.map
|
package/dist/cli.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cli.js","sources":["../src/cli.ts"],"sourcesContent":["#!/usr/bin/env node\nimport \"dotenv/config\";\nimport { Command } from \"commander\";\nimport packageJson from \"../package.json\";\nimport { PipelineManager } from \"./pipeline/PipelineManager\";\nimport { FileFetcher, HttpFetcher } from \"./scraper/fetcher\";\nimport { ScrapeMode } from \"./scraper/types\"; // Import ScrapeMode enum\nimport { DocumentManagementService } from \"./store/DocumentManagementService\";\nimport {\n FetchUrlTool,\n FindVersionTool,\n ListLibrariesTool,\n ScrapeTool,\n SearchTool,\n} from \"./tools\";\nimport {\n DEFAULT_MAX_CONCURRENCY,\n DEFAULT_MAX_DEPTH,\n DEFAULT_MAX_PAGES,\n} from \"./utils/config\";\nimport { LogLevel, setLogLevel } from \"./utils/logger\";\n\nconst formatOutput = (data: unknown) => JSON.stringify(data, null, 2);\n\nasync function main() {\n let docService: DocumentManagementService | undefined;\n let pipelineManager: PipelineManager | undefined;\n\n try {\n docService = new DocumentManagementService();\n await docService.initialize();\n\n // Instantiate PipelineManager for CLI use\n pipelineManager = new PipelineManager(docService); // Assign inside try\n // Start the manager for the CLI session\n await pipelineManager.start();\n\n const tools = {\n listLibraries: new ListLibrariesTool(docService),\n findVersion: new FindVersionTool(docService),\n scrape: new ScrapeTool(docService, pipelineManager), // Pass manager\n search: new SearchTool(docService),\n fetchUrl: new FetchUrlTool(new HttpFetcher(), new FileFetcher()),\n };\n\n const program = new Command();\n\n // Handle cleanup on SIGINT\n process.on(\"SIGINT\", async () => {\n if (pipelineManager) await pipelineManager.stop(); // Check before stopping\n if (docService) await docService.shutdown(); // Check before stopping\n process.exit(0);\n });\n\n program\n .name(\"docs-mcp\")\n .description(\"CLI for managing documentation vector store\")\n .version(packageJson.version)\n // Add global options for logging level\n .option(\"--verbose\", \"Enable verbose (debug) logging\", false)\n .option(\"--silent\", \"Disable all logging except errors\", false);\n\n program\n .command(\"scrape <library> <url>\") // Remove <version> as positional\n .description(\"Scrape and index documentation from a URL\")\n .option(\"-v, --version <string>\", \"Version of the library (optional)\") // Add optional version flag\n .option(\n \"-p, --max-pages <number>\",\n \"Maximum pages to scrape\",\n DEFAULT_MAX_PAGES.toString(),\n )\n .option(\n \"-d, --max-depth <number>\",\n \"Maximum navigation depth\",\n DEFAULT_MAX_DEPTH.toString(),\n )\n .option(\n \"-c, --max-concurrency <number>\",\n \"Maximum concurrent page requests\",\n DEFAULT_MAX_CONCURRENCY.toString(),\n )\n .option(\"--ignore-errors\", \"Ignore errors during scraping\", true)\n .option(\n \"--scope <scope>\",\n \"Crawling boundary: 'subpages' (default), 'hostname', or 'domain'\",\n (value) => {\n const validScopes = [\"subpages\", \"hostname\", \"domain\"];\n if (!validScopes.includes(value)) {\n console.warn(`Warning: Invalid scope '${value}'. Using default 'subpages'.`);\n return \"subpages\";\n }\n return value;\n },\n \"subpages\",\n )\n .option(\n \"--no-follow-redirects\",\n \"Disable following HTTP redirects (default: follow redirects)\",\n )\n .option(\n \"--scrape-mode <mode>\",\n `HTML processing strategy: '${ScrapeMode.Fetch}', '${ScrapeMode.Playwright}', '${ScrapeMode.Auto}' (default)`,\n (value: string): ScrapeMode => {\n const validModes = Object.values(ScrapeMode);\n if (!validModes.includes(value as ScrapeMode)) {\n console.warn(\n `Warning: Invalid scrape mode '${value}'. Using default '${ScrapeMode.Auto}'.`,\n );\n return ScrapeMode.Auto;\n }\n return value as ScrapeMode; // Cast to enum type\n },\n ScrapeMode.Auto, // Use enum default\n )\n .action(async (library, url, options) => {\n // Update action parameters\n const result = await tools.scrape.execute({\n url,\n library,\n version: options.version, // Get version from options\n options: {\n maxPages: Number.parseInt(options.maxPages),\n maxDepth: Number.parseInt(options.maxDepth),\n maxConcurrency: Number.parseInt(options.maxConcurrency),\n ignoreErrors: options.ignoreErrors,\n scope: options.scope,\n followRedirects: options.followRedirects, // This will be `true` by default, or `false` if --no-follow-redirects is used\n scrapeMode: options.scrapeMode, // Pass the new scrapeMode option\n },\n // CLI always waits for completion (default behavior)\n });\n // Type guard to satisfy TypeScript\n if (\"pagesScraped\" in result) {\n console.log(`✅ Successfully scraped ${result.pagesScraped} pages`);\n } else {\n // This branch should not be hit by the CLI\n console.log(`🚀 Scraping job started with ID: ${result.jobId}`);\n }\n });\n\n program\n .command(\"search <library> <query>\") // Remove <version> as positional\n .description(\n \"Search documents in a library. Version matching examples:\\n\" +\n \" - search react --version 18.0.0 'hooks' -> matches docs for React 18.0.0 or earlier versions\\n\" +\n \" - search react --version 18.0.0 'hooks' --exact-match -> only matches React 18.0.0\\n\" +\n \" - search typescript --version 5.x 'types' -> matches any TypeScript 5.x.x version\\n\" +\n \" - search typescript --version 5.2.x 'types' -> matches any TypeScript 5.2.x version\",\n )\n .option(\n \"-v, --version <string>\", // Add optional version flag\n \"Version of the library (optional, supports ranges)\",\n )\n .option(\"-l, --limit <number>\", \"Maximum number of results\", \"5\")\n .option(\n \"-e, --exact-match\",\n \"Only use exact version match (e.g., '18.0.0' matches only 18.0.0, not 17.x.x) (default: false)\",\n false,\n )\n .action(async (library, query, options) => {\n // Update action parameters\n const result = await tools.search.execute({\n library,\n version: options.version, // Get version from options\n query,\n limit: Number.parseInt(options.limit),\n exactMatch: options.exactMatch,\n });\n console.log(formatOutput(result.results));\n });\n\n program\n .command(\"list\")\n .description(\"List all available libraries and their versions\")\n .action(async () => {\n const result = await tools.listLibraries.execute();\n console.log(formatOutput(result.libraries));\n });\n\n program\n .command(\"find-version <library>\") // Remove [targetVersion] positional\n .description(\"Find the best matching version for a library\")\n .option(\n \"-v, --version <string>\", // Add optional version flag\n \"Pattern to match (optional, supports ranges)\",\n )\n .action(async (library, options) => {\n // Update action parameters\n const versionInfo = await tools.findVersion.execute({\n library,\n targetVersion: options.version, // Get version from options\n });\n // findVersion.execute now returns a string, handle potential error messages within it\n if (!versionInfo) {\n // Should not happen with current tool logic, but good practice\n throw new Error(\"Failed to get version information\");\n }\n console.log(versionInfo); // Log the descriptive string from the tool\n });\n\n program\n .command(\"remove <library>\") // Library as positional argument\n .description(\"Remove documents for a specific library and version\")\n .option(\n \"-v, --version <string>\",\n \"Version to remove (optional, removes unversioned if omitted)\",\n )\n .action(async (library, options) => {\n // library is now the first arg\n if (!docService) {\n throw new Error(\"Document service not initialized.\");\n }\n const { version } = options; // Get version from options\n try {\n await docService.removeAllDocuments(library, version);\n console.log(\n `✅ Successfully removed documents for ${library}${version ? `@${version}` : \" (unversioned)\"}.`,\n );\n } catch (error) {\n console.error(\n `❌ Failed to remove documents for ${library}${version ? `@${version}` : \" (unversioned)\"}:`,\n error instanceof Error ? error.message : String(error),\n );\n // Re-throw to trigger the main catch block for shutdown\n throw error;\n }\n });\n\n program\n .command(\"fetch-url <url>\")\n .description(\"Fetch a URL and convert its content to Markdown\")\n .option(\n \"--no-follow-redirects\",\n \"Disable following HTTP redirects (default: follow redirects)\",\n )\n .option(\n \"--scrape-mode <mode>\",\n `HTML processing strategy: '${ScrapeMode.Fetch}', '${ScrapeMode.Playwright}', '${ScrapeMode.Auto}' (default)`,\n (value: string): ScrapeMode => {\n const validModes = Object.values(ScrapeMode);\n if (!validModes.includes(value as ScrapeMode)) {\n console.warn(\n `Warning: Invalid scrape mode '${value}'. Using default '${ScrapeMode.Auto}'.`,\n );\n return ScrapeMode.Auto;\n }\n return value as ScrapeMode; // Cast to enum type\n },\n ScrapeMode.Auto, // Use enum default\n )\n .action(async (url, options) => {\n const content = await tools.fetchUrl.execute({\n url,\n followRedirects: options.followRedirects,\n scrapeMode: options.scrapeMode, // Pass the scrapeMode option\n });\n console.log(content);\n });\n\n // Hook to set log level after parsing global options but before executing command action\n program.hook(\"preAction\", (thisCommand) => {\n // Global options are attached to the program (thisCommand)\n const options = thisCommand.opts();\n if (options.silent) {\n // If silent is true, it overrides verbose\n setLogLevel(LogLevel.ERROR);\n } else if (options.verbose) {\n setLogLevel(LogLevel.DEBUG);\n }\n // Otherwise, the default LogLevel.INFO remains set from logger.ts\n });\n\n await program.parseAsync();\n } catch (error) {\n console.error(\"Error:\", error instanceof Error ? error.message : String(error));\n if (pipelineManager) await pipelineManager.stop(); // Check before stopping\n if (docService) await docService.shutdown();\n process.exit(1);\n }\n\n // Clean shutdown after successful execution\n if (pipelineManager) await pipelineManager.stop(); // Check before stopping\n await docService.shutdown();\n process.exit(0);\n}\n\nmain().catch((error) => {\n console.error(\"Fatal error:\", error);\n process.exit(1);\n});\n"],"names":["version"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsBA,MAAM,eAAe,CAAC,SAAkB,KAAK,UAAU,MAAM,MAAM,CAAC;AAEpE,eAAe,OAAO;AAChB,MAAA;AACA,MAAA;AAEA,MAAA;AACF,iBAAa,IAAI,0BAA0B;AAC3C,UAAM,WAAW,WAAW;AAGV,sBAAA,IAAI,gBAAgB,UAAU;AAEhD,UAAM,gBAAgB,MAAM;AAE5B,UAAM,QAAQ;AAAA,MACZ,eAAe,IAAI,kBAAkB,UAAU;AAAA,MAC/C,aAAa,IAAI,gBAAgB,UAAU;AAAA,MAC3C,QAAQ,IAAI,WAAW,YAAY,eAAe;AAAA;AAAA,MAClD,QAAQ,IAAI,WAAW,UAAU;AAAA,MACjC,UAAU,IAAI,aAAa,IAAI,YAAe,GAAA,IAAI,YAAa,CAAA;AAAA,IACjE;AAEM,UAAA,UAAU,IAAI,QAAQ;AAGpB,YAAA,GAAG,UAAU,YAAY;AAC3B,UAAA,gBAAuB,OAAA,gBAAgB,KAAK;AAC5C,UAAA,WAAkB,OAAA,WAAW,SAAS;AAC1C,cAAQ,KAAK,CAAC;AAAA,IAAA,CACf;AAED,YACG,KAAK,UAAU,EACf,YAAY,6CAA6C,EACzD,QAAQ,YAAY,OAAO,EAE3B,OAAO,aAAa,kCAAkC,KAAK,EAC3D,OAAO,YAAY,qCAAqC,KAAK;AAG7D,YAAA,QAAQ,wBAAwB,EAChC,YAAY,2CAA2C,EACvD,OAAO,0BAA0B,mCAAmC,EACpE;AAAA,MACC;AAAA,MACA;AAAA,MACA,kBAAkB,SAAS;AAAA,IAAA,EAE5B;AAAA,MACC;AAAA,MACA;AAAA,MACA,kBAAkB,SAAS;AAAA,IAAA,EAE5B;AAAA,MACC;AAAA,MACA;AAAA,MACA,wBAAwB,SAAS;AAAA,IAElC,EAAA,OAAO,mBAAmB,iCAAiC,IAAI,EAC/D;AAAA,MACC;AAAA,MACA;AAAA,MACA,CAAC,UAAU;AACT,cAAM,cAAc,CAAC,YAAY,YAAY,QAAQ;AACrD,YAAI,CAAC,YAAY,SAAS,KAAK,GAAG;AACxB,kBAAA,KAAK,2BAA2B,KAAK,8BAA8B;AACpE,iBAAA;AAAA,QAAA;AAEF,eAAA;AAAA,MACT;AAAA,MACA;AAAA,IAAA,EAED;AAAA,MACC;AAAA,MACA;AAAA,IAAA,EAED;AAAA,MACC;AAAA,MACA,8BAA8B,WAAW,KAAK,OAAO,WAAW,UAAU,OAAO,WAAW,IAAI;AAAA,MAChG,CAAC,UAA8B;AACvB,cAAA,aAAa,OAAO,OAAO,UAAU;AAC3C,YAAI,CAAC,WAAW,SAAS,KAAmB,GAAG;AACrC,kBAAA;AAAA,YACN,iCAAiC,KAAK,qBAAqB,WAAW,IAAI;AAAA,UAC5E;AACA,iBAAO,WAAW;AAAA,QAAA;AAEb,eAAA;AAAA,MACT;AAAA,MACA,WAAW;AAAA;AAAA,IAEZ,EAAA,OAAO,OAAO,SAAS,KAAK,YAAY;AAEvC,YAAM,SAAS,MAAM,MAAM,OAAO,QAAQ;AAAA,QACxC;AAAA,QACA;AAAA,QACA,SAAS,QAAQ;AAAA;AAAA,QACjB,SAAS;AAAA,UACP,UAAU,OAAO,SAAS,QAAQ,QAAQ;AAAA,UAC1C,UAAU,OAAO,SAAS,QAAQ,QAAQ;AAAA,UAC1C,gBAAgB,OAAO,SAAS,QAAQ,cAAc;AAAA,UACtD,cAAc,QAAQ;AAAA,UACtB,OAAO,QAAQ;AAAA,UACf,iBAAiB,QAAQ;AAAA;AAAA,UACzB,YAAY,QAAQ;AAAA;AAAA,QAAA;AAAA;AAAA,MACtB,CAED;AAED,UAAI,kBAAkB,QAAQ;AAC5B,gBAAQ,IAAI,0BAA0B,OAAO,YAAY,QAAQ;AAAA,MAAA,OAC5D;AAEL,gBAAQ,IAAI,oCAAoC,OAAO,KAAK,EAAE;AAAA,MAAA;AAAA,IAChE,CACD;AAGA,YAAA,QAAQ,0BAA0B,EAClC;AAAA,MACC;AAAA,IAAA,EAMD;AAAA,MACC;AAAA;AAAA,MACA;AAAA,IAED,EAAA,OAAO,wBAAwB,6BAA6B,GAAG,EAC/D;AAAA,MACC;AAAA,MACA;AAAA,MACA;AAAA,IAED,EAAA,OAAO,OAAO,SAAS,OAAO,YAAY;AAEzC,YAAM,SAAS,MAAM,MAAM,OAAO,QAAQ;AAAA,QACxC;AAAA,QACA,SAAS,QAAQ;AAAA;AAAA,QACjB;AAAA,QACA,OAAO,OAAO,SAAS,QAAQ,KAAK;AAAA,QACpC,YAAY,QAAQ;AAAA,MAAA,CACrB;AACD,cAAQ,IAAI,aAAa,OAAO,OAAO,CAAC;AAAA,IAAA,CACzC;AAEH,YACG,QAAQ,MAAM,EACd,YAAY,iDAAiD,EAC7D,OAAO,YAAY;AAClB,YAAM,SAAS,MAAM,MAAM,cAAc,QAAQ;AACjD,cAAQ,IAAI,aAAa,OAAO,SAAS,CAAC;AAAA,IAAA,CAC3C;AAEH,YACG,QAAQ,wBAAwB,EAChC,YAAY,8CAA8C,EAC1D;AAAA,MACC;AAAA;AAAA,MACA;AAAA,IAAA,EAED,OAAO,OAAO,SAAS,YAAY;AAElC,YAAM,cAAc,MAAM,MAAM,YAAY,QAAQ;AAAA,QAClD;AAAA,QACA,eAAe,QAAQ;AAAA;AAAA,MAAA,CACxB;AAED,UAAI,CAAC,aAAa;AAEV,cAAA,IAAI,MAAM,mCAAmC;AAAA,MAAA;AAErD,cAAQ,IAAI,WAAW;AAAA,IAAA,CACxB;AAEH,YACG,QAAQ,kBAAkB,EAC1B,YAAY,qDAAqD,EACjE;AAAA,MACC;AAAA,MACA;AAAA,IAAA,EAED,OAAO,OAAO,SAAS,YAAY;AAElC,UAAI,CAAC,YAAY;AACT,cAAA,IAAI,MAAM,mCAAmC;AAAA,MAAA;AAE/C,YAAA,EAAE,SAAAA,aAAY;AAChB,UAAA;AACI,cAAA,WAAW,mBAAmB,SAASA,QAAO;AAC5C,gBAAA;AAAA,UACN,wCAAwC,OAAO,GAAGA,WAAU,IAAIA,QAAO,KAAK,gBAAgB;AAAA,QAC9F;AAAA,eACO,OAAO;AACN,gBAAA;AAAA,UACN,oCAAoC,OAAO,GAAGA,WAAU,IAAIA,QAAO,KAAK,gBAAgB;AAAA,UACxF,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,QACvD;AAEM,cAAA;AAAA,MAAA;AAAA,IACR,CACD;AAEH,YACG,QAAQ,iBAAiB,EACzB,YAAY,iDAAiD,EAC7D;AAAA,MACC;AAAA,MACA;AAAA,IAAA,EAED;AAAA,MACC;AAAA,MACA,8BAA8B,WAAW,KAAK,OAAO,WAAW,UAAU,OAAO,WAAW,IAAI;AAAA,MAChG,CAAC,UAA8B;AACvB,cAAA,aAAa,OAAO,OAAO,UAAU;AAC3C,YAAI,CAAC,WAAW,SAAS,KAAmB,GAAG;AACrC,kBAAA;AAAA,YACN,iCAAiC,KAAK,qBAAqB,WAAW,IAAI;AAAA,UAC5E;AACA,iBAAO,WAAW;AAAA,QAAA;AAEb,eAAA;AAAA,MACT;AAAA,MACA,WAAW;AAAA;AAAA,IAAA,EAEZ,OAAO,OAAO,KAAK,YAAY;AAC9B,YAAM,UAAU,MAAM,MAAM,SAAS,QAAQ;AAAA,QAC3C;AAAA,QACA,iBAAiB,QAAQ;AAAA,QACzB,YAAY,QAAQ;AAAA;AAAA,MAAA,CACrB;AACD,cAAQ,IAAI,OAAO;AAAA,IAAA,CACpB;AAGK,YAAA,KAAK,aAAa,CAAC,gBAAgB;AAEnC,YAAA,UAAU,YAAY,KAAK;AACjC,UAAI,QAAQ,QAAQ;AAElB,oBAAY,SAAS,KAAK;AAAA,MAAA,WACjB,QAAQ,SAAS;AAC1B,oBAAY,SAAS,KAAK;AAAA,MAAA;AAAA,IAC5B,CAED;AAED,UAAM,QAAQ,WAAW;AAAA,WAClB,OAAO;AACN,YAAA,MAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,CAAC;AAC1E,QAAA,gBAAuB,OAAA,gBAAgB,KAAK;AAC5C,QAAA,WAAkB,OAAA,WAAW,SAAS;AAC1C,YAAQ,KAAK,CAAC;AAAA,EAAA;AAIZ,MAAA,gBAAuB,OAAA,gBAAgB,KAAK;AAChD,QAAM,WAAW,SAAS;AAC1B,UAAQ,KAAK,CAAC;AAChB;AAEA,OAAO,MAAM,CAAC,UAAU;AACd,UAAA,MAAM,gBAAgB,KAAK;AACnC,UAAQ,KAAK,CAAC;AAChB,CAAC;"}
|