@gulibs/safe-coder 0.0.26 → 0.0.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (192) hide show
  1. package/README.md +678 -994
  2. package/dist/cache/cache-manager.d.ts +71 -0
  3. package/dist/cache/cache-manager.d.ts.map +1 -0
  4. package/dist/cache/cache-manager.js +244 -0
  5. package/dist/cache/cache-manager.js.map +1 -0
  6. package/dist/executor/cli-executor.d.ts +106 -0
  7. package/dist/executor/cli-executor.d.ts.map +1 -0
  8. package/dist/executor/cli-executor.js +133 -0
  9. package/dist/executor/cli-executor.js.map +1 -0
  10. package/dist/executor/dependency-checker.d.ts +23 -0
  11. package/dist/executor/dependency-checker.d.ts.map +1 -0
  12. package/dist/executor/dependency-checker.js +62 -0
  13. package/dist/executor/dependency-checker.js.map +1 -0
  14. package/dist/index.js +3 -4
  15. package/dist/index.js.map +1 -1
  16. package/dist/processor/content-processor.d.ts +76 -0
  17. package/dist/processor/content-processor.d.ts.map +1 -0
  18. package/dist/processor/content-processor.js +182 -0
  19. package/dist/processor/content-processor.js.map +1 -0
  20. package/dist/processor/guide-generator.d.ts +68 -0
  21. package/dist/processor/guide-generator.d.ts.map +1 -0
  22. package/dist/processor/guide-generator.js +189 -0
  23. package/dist/processor/guide-generator.js.map +1 -0
  24. package/dist/server/safe-coder-mcp.d.ts +18 -0
  25. package/dist/server/safe-coder-mcp.d.ts.map +1 -0
  26. package/dist/server/safe-coder-mcp.js +164 -0
  27. package/dist/server/safe-coder-mcp.js.map +1 -0
  28. package/dist/tools/cache-tools.d.ts +42 -0
  29. package/dist/tools/cache-tools.d.ts.map +1 -0
  30. package/dist/tools/cache-tools.js +70 -0
  31. package/dist/tools/cache-tools.js.map +1 -0
  32. package/dist/tools/crawl-documentation.d.ts +57 -0
  33. package/dist/tools/crawl-documentation.d.ts.map +1 -0
  34. package/dist/tools/crawl-documentation.js +96 -0
  35. package/dist/tools/crawl-documentation.js.map +1 -0
  36. package/dist/tools/index.d.ts +4 -0
  37. package/dist/tools/index.d.ts.map +1 -0
  38. package/dist/tools/index.js +4 -0
  39. package/dist/tools/index.js.map +1 -0
  40. package/dist/tools/save-skill.d.ts +49 -0
  41. package/dist/tools/save-skill.d.ts.map +1 -0
  42. package/dist/tools/save-skill.js +207 -0
  43. package/dist/tools/save-skill.js.map +1 -0
  44. package/package.json +18 -28
  45. package/dist/documentation/browser-doc-browser.d.ts +0 -41
  46. package/dist/documentation/browser-doc-browser.d.ts.map +0 -1
  47. package/dist/documentation/browser-doc-browser.js +0 -357
  48. package/dist/documentation/browser-doc-browser.js.map +0 -1
  49. package/dist/documentation/browser-manager.d.ts +0 -51
  50. package/dist/documentation/browser-manager.d.ts.map +0 -1
  51. package/dist/documentation/browser-manager.js +0 -260
  52. package/dist/documentation/browser-manager.js.map +0 -1
  53. package/dist/documentation/cache.d.ts +0 -13
  54. package/dist/documentation/cache.d.ts.map +0 -1
  55. package/dist/documentation/cache.js +0 -48
  56. package/dist/documentation/cache.js.map +0 -1
  57. package/dist/documentation/checkpoint-manager.d.ts +0 -38
  58. package/dist/documentation/checkpoint-manager.d.ts.map +0 -1
  59. package/dist/documentation/checkpoint-manager.js +0 -101
  60. package/dist/documentation/checkpoint-manager.js.map +0 -1
  61. package/dist/documentation/doc-crawler.d.ts +0 -221
  62. package/dist/documentation/doc-crawler.d.ts.map +0 -1
  63. package/dist/documentation/doc-crawler.js +0 -1415
  64. package/dist/documentation/doc-crawler.js.map +0 -1
  65. package/dist/documentation/github-client.d.ts +0 -13
  66. package/dist/documentation/github-client.d.ts.map +0 -1
  67. package/dist/documentation/github-client.js +0 -90
  68. package/dist/documentation/github-client.js.map +0 -1
  69. package/dist/documentation/http-fetcher.d.ts +0 -8
  70. package/dist/documentation/http-fetcher.d.ts.map +0 -1
  71. package/dist/documentation/http-fetcher.js +0 -31
  72. package/dist/documentation/http-fetcher.js.map +0 -1
  73. package/dist/documentation/index.d.ts +0 -16
  74. package/dist/documentation/index.d.ts.map +0 -1
  75. package/dist/documentation/index.js +0 -159
  76. package/dist/documentation/index.js.map +0 -1
  77. package/dist/documentation/llms-txt/detector.d.ts +0 -31
  78. package/dist/documentation/llms-txt/detector.d.ts.map +0 -1
  79. package/dist/documentation/llms-txt/detector.js +0 -77
  80. package/dist/documentation/llms-txt/detector.js.map +0 -1
  81. package/dist/documentation/llms-txt/downloader.d.ts +0 -30
  82. package/dist/documentation/llms-txt/downloader.d.ts.map +0 -1
  83. package/dist/documentation/llms-txt/downloader.js +0 -84
  84. package/dist/documentation/llms-txt/downloader.js.map +0 -1
  85. package/dist/documentation/llms-txt/index.d.ts +0 -4
  86. package/dist/documentation/llms-txt/index.d.ts.map +0 -1
  87. package/dist/documentation/llms-txt/index.js +0 -4
  88. package/dist/documentation/llms-txt/index.js.map +0 -1
  89. package/dist/documentation/llms-txt/parser.d.ts +0 -43
  90. package/dist/documentation/llms-txt/parser.d.ts.map +0 -1
  91. package/dist/documentation/llms-txt/parser.js +0 -177
  92. package/dist/documentation/llms-txt/parser.js.map +0 -1
  93. package/dist/documentation/normalizer.d.ts +0 -6
  94. package/dist/documentation/normalizer.d.ts.map +0 -1
  95. package/dist/documentation/normalizer.js +0 -38
  96. package/dist/documentation/normalizer.js.map +0 -1
  97. package/dist/documentation/npm-client.d.ts +0 -19
  98. package/dist/documentation/npm-client.d.ts.map +0 -1
  99. package/dist/documentation/npm-client.js +0 -182
  100. package/dist/documentation/npm-client.js.map +0 -1
  101. package/dist/documentation/skill-generator.d.ts +0 -108
  102. package/dist/documentation/skill-generator.d.ts.map +0 -1
  103. package/dist/documentation/skill-generator.js +0 -642
  104. package/dist/documentation/skill-generator.js.map +0 -1
  105. package/dist/documentation/web-doc-browser.d.ts +0 -67
  106. package/dist/documentation/web-doc-browser.d.ts.map +0 -1
  107. package/dist/documentation/web-doc-browser.js +0 -555
  108. package/dist/documentation/web-doc-browser.js.map +0 -1
  109. package/dist/errors/api-validator.d.ts +0 -9
  110. package/dist/errors/api-validator.d.ts.map +0 -1
  111. package/dist/errors/api-validator.js +0 -57
  112. package/dist/errors/api-validator.js.map +0 -1
  113. package/dist/errors/contextual-analysis.d.ts +0 -14
  114. package/dist/errors/contextual-analysis.d.ts.map +0 -1
  115. package/dist/errors/contextual-analysis.js +0 -173
  116. package/dist/errors/contextual-analysis.js.map +0 -1
  117. package/dist/errors/cross-file-analyzer.d.ts +0 -16
  118. package/dist/errors/cross-file-analyzer.d.ts.map +0 -1
  119. package/dist/errors/cross-file-analyzer.js +0 -172
  120. package/dist/errors/cross-file-analyzer.js.map +0 -1
  121. package/dist/errors/eslint-integration.d.ts +0 -9
  122. package/dist/errors/eslint-integration.d.ts.map +0 -1
  123. package/dist/errors/eslint-integration.js +0 -131
  124. package/dist/errors/eslint-integration.js.map +0 -1
  125. package/dist/errors/framework-detector.d.ts +0 -10
  126. package/dist/errors/framework-detector.d.ts.map +0 -1
  127. package/dist/errors/framework-detector.js +0 -126
  128. package/dist/errors/framework-detector.js.map +0 -1
  129. package/dist/errors/index.d.ts +0 -18
  130. package/dist/errors/index.d.ts.map +0 -1
  131. package/dist/errors/index.js +0 -134
  132. package/dist/errors/index.js.map +0 -1
  133. package/dist/errors/pattern-matcher.d.ts +0 -25
  134. package/dist/errors/pattern-matcher.d.ts.map +0 -1
  135. package/dist/errors/pattern-matcher.js +0 -44
  136. package/dist/errors/pattern-matcher.js.map +0 -1
  137. package/dist/errors/patterns.d.ts +0 -11
  138. package/dist/errors/patterns.d.ts.map +0 -1
  139. package/dist/errors/patterns.js +0 -351
  140. package/dist/errors/patterns.js.map +0 -1
  141. package/dist/errors/performance-detector.d.ts +0 -11
  142. package/dist/errors/performance-detector.d.ts.map +0 -1
  143. package/dist/errors/performance-detector.js +0 -119
  144. package/dist/errors/performance-detector.js.map +0 -1
  145. package/dist/errors/runtime-detector.d.ts +0 -7
  146. package/dist/errors/runtime-detector.d.ts.map +0 -1
  147. package/dist/errors/runtime-detector.js +0 -86
  148. package/dist/errors/runtime-detector.js.map +0 -1
  149. package/dist/errors/security-detector.d.ts +0 -6
  150. package/dist/errors/security-detector.d.ts.map +0 -1
  151. package/dist/errors/security-detector.js +0 -75
  152. package/dist/errors/security-detector.js.map +0 -1
  153. package/dist/errors/typescript-integration.d.ts +0 -6
  154. package/dist/errors/typescript-integration.d.ts.map +0 -1
  155. package/dist/errors/typescript-integration.js +0 -46
  156. package/dist/errors/typescript-integration.js.map +0 -1
  157. package/dist/server/mcp-server.d.ts +0 -14
  158. package/dist/server/mcp-server.d.ts.map +0 -1
  159. package/dist/server/mcp-server.js +0 -793
  160. package/dist/server/mcp-server.js.map +0 -1
  161. package/dist/types/documentation.d.ts +0 -26
  162. package/dist/types/documentation.d.ts.map +0 -1
  163. package/dist/types/documentation.js +0 -2
  164. package/dist/types/documentation.js.map +0 -1
  165. package/dist/utils/config.d.ts +0 -21
  166. package/dist/utils/config.d.ts.map +0 -1
  167. package/dist/utils/config.js +0 -34
  168. package/dist/utils/config.js.map +0 -1
  169. package/dist/utils/http-client.d.ts +0 -17
  170. package/dist/utils/http-client.d.ts.map +0 -1
  171. package/dist/utils/http-client.js +0 -62
  172. package/dist/utils/http-client.js.map +0 -1
  173. package/dist/utils/logger.d.ts +0 -36
  174. package/dist/utils/logger.d.ts.map +0 -1
  175. package/dist/utils/logger.js +0 -128
  176. package/dist/utils/logger.js.map +0 -1
  177. package/dist/utils/rate-limiter.d.ts +0 -9
  178. package/dist/utils/rate-limiter.d.ts.map +0 -1
  179. package/dist/utils/rate-limiter.js +0 -26
  180. package/dist/utils/rate-limiter.js.map +0 -1
  181. package/dist/validation/auto-fix.d.ts +0 -15
  182. package/dist/validation/auto-fix.d.ts.map +0 -1
  183. package/dist/validation/auto-fix.js +0 -49
  184. package/dist/validation/auto-fix.js.map +0 -1
  185. package/dist/validation/index.d.ts +0 -21
  186. package/dist/validation/index.d.ts.map +0 -1
  187. package/dist/validation/index.js +0 -45
  188. package/dist/validation/index.js.map +0 -1
  189. package/dist/validation/resolution-db.d.ts +0 -15
  190. package/dist/validation/resolution-db.d.ts.map +0 -1
  191. package/dist/validation/resolution-db.js +0 -62
  192. package/dist/validation/resolution-db.js.map +0 -1
@@ -1,84 +0,0 @@
1
- import { HttpClient } from '../../utils/http-client.js';
2
- import { logger } from '../../utils/logger.js';
3
- /**
4
- * Downloader for llms.txt files
5
- * Based on Skill_Seekers implementation
6
- */
7
- export class LlmsTxtDownloader {
8
- httpClient;
9
- constructor(httpClient) {
10
- this.httpClient = httpClient || new HttpClient();
11
- }
12
- /**
13
- * Download a single llms.txt file
14
- */
15
- async download(url) {
16
- logger.info('Downloading llms.txt file', { url });
17
- try {
18
- const response = await this.httpClient.get(url, {
19
- responseType: 'text',
20
- timeout: 30000, // 30 second timeout
21
- });
22
- const content = response.data;
23
- logger.info('llms.txt file downloaded', {
24
- url,
25
- size: content.length,
26
- });
27
- return content;
28
- }
29
- catch (error) {
30
- const errorMessage = error instanceof Error ? error.message : String(error);
31
- logger.error('Failed to download llms.txt file', {
32
- url,
33
- error: errorMessage,
34
- });
35
- throw new Error(`Failed to download llms.txt from ${url}: ${errorMessage}`);
36
- }
37
- }
38
- /**
39
- * Download multiple llms.txt variants
40
- */
41
- async downloadAll(variants) {
42
- logger.info('Downloading multiple llms.txt variants', {
43
- count: variants.length,
44
- });
45
- const results = [];
46
- for (const variant of variants) {
47
- try {
48
- const content = await this.download(variant.url);
49
- results.push({
50
- url: variant.url,
51
- variant: variant.variant,
52
- content,
53
- size: content.length,
54
- });
55
- }
56
- catch (error) {
57
- logger.warn('Failed to download variant, continuing', {
58
- variant: variant.variant,
59
- error: error instanceof Error ? error.message : String(error),
60
- });
61
- // Continue with other variants even if one fails
62
- }
63
- }
64
- logger.info('Batch download complete', {
65
- successful: results.length,
66
- total: variants.length,
67
- });
68
- return results;
69
- }
70
- /**
71
- * Get proper filename for saving content
72
- * Ensures .md extension for proper markdown rendering
73
- */
74
- getProperFilename(url) {
75
- const urlObj = new URL(url);
76
- const filename = urlObj.pathname.split('/').pop() || 'llms.txt';
77
- // Add .md extension if not present
78
- if (!filename.endsWith('.md')) {
79
- return `${filename}.md`;
80
- }
81
- return filename;
82
- }
83
- }
84
- //# sourceMappingURL=downloader.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"downloader.js","sourceRoot":"","sources":["../../../src/documentation/llms-txt/downloader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACxD,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAU/C;;;GAGG;AACH,MAAM,OAAO,iBAAiB;IACpB,UAAU,CAAa;IAE/B,YAAY,UAAuB;QACjC,IAAI,CAAC,UAAU,GAAG,UAAU,IAAI,IAAI,UAAU,EAAE,CAAC;IACnD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,GAAW;QACxB,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QAElD,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE;gBAC9C,YAAY,EAAE,MAAM;gBACpB,OAAO,EAAE,KAAK,EAAE,oBAAoB;aACrC,CAAC,CAAC;YAEH,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAc,CAAC;YACxC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE;gBACtC,GAAG;gBACH,IAAI,EAAE,OAAO,CAAC,MAAM;aACrB,CAAC,CAAC;YAEH,OAAO,OAAO,CAAC;QACjB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5E,MAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE;gBAC/C,GAAG;gBACH,KAAK,EAAE,YAAY;aACpB,CAAC,CAAC;YACH,MAAM,IAAI,KAAK,CAAC,oCAAoC,GAAG,KAAK,YAAY,EAAE,CAAC,CAAC;QAC9E,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,QAA0B;QAC1C,MAAM,CAAC,IAAI,CAAC,wCAAwC,EAAE;YACpD,KAAK,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;QAEH,MAAM,OAAO,GAAwB,EAAE,CAAC;QAExC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBACjD,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,EAAE,OAAO,CAAC,GAAG;oBAChB,OAAO,EAAE,OAAO,CAAC,OAAO;oBACxB,OAAO;oBACP,IAAI,EAAE,OAAO,CAAC,MAAM;iBACrB,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,IAAI,CAAC,wCAAwC,EAAE;oBACpD,OAAO,EAAE,OAAO,CAAC,OAAO;oBACxB,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;iBAC9D,CAAC,CAAC;gBACH,iDAAiD;YACnD,CAAC;QACH,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE;YACrC,UAAU,EAAE,OAAO,CAAC,MAAM;YAC1B,KAAK,EAAE,QAAQ,CAAC,MAAM;SACvB,CAAC,CAAC;QAEH,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;OAGG;IACH,iBAAiB,CAAC,GAAW;QAC3B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,UAAU,CAAC;QAEhE,mCAAmC;QACnC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,OAAO,GAAG,QAAQ,KAAK,CAAC;QAC1B,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF"}
@@ -1,4 +0,0 @@
1
- export { LlmsTxtDetector, type LlmsTxtVariant } from './detector.js';
2
- export { LlmsTxtDownloader, type DownloadedContent } from './downloader.js';
3
- export { LlmsTxtParser, type ParsedSection } from './parser.js';
4
- //# sourceMappingURL=index.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/documentation/llms-txt/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AACrE,OAAO,EAAE,iBAAiB,EAAE,KAAK,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAC5E,OAAO,EAAE,aAAa,EAAE,KAAK,aAAa,EAAE,MAAM,aAAa,CAAC"}
@@ -1,4 +0,0 @@
1
- export { LlmsTxtDetector } from './detector.js';
2
- export { LlmsTxtDownloader } from './downloader.js';
3
- export { LlmsTxtParser } from './parser.js';
4
- //# sourceMappingURL=index.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/documentation/llms-txt/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAuB,MAAM,eAAe,CAAC;AACrE,OAAO,EAAE,iBAAiB,EAA0B,MAAM,iBAAiB,CAAC;AAC5E,OAAO,EAAE,aAAa,EAAsB,MAAM,aAAa,CAAC"}
@@ -1,43 +0,0 @@
1
- export interface ParsedSection {
2
- title: string;
3
- content: string;
4
- url?: string;
5
- }
6
- /**
7
- * Parser for llms.txt file content
8
- * Extracts URLs and structured content
9
- * Based on Skill_Seekers implementation
10
- */
11
- export declare class LlmsTxtParser {
12
- private content;
13
- private baseUrl;
14
- constructor(content: string, baseUrl: string);
15
- /**
16
- * Extract all URLs from llms.txt content
17
- * Looks for URLs in markdown links and plain text
18
- */
19
- extractUrls(): string[];
20
- /**
21
- * Parse llms.txt into structured sections
22
- * Useful for section-based content extraction
23
- */
24
- parseSections(): ParsedSection[];
25
- /**
26
- * Check if URL is valid for crawling
27
- */
28
- private isValidUrl;
29
- /**
30
- * Normalize URL (make absolute, remove fragments)
31
- */
32
- private normalizeUrl;
33
- /**
34
- * Resolve relative URL to absolute
35
- */
36
- private resolveRelativeUrl;
37
- /**
38
- * Extract summary/description from llms.txt
39
- * Useful for skill metadata
40
- */
41
- extractSummary(maxLength?: number): string;
42
- }
43
- //# sourceMappingURL=parser.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"parser.d.ts","sourceRoot":"","sources":["../../../src/documentation/llms-txt/parser.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED;;;;GAIG;AACH,qBAAa,aAAa;IACxB,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,OAAO,CAAS;gBAEZ,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;IAK5C;;;OAGG;IACH,WAAW,IAAI,MAAM,EAAE;IA2CvB;;;OAGG;IACH,aAAa,IAAI,aAAa,EAAE;IAyDhC;;OAEG;IACH,OAAO,CAAC,UAAU;IASlB;;OAEG;IACH,OAAO,CAAC,YAAY;IAWpB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAW1B;;;OAGG;IACH,cAAc,CAAC,SAAS,GAAE,MAAY,GAAG,MAAM;CAyBhD"}
@@ -1,177 +0,0 @@
1
- import { logger } from '../../utils/logger.js';
2
- /**
3
- * Parser for llms.txt file content
4
- * Extracts URLs and structured content
5
- * Based on Skill_Seekers implementation
6
- */
7
- export class LlmsTxtParser {
8
- content;
9
- baseUrl;
10
- constructor(content, baseUrl) {
11
- this.content = content;
12
- this.baseUrl = baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`;
13
- }
14
- /**
15
- * Extract all URLs from llms.txt content
16
- * Looks for URLs in markdown links and plain text
17
- */
18
- extractUrls() {
19
- const urls = new Set();
20
- // Extract markdown links: [text](url)
21
- const markdownLinkRegex = /\[([^\]]*)\]\(([^)]+)\)/g;
22
- let match;
23
- while ((match = markdownLinkRegex.exec(this.content)) !== null) {
24
- const url = match[2].trim();
25
- if (this.isValidUrl(url)) {
26
- urls.add(this.normalizeUrl(url));
27
- }
28
- }
29
- // Extract plain URLs (http:// or https://)
30
- const plainUrlRegex = /https?:\/\/[^\s<>"{}|\\^`\[\]]+/g;
31
- while ((match = plainUrlRegex.exec(this.content)) !== null) {
32
- const url = match[0].trim();
33
- if (this.isValidUrl(url)) {
34
- urls.add(this.normalizeUrl(url));
35
- }
36
- }
37
- // Extract relative URLs (lines starting with / or ./)
38
- const lines = this.content.split('\n');
39
- for (const line of lines) {
40
- const trimmed = line.trim();
41
- if (trimmed.startsWith('/') || trimmed.startsWith('./')) {
42
- const url = this.resolveRelativeUrl(trimmed);
43
- if (url) {
44
- urls.add(url);
45
- }
46
- }
47
- }
48
- const urlArray = Array.from(urls);
49
- logger.info('Extracted URLs from llms.txt', {
50
- count: urlArray.length,
51
- baseUrl: this.baseUrl,
52
- });
53
- return urlArray;
54
- }
55
- /**
56
- * Parse llms.txt into structured sections
57
- * Useful for section-based content extraction
58
- */
59
- parseSections() {
60
- const sections = [];
61
- const lines = this.content.split('\n');
62
- let currentSection = null;
63
- let currentContent = [];
64
- for (const line of lines) {
65
- const trimmed = line.trim();
66
- // Check if this is a section header (markdown heading)
67
- const headerMatch = trimmed.match(/^(#{1,6})\s+(.+)$/);
68
- if (headerMatch) {
69
- // Save previous section if exists
70
- if (currentSection) {
71
- currentSection.content = currentContent.join('\n').trim();
72
- if (currentSection.content.length > 0) {
73
- sections.push(currentSection);
74
- }
75
- }
76
- // Start new section
77
- currentSection = {
78
- title: headerMatch[2],
79
- content: '',
80
- };
81
- currentContent = [];
82
- }
83
- else if (currentSection) {
84
- currentContent.push(line);
85
- }
86
- else {
87
- // Content before any header - create default section
88
- if (!currentSection) {
89
- currentSection = {
90
- title: 'Introduction',
91
- content: '',
92
- };
93
- currentContent = [];
94
- }
95
- currentContent.push(line);
96
- }
97
- }
98
- // Save last section
99
- if (currentSection) {
100
- currentSection.content = currentContent.join('\n').trim();
101
- if (currentSection.content.length > 0) {
102
- sections.push(currentSection);
103
- }
104
- }
105
- logger.info('Parsed llms.txt into sections', {
106
- sectionCount: sections.length,
107
- });
108
- return sections;
109
- }
110
- /**
111
- * Check if URL is valid for crawling
112
- */
113
- isValidUrl(url) {
114
- try {
115
- const parsed = new URL(url, this.baseUrl);
116
- return parsed.protocol === 'http:' || parsed.protocol === 'https:';
117
- }
118
- catch {
119
- return false;
120
- }
121
- }
122
- /**
123
- * Normalize URL (make absolute, remove fragments)
124
- */
125
- normalizeUrl(url) {
126
- try {
127
- const parsed = new URL(url, this.baseUrl);
128
- // Remove fragment (#anchor)
129
- parsed.hash = '';
130
- return parsed.href;
131
- }
132
- catch {
133
- return url;
134
- }
135
- }
136
- /**
137
- * Resolve relative URL to absolute
138
- */
139
- resolveRelativeUrl(path) {
140
- try {
141
- const url = new URL(path, this.baseUrl);
142
- // Remove fragment
143
- url.hash = '';
144
- return url.href;
145
- }
146
- catch {
147
- return null;
148
- }
149
- }
150
- /**
151
- * Extract summary/description from llms.txt
152
- * Useful for skill metadata
153
- */
154
- extractSummary(maxLength = 500) {
155
- const lines = this.content.split('\n');
156
- const contentLines = [];
157
- for (const line of lines) {
158
- const trimmed = line.trim();
159
- // Skip empty lines and headers
160
- if (trimmed.length === 0 || trimmed.startsWith('#')) {
161
- continue;
162
- }
163
- // Skip URLs
164
- if (trimmed.match(/^https?:\/\//)) {
165
- continue;
166
- }
167
- contentLines.push(trimmed);
168
- const currentLength = contentLines.join(' ').length;
169
- if (currentLength >= maxLength) {
170
- break;
171
- }
172
- }
173
- const summary = contentLines.join(' ').substring(0, maxLength);
174
- return summary.trim();
175
- }
176
- }
177
- //# sourceMappingURL=parser.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"parser.js","sourceRoot":"","sources":["../../../src/documentation/llms-txt/parser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAQ/C;;;;GAIG;AACH,MAAM,OAAO,aAAa;IAChB,OAAO,CAAS;IAChB,OAAO,CAAS;IAExB,YAAY,OAAe,EAAE,OAAe;QAC1C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,GAAG,CAAC;IACjE,CAAC;IAED;;;OAGG;IACH,WAAW;QACT,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAE/B,sCAAsC;QACtC,MAAM,iBAAiB,GAAG,0BAA0B,CAAC;QACrD,IAAI,KAAK,CAAC;QACV,OAAO,CAAC,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC/D,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC5B,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACzB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;QAED,2CAA2C;QAC3C,MAAM,aAAa,GAAG,kCAAkC,CAAC;QACzD,OAAO,CAAC,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC3D,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC5B,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACzB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;QAED,sDAAsD;QACtD,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACvC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxD,MAAM,GAAG,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC;gBAC7C,IAAI,GAAG,EAAE,CAAC;oBACR,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBAChB,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,CAAC,IAAI,CAAC,8BAA8B,EAAE;YAC1C,KAAK,EAAE,QAAQ,CAAC,MAAM;YACtB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QAEH,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;;OAGG;IACH,aAAa;QACX,MAAM,QAAQ,GAAoB,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEvC,IAAI,cAAc,GAAyB,IAAI,CAAC;QAChD,IAAI,cAAc,GAAa,EAAE,CAAC;QAElC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAE5B,uDAAuD;YACvD,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;YACvD,IAAI,WAAW,EAAE,CAAC;gBAChB,kCAAkC;gBAClC,IAAI,cAAc,EAAE,CAAC;oBACnB,cAAc,CAAC,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;oBAC1D,IAAI,cAAc,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBACtC,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;oBAChC,CAAC;gBACH,CAAC;gBAED,oBAAoB;gBACpB,cAAc,GAAG;oBACf,KAAK,EAAE,WAAW,CAAC,CAAC,CAAC;oBACrB,OAAO,EAAE,EAAE;iBACZ,CAAC;gBACF,cAAc,GAAG,EAAE,CAAC;YACtB,CAAC;iBAAM,IAAI,cAAc,EAAE,CAAC;gBAC1B,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5B,CAAC;iBAAM,CAAC;gBACN,qDAAqD;gBACrD,IAAI,CAAC,cAAc,EAAE,CAAC;oBACpB,cAAc,GAAG;wBACf,KAAK,EAAE,cAAc;wBACrB,OAAO,EAAE,EAAE;qBACZ,CAAC;oBACF,cAAc,GAAG,EAAE,CAAC;gBACtB,CAAC;gBACD,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,oBAAoB;QACpB,IAAI,cAAc,EAAE,CAAC;YACnB,cAAc,CAAC,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;YAC1D,IAAI,cAAc,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtC,QAAQ,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE;YAC3C,YAAY,EAAE,QAAQ,CAAC,MAAM;SAC9B,CAAC,CAAC;QAEH,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,GAAW;QAC5B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YAC1C,OAAO,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,CAAC;QACrE,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED;;OAEG;IACK,YAAY,CAAC,GAAW;QAC9B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YAC1C,4BAA4B;YAC5B,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;YACjB,OAAO,MAAM,CAAC,IAAI,CAAC;QACrB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,GAAG,CAAC;QACb,CAAC;IACH,CAAC;IAED;;OAEG;IACK,kBAAkB,CAAC,IAAY;QACrC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YACxC,kBAAkB;YAClB,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC;YACd,OAAO,GAAG,CAAC,IAAI,CAAC;QAClB,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,cAAc,CAAC,YAAoB,GAAG;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,YAAY,GAAa,EAAE,CAAC;QAElC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,+BAA+B;YAC/B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACpD,SAAS;YACX,CAAC;YACD,YAAY;YACZ,IAAI,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;gBAClC,SAAS;YACX,CAAC;YAED,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC3B,MAAM,aAAa,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;YACpD,IAAI,aAAa,IAAI,SAAS,EAAE,CAAC;gBAC/B,MAAM;YACR,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAC/D,OAAO,OAAO,CAAC,IAAI,EAAE,CAAC;IACxB,CAAC;CACF"}
@@ -1,6 +0,0 @@
1
- import type { Documentation, IDocumentationNormalizer } from '../types/documentation.js';
2
- export declare class DocumentationNormalizer implements IDocumentationNormalizer {
3
- normalize(content: string, source: 'npm' | 'github' | 'http'): Documentation;
4
- private extractSections;
5
- }
6
- //# sourceMappingURL=normalizer.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"normalizer.d.ts","sourceRoot":"","sources":["../../src/documentation/normalizer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,wBAAwB,EAAE,MAAM,2BAA2B,CAAC;AAEzF,qBAAa,uBAAwB,YAAW,wBAAwB;IACtE,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,aAAa;IAa5E,OAAO,CAAC,eAAe;CA6BxB"}
@@ -1,38 +0,0 @@
1
- export class DocumentationNormalizer {
2
- normalize(content, source) {
3
- // Extract sections from markdown content
4
- const sections = this.extractSections(content);
5
- return {
6
- name: '',
7
- content,
8
- source,
9
- lastUpdated: new Date(),
10
- sections,
11
- };
12
- }
13
- extractSections(content) {
14
- const sections = {};
15
- // Extract README
16
- const readmeMatch = content.match(/#\s+(.+?)(?=\n|$)/);
17
- if (readmeMatch) {
18
- sections.readme = content;
19
- }
20
- // Extract API section
21
- const apiMatch = content.match(/##?\s+API[^#]*/is);
22
- if (apiMatch) {
23
- sections.api = apiMatch[0];
24
- }
25
- // Extract Examples section
26
- const examplesMatch = content.match(/##?\s+Examples?[^#]*/is);
27
- if (examplesMatch) {
28
- sections.examples = examplesMatch[0];
29
- }
30
- // Extract Installation section
31
- const installMatch = content.match(/##?\s+Installation[^#]*/is);
32
- if (installMatch) {
33
- sections.installation = installMatch[0];
34
- }
35
- return sections;
36
- }
37
- }
38
- //# sourceMappingURL=normalizer.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"normalizer.js","sourceRoot":"","sources":["../../src/documentation/normalizer.ts"],"names":[],"mappings":"AAEA,MAAM,OAAO,uBAAuB;IAClC,SAAS,CAAC,OAAe,EAAE,MAAiC;QAC1D,yCAAyC;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QAE/C,OAAO;YACL,IAAI,EAAE,EAAE;YACR,OAAO;YACP,MAAM;YACN,WAAW,EAAE,IAAI,IAAI,EAAE;YACvB,QAAQ;SACT,CAAC;IACJ,CAAC;IAEO,eAAe,CAAC,OAAe;QACrC,MAAM,QAAQ,GAA8B,EAAE,CAAC;QAE/C,iBAAiB;QACjB,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;QACvD,IAAI,WAAW,EAAE,CAAC;YAChB,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC;QAC5B,CAAC;QAED,sBAAsB;QACtB,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACnD,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,GAAG,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;QAED,2BAA2B;QAC3B,MAAM,aAAa,GAAG,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAC9D,IAAI,aAAa,EAAE,CAAC;YAClB,QAAQ,CAAC,QAAQ,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QACvC,CAAC;QAED,+BAA+B;QAC/B,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;QAChE,IAAI,YAAY,EAAE,CAAC;YACjB,QAAQ,CAAC,YAAY,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;QAC1C,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF"}
@@ -1,19 +0,0 @@
1
- import type { Documentation, DocumentationSource } from '../types/documentation.js';
2
- import { HttpClient } from '../utils/http-client.js';
3
- export declare class NpmRegistryClient implements DocumentationSource {
4
- private readonly registryUrl;
5
- private httpClient;
6
- constructor(httpClient?: HttpClient);
7
- getPackageMetadata(packageName: string): Promise<{
8
- repository?: {
9
- url?: string;
10
- type?: string;
11
- };
12
- homepage?: string;
13
- } | null>;
14
- fetch(packageName: string, version?: string): Promise<Documentation>;
15
- private fetchFromRepository;
16
- private fetchFromHomepage;
17
- private generateFallbackMessage;
18
- }
19
- //# sourceMappingURL=npm-client.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"npm-client.d.ts","sourceRoot":"","sources":["../../src/documentation/npm-client.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AACpF,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAErD,qBAAa,iBAAkB,YAAW,mBAAmB;IAC3D,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAgC;IAC5D,OAAO,CAAC,UAAU,CAAa;gBAEnB,UAAU,CAAC,EAAE,UAAU;IAK7B,kBAAkB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC;QACrD,UAAU,CAAC,EAAE;YAAE,GAAG,CAAC,EAAE,MAAM,CAAC;YAAC,IAAI,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC7C,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,GAAG,IAAI,CAAC;IAqBH,KAAK,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;YAmE5D,mBAAmB;YAoCnB,iBAAiB;IAsC/B,OAAO,CAAC,uBAAuB;CAmChC"}
@@ -1,182 +0,0 @@
1
- import { HttpClient } from '../utils/http-client.js';
2
- export class NpmRegistryClient {
3
- registryUrl = 'https://registry.npmjs.org';
4
- httpClient;
5
- constructor(httpClient) {
6
- this.httpClient = httpClient || new HttpClient();
7
- }
8
- // Expose method to get package metadata for fallback strategies
9
- async getPackageMetadata(packageName) {
10
- try {
11
- const packageUrl = `${this.registryUrl}/${packageName}`;
12
- const packageResponse = await this.httpClient.get(packageUrl);
13
- const packageData = packageResponse.data;
14
- const latestVersion = packageData['dist-tags']?.latest;
15
- const versionData = packageData.versions[latestVersion];
16
- if (!versionData) {
17
- return null;
18
- }
19
- return {
20
- repository: versionData.repository,
21
- homepage: versionData.homepage,
22
- };
23
- }
24
- catch (error) {
25
- return null;
26
- }
27
- }
28
- async fetch(packageName, version) {
29
- try {
30
- // Fetch package metadata
31
- const packageUrl = `${this.registryUrl}/${packageName}`;
32
- const packageResponse = await this.httpClient.get(packageUrl);
33
- const packageData = packageResponse.data;
34
- // Get latest version from dist-tags (always up-to-date from registry)
35
- const latestVersion = packageData['dist-tags']?.latest;
36
- const targetVersion = version || latestVersion || 'latest';
37
- const versionData = packageData.versions[targetVersion] || packageData.versions[latestVersion];
38
- if (!versionData) {
39
- throw new Error(`Version ${targetVersion} not found for package ${packageName}`);
40
- }
41
- // Get package publish time from npm registry's time field
42
- // This helps determine if package was updated
43
- const packagePublishTime = packageData.time?.[targetVersion]
44
- ? new Date(packageData.time[targetVersion])
45
- : new Date();
46
- // Try to get README from package data
47
- let content = versionData.readme || '';
48
- const repository = versionData.repository;
49
- const homepage = versionData.homepage;
50
- let sourceUrl = repository?.url || homepage;
51
- // Fallback strategy: If no README in package, try multiple sources
52
- if (!content || content.trim().length < 100) {
53
- // Strategy 1: Try fetching from GitHub repository
54
- if (repository) {
55
- const repoContent = await this.fetchFromRepository(repository, version);
56
- if (repoContent) {
57
- content = repoContent;
58
- }
59
- }
60
- // Strategy 2: If still no content, try homepage
61
- if ((!content || content.trim().length < 100) && homepage) {
62
- const homepageContent = await this.fetchFromHomepage(homepage);
63
- if (homepageContent) {
64
- content = homepageContent;
65
- sourceUrl = homepage;
66
- }
67
- }
68
- }
69
- // If still no content, provide helpful error message with available links
70
- if (!content || content.trim().length < 100) {
71
- const fallbackMessage = this.generateFallbackMessage(packageName, targetVersion, repository, homepage);
72
- content = fallbackMessage;
73
- }
74
- return {
75
- name: packageName,
76
- version: targetVersion,
77
- content: content,
78
- source: 'npm',
79
- url: sourceUrl,
80
- lastUpdated: packagePublishTime, // Use actual publish time instead of current time
81
- };
82
- }
83
- catch (error) {
84
- throw new Error(`Failed to fetch npm documentation for ${packageName}: ${error instanceof Error ? error.message : 'Unknown error'}`);
85
- }
86
- }
87
- async fetchFromRepository(repo, version) {
88
- if (!repo.url || repo.type !== 'git') {
89
- return '';
90
- }
91
- try {
92
- // Extract GitHub URL
93
- const githubMatch = repo.url.match(/github\.com[\/:]([^\/]+)\/([^\/]+?)(?:\.git)?$/);
94
- if (githubMatch) {
95
- const [, owner, repoName] = githubMatch;
96
- // Try to fetch README from GitHub API
97
- const ref = version ? `?ref=${version}` : '';
98
- const url = `https://api.github.com/repos/${owner}/${repoName}/readme${ref}`;
99
- try {
100
- const response = await this.httpClient.get(url, {
101
- headers: {
102
- 'Accept': 'application/vnd.github.v3.raw',
103
- },
104
- timeout: 5000,
105
- });
106
- return response.data || '';
107
- }
108
- catch (error) {
109
- // GitHub fetch failed, return empty to try next strategy
110
- return '';
111
- }
112
- }
113
- // Try other git hosting services (GitLab, Bitbucket, etc.)
114
- // For now, return empty and let other strategies handle it
115
- return '';
116
- }
117
- catch (error) {
118
- return '';
119
- }
120
- }
121
- async fetchFromHomepage(homepage) {
122
- if (!homepage || !homepage.startsWith('http')) {
123
- return '';
124
- }
125
- try {
126
- const response = await this.httpClient.get(homepage, {
127
- timeout: 5000,
128
- headers: {
129
- 'User-Agent': 'Safe-Coder-MCP/1.0.0',
130
- },
131
- });
132
- // Try to extract meaningful content from HTML
133
- const html = response.data;
134
- if (typeof html === 'string') {
135
- // Simple extraction: try to get text content
136
- // Remove scripts and styles
137
- let text = html
138
- .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
139
- .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
140
- .replace(/<[^>]+>/g, ' ')
141
- .replace(/\s+/g, ' ')
142
- .trim();
143
- // If we got reasonable content, return it
144
- if (text.length > 200) {
145
- return `# Documentation from Homepage\n\n${text.substring(0, 5000)}...`;
146
- }
147
- }
148
- return '';
149
- }
150
- catch (error) {
151
- // Homepage fetch failed
152
- return '';
153
- }
154
- }
155
- generateFallbackMessage(packageName, version, repository, homepage) {
156
- const lines = [
157
- `# ${packageName}@${version}`,
158
- '',
159
- '⚠️ No documentation found in npm package.',
160
- '',
161
- '**Available resources:**',
162
- ];
163
- if (repository?.url) {
164
- const repoUrl = repository.url.replace(/\.git$/, '');
165
- lines.push(`- 📦 Repository: ${repoUrl}`);
166
- if (repoUrl.includes('github.com')) {
167
- lines.push(` - Try: \`get_documentation("${repoUrl.replace('https://github.com/', '')}", source="github")\``);
168
- }
169
- }
170
- if (homepage) {
171
- lines.push(`- 🌐 Homepage: ${homepage}`);
172
- lines.push(` - Try: \`get_documentation("${homepage}", source="http")\``);
173
- }
174
- lines.push('');
175
- lines.push('**Suggestions:**');
176
- lines.push('1. Check the package repository for README.md');
177
- lines.push('2. Visit the package homepage');
178
- lines.push('3. Check npm package page: https://www.npmjs.com/package/' + packageName);
179
- return lines.join('\n');
180
- }
181
- }
182
- //# sourceMappingURL=npm-client.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"npm-client.js","sourceRoot":"","sources":["../../src/documentation/npm-client.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAErD,MAAM,OAAO,iBAAiB;IACX,WAAW,GAAG,4BAA4B,CAAC;IACpD,UAAU,CAAa;IAE/B,YAAY,UAAuB;QACjC,IAAI,CAAC,UAAU,GAAG,UAAU,IAAI,IAAI,UAAU,EAAE,CAAC;IACnD,CAAC;IAED,gEAAgE;IAChE,KAAK,CAAC,kBAAkB,CAAC,WAAmB;QAI1C,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,GAAG,IAAI,CAAC,WAAW,IAAI,WAAW,EAAE,CAAC;YACxD,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC9D,MAAM,WAAW,GAAG,eAAe,CAAC,IAAI,CAAC;YACzC,MAAM,aAAa,GAAG,WAAW,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;YACvD,MAAM,WAAW,GAAG,WAAW,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;YAExD,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,OAAO,IAAI,CAAC;YACd,CAAC;YAED,OAAO;gBACL,UAAU,EAAE,WAAW,CAAC,UAAU;gBAClC,QAAQ,EAAE,WAAW,CAAC,QAAQ;aAC/B,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,WAAmB,EAAE,OAAgB;QAC/C,IAAI,CAAC;YACH,yBAAyB;YACzB,MAAM,UAAU,GAAG,GAAG,IAAI,CAAC,WAAW,IAAI,WAAW,EAAE,CAAC;YACxD,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC9D,MAAM,WAAW,GAAG,eAAe,CAAC,IAAI,CAAC;YAEzC,sEAAsE;YACtE,MAAM,aAAa,GAAG,WAAW,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;YACvD,MAAM,aAAa,GAAG,OAAO,IAAI,aAAa,IAAI,QAAQ,CAAC;YAC3D,MAAM,WAAW,GAAG,WAAW,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;YAE/F,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,WAAW,aAAa,0BAA0B,WAAW,EAAE,CAAC,CAAC;YACnF,CAAC;YAED,0DAA0D;YAC1D,8CAA8C;YAC9C,MAAM,kBAAkB,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC,aAAa,CAAC;gBAC1D,CAAC,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;gBAC3C,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;YAEf,sCAAsC;YACtC,IAAI,OAAO,GAAG,WAAW,CAAC,MAAM,IAAI,EAAE,CAAC;YACvC,MAAM,UAAU,GAAG,WAAW,CAAC,UAAU,CAAC;YAC1C,MAAM,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAC;YACtC,IAAI,SAAS,GAAG,UAAU,EAAE,GAAG,IAAI,QAAQ,CAAC;YAE5C,mEAAmE;YACnE,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAC5C,kDAAkD;gBAClD,IAAI,UAAU,EAAE,CAAC;oBACf,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;oBACxE,IAAI,WAAW,EAAE,CAAC;wBAChB,OAAO,GAAG,WAAW,CAAC;oBACxB,CAAC;gBACH,CAAC;gBAED,gDAAgD;gBAChD,IAAI,CAAC,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,GAAG,CAAC,IAAI,QAAQ,EAAE,CAAC;oBAC1D,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC;oBAC/D,IAAI,eAAe,EAAE,CAAC;wBACpB,OAAO,GAAG,eAAe,CAAC;wBAC1B,SAAS,GAAG,QAAQ,CAAC;oBACvB,CAAC;gBACH,CAAC;YACH,CAAC;YAED,0EAA0E;YAC1E,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAC5C,MAAM,eAAe,GAAG,IAAI,CAAC,uBAAuB,CAAC,WAAW,EAAE,aAAa,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;gBACvG,OAAO,GAAG,eAAe,CAAC;YAC5B,CAAC;YAED,OAAO;gBACL,IAAI,EAAE,WAAW;gBACjB,OAAO,EAAE,aAAa;gBACtB,OAAO,EAAE,OAAO;gBAChB,MAAM,EAAE,KAAK;gBACb,GAAG,EAAE,SAAS;gBACd,WAAW,EAAE,kBAAkB,EAAE,kDAAkD;aACpF,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,yCAAyC,WAAW,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QACvI,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,IAAqC,EAAE,OAAgB;QACvF,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YACrC,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,IAAI,CAAC;YACH,qBAAqB;YACrB,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;YACrF,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,GAAG,WAAW,CAAC;gBACxC,sCAAsC;gBACtC,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,QAAQ,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7C,MAAM,GAAG,GAAG,gCAAgC,KAAK,IAAI,QAAQ,UAAU,GAAG,EAAE,CAAC;gBAE7E,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE;wBAC9C,OAAO,EAAE;4BACP,QAAQ,EAAE,+BAA+B;yBAC1C;wBACD,OAAO,EAAE,IAAI;qBACd,CAAC,CAAC;oBACH,OAAO,QAAQ,CAAC,IAAI,IAAI,EAAE,CAAC;gBAC7B,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,yDAAyD;oBACzD,OAAO,EAAE,CAAC;gBACZ,CAAC;YACH,CAAC;YAED,2DAA2D;YAC3D,2DAA2D;YAC3D,OAAO,EAAE,CAAC;QACZ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,QAAgB;QAC9C,IAAI,CAAC,QAAQ,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9C,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE;gBACnD,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE;oBACP,YAAY,EAAE,sBAAsB;iBACrC;aACF,CAAC,CAAC;YAEH,8CAA8C;YAC9C,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;YAC3B,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC7B,6CAA6C;gBAC7C,4BAA4B;gBAC5B,IAAI,IAAI,GAAG,IAAI;qBACZ,OAAO,CAAC,mCAAmC,EAAE,EAAE,CAAC;qBAChD,OAAO,CAAC,iCAAiC,EAAE,EAAE,CAAC;qBAC9C,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;qBACxB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;qBACpB,IAAI,EAAE,CAAC;gBAEV,0CAA0C;gBAC1C,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;oBACtB,OAAO,oCAAoC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC;gBAC1E,CAAC;YACH,CAAC;YAED,OAAO,EAAE,CAAC;QACZ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,wBAAwB;YACxB,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,uBAAuB,CAC7B,WAAmB,EACnB,OAAe,EACf,UAA4C,EAC5C,QAAiB;QAEjB,MAAM,KAAK,GAAG;YACZ,KAAK,WAAW,IAAI,OAAO,EAAE;YAC7B,EAAE;YACF,2CAA2C;YAC3C,EAAE;YACF,0BAA0B;SAC3B,CAAC;QAEF,IAAI,UAAU,EAAE,GAAG,EAAE,CAAC;YACpB,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;YACrD,KAAK,CAAC,IAAI,CAAC,oBAAoB,OAAO,EAAE,CAAC,CAAC;YAC1C,IAAI,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBACnC,KAAK,CAAC,IAAI,CAAC,iCAAiC,OAAO,CAAC,OAAO,CAAC,qBAAqB,EAAE,EAAE,CAAC,uBAAuB,CAAC,CAAC;YACjH,CAAC;QACH,CAAC;QAED,IAAI,QAAQ,EAAE,CAAC;YACb,KAAK,CAAC,IAAI,CAAC,kBAAkB,QAAQ,EAAE,CAAC,CAAC;YACzC,KAAK,CAAC,IAAI,CAAC,iCAAiC,QAAQ,qBAAqB,CAAC,CAAC;QAC7E,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QAC5C,KAAK,CAAC,IAAI,CAAC,2DAA2D,GAAG,WAAW,CAAC,CAAC;QAEtF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;CACF"}