@gulibs/safe-coder 0.0.26 → 0.0.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (192) hide show
  1. package/README.md +678 -994
  2. package/dist/cache/cache-manager.d.ts +71 -0
  3. package/dist/cache/cache-manager.d.ts.map +1 -0
  4. package/dist/cache/cache-manager.js +244 -0
  5. package/dist/cache/cache-manager.js.map +1 -0
  6. package/dist/executor/cli-executor.d.ts +106 -0
  7. package/dist/executor/cli-executor.d.ts.map +1 -0
  8. package/dist/executor/cli-executor.js +133 -0
  9. package/dist/executor/cli-executor.js.map +1 -0
  10. package/dist/executor/dependency-checker.d.ts +23 -0
  11. package/dist/executor/dependency-checker.d.ts.map +1 -0
  12. package/dist/executor/dependency-checker.js +62 -0
  13. package/dist/executor/dependency-checker.js.map +1 -0
  14. package/dist/index.js +3 -4
  15. package/dist/index.js.map +1 -1
  16. package/dist/processor/content-processor.d.ts +76 -0
  17. package/dist/processor/content-processor.d.ts.map +1 -0
  18. package/dist/processor/content-processor.js +182 -0
  19. package/dist/processor/content-processor.js.map +1 -0
  20. package/dist/processor/guide-generator.d.ts +68 -0
  21. package/dist/processor/guide-generator.d.ts.map +1 -0
  22. package/dist/processor/guide-generator.js +189 -0
  23. package/dist/processor/guide-generator.js.map +1 -0
  24. package/dist/server/safe-coder-mcp.d.ts +18 -0
  25. package/dist/server/safe-coder-mcp.d.ts.map +1 -0
  26. package/dist/server/safe-coder-mcp.js +164 -0
  27. package/dist/server/safe-coder-mcp.js.map +1 -0
  28. package/dist/tools/cache-tools.d.ts +42 -0
  29. package/dist/tools/cache-tools.d.ts.map +1 -0
  30. package/dist/tools/cache-tools.js +70 -0
  31. package/dist/tools/cache-tools.js.map +1 -0
  32. package/dist/tools/crawl-documentation.d.ts +57 -0
  33. package/dist/tools/crawl-documentation.d.ts.map +1 -0
  34. package/dist/tools/crawl-documentation.js +96 -0
  35. package/dist/tools/crawl-documentation.js.map +1 -0
  36. package/dist/tools/index.d.ts +4 -0
  37. package/dist/tools/index.d.ts.map +1 -0
  38. package/dist/tools/index.js +4 -0
  39. package/dist/tools/index.js.map +1 -0
  40. package/dist/tools/save-skill.d.ts +49 -0
  41. package/dist/tools/save-skill.d.ts.map +1 -0
  42. package/dist/tools/save-skill.js +207 -0
  43. package/dist/tools/save-skill.js.map +1 -0
  44. package/package.json +18 -28
  45. package/dist/documentation/browser-doc-browser.d.ts +0 -41
  46. package/dist/documentation/browser-doc-browser.d.ts.map +0 -1
  47. package/dist/documentation/browser-doc-browser.js +0 -357
  48. package/dist/documentation/browser-doc-browser.js.map +0 -1
  49. package/dist/documentation/browser-manager.d.ts +0 -51
  50. package/dist/documentation/browser-manager.d.ts.map +0 -1
  51. package/dist/documentation/browser-manager.js +0 -260
  52. package/dist/documentation/browser-manager.js.map +0 -1
  53. package/dist/documentation/cache.d.ts +0 -13
  54. package/dist/documentation/cache.d.ts.map +0 -1
  55. package/dist/documentation/cache.js +0 -48
  56. package/dist/documentation/cache.js.map +0 -1
  57. package/dist/documentation/checkpoint-manager.d.ts +0 -38
  58. package/dist/documentation/checkpoint-manager.d.ts.map +0 -1
  59. package/dist/documentation/checkpoint-manager.js +0 -101
  60. package/dist/documentation/checkpoint-manager.js.map +0 -1
  61. package/dist/documentation/doc-crawler.d.ts +0 -221
  62. package/dist/documentation/doc-crawler.d.ts.map +0 -1
  63. package/dist/documentation/doc-crawler.js +0 -1415
  64. package/dist/documentation/doc-crawler.js.map +0 -1
  65. package/dist/documentation/github-client.d.ts +0 -13
  66. package/dist/documentation/github-client.d.ts.map +0 -1
  67. package/dist/documentation/github-client.js +0 -90
  68. package/dist/documentation/github-client.js.map +0 -1
  69. package/dist/documentation/http-fetcher.d.ts +0 -8
  70. package/dist/documentation/http-fetcher.d.ts.map +0 -1
  71. package/dist/documentation/http-fetcher.js +0 -31
  72. package/dist/documentation/http-fetcher.js.map +0 -1
  73. package/dist/documentation/index.d.ts +0 -16
  74. package/dist/documentation/index.d.ts.map +0 -1
  75. package/dist/documentation/index.js +0 -159
  76. package/dist/documentation/index.js.map +0 -1
  77. package/dist/documentation/llms-txt/detector.d.ts +0 -31
  78. package/dist/documentation/llms-txt/detector.d.ts.map +0 -1
  79. package/dist/documentation/llms-txt/detector.js +0 -77
  80. package/dist/documentation/llms-txt/detector.js.map +0 -1
  81. package/dist/documentation/llms-txt/downloader.d.ts +0 -30
  82. package/dist/documentation/llms-txt/downloader.d.ts.map +0 -1
  83. package/dist/documentation/llms-txt/downloader.js +0 -84
  84. package/dist/documentation/llms-txt/downloader.js.map +0 -1
  85. package/dist/documentation/llms-txt/index.d.ts +0 -4
  86. package/dist/documentation/llms-txt/index.d.ts.map +0 -1
  87. package/dist/documentation/llms-txt/index.js +0 -4
  88. package/dist/documentation/llms-txt/index.js.map +0 -1
  89. package/dist/documentation/llms-txt/parser.d.ts +0 -43
  90. package/dist/documentation/llms-txt/parser.d.ts.map +0 -1
  91. package/dist/documentation/llms-txt/parser.js +0 -177
  92. package/dist/documentation/llms-txt/parser.js.map +0 -1
  93. package/dist/documentation/normalizer.d.ts +0 -6
  94. package/dist/documentation/normalizer.d.ts.map +0 -1
  95. package/dist/documentation/normalizer.js +0 -38
  96. package/dist/documentation/normalizer.js.map +0 -1
  97. package/dist/documentation/npm-client.d.ts +0 -19
  98. package/dist/documentation/npm-client.d.ts.map +0 -1
  99. package/dist/documentation/npm-client.js +0 -182
  100. package/dist/documentation/npm-client.js.map +0 -1
  101. package/dist/documentation/skill-generator.d.ts +0 -108
  102. package/dist/documentation/skill-generator.d.ts.map +0 -1
  103. package/dist/documentation/skill-generator.js +0 -642
  104. package/dist/documentation/skill-generator.js.map +0 -1
  105. package/dist/documentation/web-doc-browser.d.ts +0 -67
  106. package/dist/documentation/web-doc-browser.d.ts.map +0 -1
  107. package/dist/documentation/web-doc-browser.js +0 -555
  108. package/dist/documentation/web-doc-browser.js.map +0 -1
  109. package/dist/errors/api-validator.d.ts +0 -9
  110. package/dist/errors/api-validator.d.ts.map +0 -1
  111. package/dist/errors/api-validator.js +0 -57
  112. package/dist/errors/api-validator.js.map +0 -1
  113. package/dist/errors/contextual-analysis.d.ts +0 -14
  114. package/dist/errors/contextual-analysis.d.ts.map +0 -1
  115. package/dist/errors/contextual-analysis.js +0 -173
  116. package/dist/errors/contextual-analysis.js.map +0 -1
  117. package/dist/errors/cross-file-analyzer.d.ts +0 -16
  118. package/dist/errors/cross-file-analyzer.d.ts.map +0 -1
  119. package/dist/errors/cross-file-analyzer.js +0 -172
  120. package/dist/errors/cross-file-analyzer.js.map +0 -1
  121. package/dist/errors/eslint-integration.d.ts +0 -9
  122. package/dist/errors/eslint-integration.d.ts.map +0 -1
  123. package/dist/errors/eslint-integration.js +0 -131
  124. package/dist/errors/eslint-integration.js.map +0 -1
  125. package/dist/errors/framework-detector.d.ts +0 -10
  126. package/dist/errors/framework-detector.d.ts.map +0 -1
  127. package/dist/errors/framework-detector.js +0 -126
  128. package/dist/errors/framework-detector.js.map +0 -1
  129. package/dist/errors/index.d.ts +0 -18
  130. package/dist/errors/index.d.ts.map +0 -1
  131. package/dist/errors/index.js +0 -134
  132. package/dist/errors/index.js.map +0 -1
  133. package/dist/errors/pattern-matcher.d.ts +0 -25
  134. package/dist/errors/pattern-matcher.d.ts.map +0 -1
  135. package/dist/errors/pattern-matcher.js +0 -44
  136. package/dist/errors/pattern-matcher.js.map +0 -1
  137. package/dist/errors/patterns.d.ts +0 -11
  138. package/dist/errors/patterns.d.ts.map +0 -1
  139. package/dist/errors/patterns.js +0 -351
  140. package/dist/errors/patterns.js.map +0 -1
  141. package/dist/errors/performance-detector.d.ts +0 -11
  142. package/dist/errors/performance-detector.d.ts.map +0 -1
  143. package/dist/errors/performance-detector.js +0 -119
  144. package/dist/errors/performance-detector.js.map +0 -1
  145. package/dist/errors/runtime-detector.d.ts +0 -7
  146. package/dist/errors/runtime-detector.d.ts.map +0 -1
  147. package/dist/errors/runtime-detector.js +0 -86
  148. package/dist/errors/runtime-detector.js.map +0 -1
  149. package/dist/errors/security-detector.d.ts +0 -6
  150. package/dist/errors/security-detector.d.ts.map +0 -1
  151. package/dist/errors/security-detector.js +0 -75
  152. package/dist/errors/security-detector.js.map +0 -1
  153. package/dist/errors/typescript-integration.d.ts +0 -6
  154. package/dist/errors/typescript-integration.d.ts.map +0 -1
  155. package/dist/errors/typescript-integration.js +0 -46
  156. package/dist/errors/typescript-integration.js.map +0 -1
  157. package/dist/server/mcp-server.d.ts +0 -14
  158. package/dist/server/mcp-server.d.ts.map +0 -1
  159. package/dist/server/mcp-server.js +0 -793
  160. package/dist/server/mcp-server.js.map +0 -1
  161. package/dist/types/documentation.d.ts +0 -26
  162. package/dist/types/documentation.d.ts.map +0 -1
  163. package/dist/types/documentation.js +0 -2
  164. package/dist/types/documentation.js.map +0 -1
  165. package/dist/utils/config.d.ts +0 -21
  166. package/dist/utils/config.d.ts.map +0 -1
  167. package/dist/utils/config.js +0 -34
  168. package/dist/utils/config.js.map +0 -1
  169. package/dist/utils/http-client.d.ts +0 -17
  170. package/dist/utils/http-client.d.ts.map +0 -1
  171. package/dist/utils/http-client.js +0 -62
  172. package/dist/utils/http-client.js.map +0 -1
  173. package/dist/utils/logger.d.ts +0 -36
  174. package/dist/utils/logger.d.ts.map +0 -1
  175. package/dist/utils/logger.js +0 -128
  176. package/dist/utils/logger.js.map +0 -1
  177. package/dist/utils/rate-limiter.d.ts +0 -9
  178. package/dist/utils/rate-limiter.d.ts.map +0 -1
  179. package/dist/utils/rate-limiter.js +0 -26
  180. package/dist/utils/rate-limiter.js.map +0 -1
  181. package/dist/validation/auto-fix.d.ts +0 -15
  182. package/dist/validation/auto-fix.d.ts.map +0 -1
  183. package/dist/validation/auto-fix.js +0 -49
  184. package/dist/validation/auto-fix.js.map +0 -1
  185. package/dist/validation/index.d.ts +0 -21
  186. package/dist/validation/index.d.ts.map +0 -1
  187. package/dist/validation/index.js +0 -45
  188. package/dist/validation/index.js.map +0 -1
  189. package/dist/validation/resolution-db.d.ts +0 -15
  190. package/dist/validation/resolution-db.d.ts.map +0 -1
  191. package/dist/validation/resolution-db.js +0 -62
  192. package/dist/validation/resolution-db.js.map +0 -1
@@ -1,642 +0,0 @@
1
- import { writeFile, mkdir, access, constants } from 'fs/promises';
2
- import { join, resolve, basename } from 'path';
3
- export class SkillGenerator {
4
- CONTENT_PREVIEW_LENGTH = 500;
5
- MIN_CATEGORIZATION_SCORE = 2;
6
- MIN_CONTENT_LENGTH = 100; // Minimum characters for valid content
7
- MIN_HEADINGS_COUNT = 1; // Minimum headings for structured content
8
- /**
9
- * Generate agent skill from crawled documentation
10
- */
11
- generateSkill(crawlResult, rootUrl, skillName) {
12
- const { pages, totalPages, maxDepthReached, errors } = crawlResult;
13
- if (pages.length === 0) {
14
- throw new Error('No pages were successfully crawled');
15
- }
16
- // Validate if content is sufficient for skill generation
17
- const validation = this.canGenerateSkill(pages);
18
- if (!validation.canGenerate) {
19
- throw new Error(`Cannot generate skill: ${validation.reason}. ` +
20
- `Pages crawled: ${totalPages}, but content is insufficient. ` +
21
- `Consider crawling more pages or a different website.`);
22
- }
23
- // Generate skill name
24
- const name = skillName || this.generateSkillName(rootUrl, pages[0]);
25
- // Generate title from root URL or first page
26
- const title = this.extractTitle(rootUrl, pages[0]);
27
- // Generate description (truncated to 1024 chars)
28
- const description = this.generateDescription(rootUrl, pages, totalPages, maxDepthReached);
29
- const truncatedDescription = description.length > 1024 ? description.substring(0, 1021) + '...' : description;
30
- // Validate description quality
31
- this.validateDescription(truncatedDescription);
32
- // Categorize pages
33
- const categories = this.categorizePages(pages);
34
- // Generate reference files
35
- const referenceFiles = this.generateReferenceFiles(categories, name);
36
- // Generate SKILL.md with YAML frontmatter
37
- const skillMd = this.generateSkillMd(name, truncatedDescription, categories, pages);
38
- // Generate metadata
39
- const metadata = {
40
- title,
41
- description: truncatedDescription,
42
- sourceUrl: rootUrl,
43
- pagesCrawled: totalPages,
44
- maxDepthReached,
45
- generatedAt: new Date().toISOString(),
46
- sourceUrls: pages.map(p => p.url),
47
- };
48
- return {
49
- skillMd,
50
- referenceFiles,
51
- metadata,
52
- skillName: name,
53
- };
54
- }
55
- /**
56
- * Generate skill name from URL or provided name (sanitized)
57
- */
58
- generateSkillName(rootUrl, firstPage) {
59
- try {
60
- const url = new URL(rootUrl);
61
- const hostname = url.hostname.replace('www.', '').replace(/\./g, '-');
62
- const pathParts = url.pathname.split('/').filter(p => p && p !== 'docs' && p !== 'documentation');
63
- const pathPart = pathParts.length > 0 ? pathParts[pathParts.length - 1] : 'docs';
64
- // Sanitize: lowercase, hyphens only, max 64 chars
65
- const sanitized = `${hostname}-${pathPart}`
66
- .toLowerCase()
67
- .replace(/[^a-z0-9-]/g, '-')
68
- .replace(/-+/g, '-')
69
- .replace(/^-|-$/g, '')
70
- .substring(0, 64);
71
- return sanitized || 'documentation-skill';
72
- }
73
- catch {
74
- // Fallback to page title
75
- const fromTitle = (firstPage.title || 'documentation')
76
- .toLowerCase()
77
- .replace(/[^a-z0-9-]/g, '-')
78
- .replace(/-+/g, '-')
79
- .replace(/^-|-$/g, '')
80
- .substring(0, 64);
81
- return fromTitle || 'documentation-skill';
82
- }
83
- }
84
- /**
85
- * Extract title from URL or first page
86
- */
87
- extractTitle(rootUrl, firstPage) {
88
- try {
89
- const url = new URL(rootUrl);
90
- const hostname = url.hostname.replace('www.', '');
91
- const pathParts = url.pathname.split('/').filter(p => p);
92
- if (pathParts.length > 0) {
93
- const lastPart = pathParts[pathParts.length - 1];
94
- return `${hostname} - ${this.capitalize(lastPart)} Documentation`;
95
- }
96
- return `${hostname} Documentation`;
97
- }
98
- catch {
99
- return firstPage.title || 'Documentation';
100
- }
101
- }
102
- /**
103
- * Generate description with "when to use" triggers and keywords extracted from pages
104
- * Description must be comprehensive, include triggers, and stay within 1024 chars
105
- */
106
- generateDescription(rootUrl, pages, totalPages, maxDepth) {
107
- // Extract key topics and concepts from pages
108
- const topics = this.extractKeyTopics(pages);
109
- const categories = this.inferCategories(pages);
110
- const categoryNames = Array.from(categories.keys());
111
- // Build description components
112
- const parts = [];
113
- // What the skill provides
114
- const domainName = this.extractDomainName(rootUrl);
115
- parts.push(`Comprehensive documentation for ${domainName}`);
116
- // Extract use cases from page titles and categories
117
- const useCases = this.extractUseCases(pages, categoryNames);
118
- if (useCases.length > 0) {
119
- // Format use cases more naturally
120
- const useCaseText = useCases.length === 1
121
- ? useCases[0]
122
- : useCases.slice(0, -1).join(', ') + (useCases.length > 1 ? `, and ${useCases[useCases.length - 1]}` : '');
123
- parts.push(`Use when working with ${domainName} ${useCaseText}`);
124
- }
125
- // Add specific triggers/keywords based on categories and topics
126
- const triggers = this.generateTriggers(categoryNames, topics);
127
- if (triggers.length > 0) {
128
- // Include triggers naturally in the description
129
- const triggerText = triggers.slice(0, 5).join(', ');
130
- parts.push(`Includes documentation for ${triggerText}`);
131
- }
132
- // Add metadata (crawled info) - keep it brief
133
- parts.push(`Generated from ${rootUrl} (${totalPages} pages, depth ${maxDepth})`);
134
- // Join and truncate to 1024 chars, preserving key information
135
- let description = parts.join('. ');
136
- if (description.length > 1024) {
137
- // Prioritize: what it does + use cases + triggers, then metadata
138
- const essential = `${parts[0]}. ${parts[1] || ''}${parts[2] ? `. ${parts[2]}` : ''}`;
139
- const metadata = parts[3] || '';
140
- const available = 1024 - essential.length - metadata.length - 10; // 10 for spacing/ellipsis
141
- if (available > 0 && metadata.length > available) {
142
- description = `${essential}. ${metadata.substring(0, available)}...`;
143
- }
144
- else {
145
- description = `${essential}. ${metadata}`.substring(0, 1021) + '...';
146
- }
147
- }
148
- return description;
149
- }
150
- /**
151
- * Extract key topics from page titles, headings, and content
152
- */
153
- extractKeyTopics(pages) {
154
- const topicSet = new Set();
155
- const commonWords = new Set(['the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by', 'from', 'as', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'should', 'could', 'may', 'might', 'must', 'can', 'this', 'that', 'these', 'those', 'what', 'which', 'who', 'when', 'where', 'why', 'how', 'get', 'got', 'go', 'goes', 'went', 'come', 'comes', 'came', 'see', 'saw', 'know', 'knows', 'knew', 'think', 'thinks', 'thought', 'take', 'takes', 'took', 'make', 'makes', 'made', 'use', 'uses', 'used', 'work', 'works', 'worked', 'call', 'calls', 'called']);
156
- // Extract from titles
157
- for (const page of pages.slice(0, 20)) { // Limit to first 20 pages for performance
158
- const titleWords = (page.title || '').toLowerCase().split(/\s+/)
159
- .filter(w => w.length > 3 && !commonWords.has(w))
160
- .slice(0, 5);
161
- titleWords.forEach(w => topicSet.add(w));
162
- }
163
- // Extract from headings
164
- for (const page of pages.slice(0, 20)) {
165
- if (page.headings) {
166
- for (const heading of page.headings.slice(0, 10)) {
167
- const headingWords = (heading.text || '').toLowerCase().split(/\s+/)
168
- .filter(w => w.length > 3 && !commonWords.has(w))
169
- .slice(0, 3);
170
- headingWords.forEach(w => topicSet.add(w));
171
- }
172
- }
173
- }
174
- return Array.from(topicSet).slice(0, 10); // Limit to top 10 topics
175
- }
176
- /**
177
- * Extract domain name from URL
178
- */
179
- extractDomainName(rootUrl) {
180
- try {
181
- const url = new URL(rootUrl);
182
- const hostname = url.hostname.replace('www.', '');
183
- const pathParts = url.pathname.split('/').filter(p => p && !['docs', 'documentation', 'en', 'stable', 'latest'].includes(p));
184
- if (pathParts.length > 0) {
185
- return `${hostname} ${pathParts[pathParts.length - 1]}`;
186
- }
187
- return hostname;
188
- }
189
- catch {
190
- return 'documentation';
191
- }
192
- }
193
- /**
194
- * Extract use cases from pages and categories
195
- */
196
- extractUseCases(pages, categoryNames) {
197
- const useCases = [];
198
- // Map categories to use cases
199
- const categoryToUseCase = {
200
- 'api': 'for API reference and methods',
201
- 'tutorials': 'for tutorials and learning',
202
- 'tutorial': 'for tutorials and learning',
203
- 'guide': 'for guides and how-tos',
204
- 'getting-started': 'for getting started',
205
- 'reference': 'for reference documentation',
206
- 'examples': 'for code examples',
207
- };
208
- for (const cat of categoryNames) {
209
- const useCase = categoryToUseCase[cat.toLowerCase()];
210
- if (useCase && !useCases.includes(useCase)) {
211
- useCases.push(useCase);
212
- }
213
- }
214
- // If no specific use cases found, add generic ones
215
- if (useCases.length === 0) {
216
- useCases.push('for understanding concepts', 'for implementing features', 'for debugging code');
217
- }
218
- return useCases.slice(0, 3); // Limit to 3 use cases
219
- }
220
- /**
221
- * Generate trigger keywords from categories and topics
222
- */
223
- generateTriggers(categoryNames, topics) {
224
- const triggers = [];
225
- // Add category-based triggers
226
- for (const cat of categoryNames.slice(0, 3)) {
227
- triggers.push(cat);
228
- }
229
- // Add top topics as triggers
230
- for (const topic of topics.slice(0, 3)) {
231
- if (!triggers.includes(topic)) {
232
- triggers.push(topic);
233
- }
234
- }
235
- return triggers.slice(0, 5); // Limit to 5 triggers
236
- }
237
- /**
238
- * Categorize pages into categories based on URL patterns and content
239
- */
240
- categorizePages(pages) {
241
- const categories = new Map();
242
- const categoryDefs = this.inferCategories(pages);
243
- // Initialize categories
244
- for (const cat of categoryDefs.keys()) {
245
- categories.set(cat, []);
246
- }
247
- categories.set('other', []);
248
- // Categorize each page
249
- for (const page of pages) {
250
- const url = page.url.toLowerCase();
251
- const title = page.title.toLowerCase();
252
- const content = page.content.toLowerCase().substring(0, this.CONTENT_PREVIEW_LENGTH);
253
- let categorized = false;
254
- // Match against keywords
255
- for (const [cat, keywords] of categoryDefs.entries()) {
256
- let score = 0;
257
- for (const keyword of keywords) {
258
- const lowerKeyword = keyword.toLowerCase();
259
- if (url.includes(lowerKeyword))
260
- score += 3;
261
- if (title.includes(lowerKeyword))
262
- score += 2;
263
- if (content.includes(lowerKeyword))
264
- score += 1;
265
- }
266
- if (score >= this.MIN_CATEGORIZATION_SCORE) {
267
- const catPages = categories.get(cat) || [];
268
- catPages.push(page);
269
- categories.set(cat, catPages);
270
- categorized = true;
271
- break;
272
- }
273
- }
274
- if (!categorized) {
275
- const otherPages = categories.get('other') || [];
276
- otherPages.push(page);
277
- categories.set('other', otherPages);
278
- }
279
- }
280
- // Remove empty categories
281
- for (const [cat, pages] of categories.entries()) {
282
- if (pages.length === 0) {
283
- categories.delete(cat);
284
- }
285
- }
286
- return categories;
287
- }
288
- /**
289
- * Infer categories from URL patterns
290
- */
291
- inferCategories(pages) {
292
- const urlSegments = new Map();
293
- for (const page of pages) {
294
- try {
295
- const url = new URL(page.url);
296
- const segments = url.pathname
297
- .split('/')
298
- .filter(s => s && !['en', 'stable', 'latest', 'docs', 'documentation'].includes(s));
299
- for (const seg of segments) {
300
- urlSegments.set(seg, (urlSegments.get(seg) || 0) + 1);
301
- }
302
- }
303
- catch {
304
- // Skip invalid URLs
305
- }
306
- }
307
- // Top segments become categories
308
- const sortedSegments = Array.from(urlSegments.entries())
309
- .sort((a, b) => b[1] - a[1])
310
- .slice(0, 8);
311
- const categories = new Map();
312
- for (const [seg, count] of sortedSegments) {
313
- if (count >= 3) {
314
- categories.set(seg, [seg]);
315
- }
316
- }
317
- // Add common defaults
318
- const allUrls = pages.map(p => p.url.toLowerCase());
319
- if (!categories.has('tutorial') && allUrls.some(u => u.includes('tutorial'))) {
320
- categories.set('tutorials', ['tutorial', 'guide', 'getting-started']);
321
- }
322
- if (!categories.has('api') && allUrls.some(u => u.includes('api') || u.includes('reference'))) {
323
- categories.set('api', ['api', 'reference', 'class']);
324
- }
325
- return categories;
326
- }
327
- /**
328
- * Generate reference files for each category
329
- * Includes full content, code examples, and table of contents from headings
330
- * Adds table of contents at the top for files longer than 100 lines
331
- */
332
- generateReferenceFiles(categories, skillName) {
333
- const referenceFiles = new Map();
334
- for (const [category, pages] of categories.entries()) {
335
- if (pages.length === 0)
336
- continue;
337
- const lines = [];
338
- lines.push(`# ${this.capitalize(skillName)} - ${category.replace('_', ' ').split('-').map(this.capitalize).join(' ')}\n`);
339
- lines.push(`**Pages:** ${pages.length}\n`);
340
- lines.push('---\n');
341
- // Collect all page titles and headings for table of contents
342
- const tocItems = [];
343
- for (const page of pages) {
344
- const headings = [];
345
- if (page.headings && page.headings.length > 0) {
346
- for (const heading of page.headings) {
347
- const level = parseInt(heading.level.replace('h', ''));
348
- headings.push({ level, text: heading.text });
349
- }
350
- }
351
- tocItems.push({ title: page.title, url: page.url, headings });
352
- }
353
- // Generate content
354
- for (const page of pages) {
355
- lines.push(`## ${page.title}\n`);
356
- lines.push(`**URL:** ${page.url}\n`);
357
- // Table of contents from headings (per page)
358
- if (page.headings && page.headings.length > 0) {
359
- lines.push('**Contents:**');
360
- for (const heading of page.headings.slice(0, 10)) {
361
- const level = parseInt(heading.level.replace('h', ''));
362
- const indent = ' '.repeat(Math.max(0, level - 2));
363
- lines.push(`${indent}- ${heading.text}`);
364
- }
365
- lines.push('');
366
- }
367
- // Full content (NO TRUNCATION)
368
- if (page.content) {
369
- lines.push(page.content);
370
- lines.push('');
371
- }
372
- // Code examples with language (NO TRUNCATION)
373
- if (page.codeSamples && page.codeSamples.length > 0) {
374
- lines.push('**Examples:**\n');
375
- for (let i = 0; i < Math.min(page.codeSamples.length, 4); i++) {
376
- const sample = page.codeSamples[i];
377
- const lang = sample.language || 'text';
378
- lines.push(`Example ${i + 1} (${lang}):`);
379
- lines.push(`\`\`\`${lang}`);
380
- lines.push(sample.code); // Full code, no truncation
381
- lines.push('```\n');
382
- }
383
- }
384
- lines.push('---\n');
385
- }
386
- const content = lines.join('\n');
387
- const lineCount = content.split('\n').length;
388
- // Add table of contents at the top for files longer than 100 lines
389
- if (lineCount > 100 && tocItems.length > 0) {
390
- const tocLines = [];
391
- tocLines.push('## Table of Contents\n');
392
- for (const item of tocItems) {
393
- tocLines.push(`- [${item.title}](#${this.slugify(item.title)})`);
394
- // Add sub-headings for major sections
395
- for (const heading of item.headings.slice(0, 3)) {
396
- if (heading.level <= 3) {
397
- const indent = ' '.repeat(heading.level - 2);
398
- tocLines.push(`${indent}- [${heading.text}](#${this.slugify(heading.text)})`);
399
- }
400
- }
401
- }
402
- tocLines.push('');
403
- tocLines.push('---\n');
404
- referenceFiles.set(category, tocLines.join('\n') + content);
405
- }
406
- else {
407
- referenceFiles.set(category, content);
408
- }
409
- }
410
- return referenceFiles;
411
- }
412
- /**
413
- * Generate SKILL.md with YAML frontmatter
414
- * Optimized for progressive disclosure - concise body focused on procedural instructions
415
- */
416
- generateSkillMd(skillName, description, categories, allPages) {
417
- const lines = [];
418
- // YAML frontmatter
419
- lines.push('---');
420
- lines.push(`name: ${skillName}`);
421
- lines.push(`description: ${description}`);
422
- lines.push('---');
423
- lines.push('');
424
- // Title
425
- lines.push(`# ${this.capitalize(skillName)} Skill\n`);
426
- // Quick Start / Overview
427
- lines.push('## Overview\n');
428
- lines.push('This skill provides comprehensive documentation extracted from official sources. Use the reference files below to access detailed information as needed.\n');
429
- // Reference Files with clear guidance on when to read them
430
- lines.push('## Reference Files\n');
431
- lines.push('The following reference files contain organized documentation. Read them when you need specific information:\n');
432
- const sortedCategories = Array.from(categories.keys()).sort();
433
- for (const category of sortedCategories) {
434
- const pages = categories.get(category) || [];
435
- const categoryDisplay = category.replace('_', ' ').split('-').map(this.capitalize).join(' ');
436
- const whenToRead = this.getWhenToReadGuidance(category);
437
- lines.push(`- **[${category}.md](references/${category}.md)** - ${categoryDisplay} documentation (${pages.length} pages)`);
438
- if (whenToRead) {
439
- lines.push(` - *Read when: ${whenToRead}*`);
440
- }
441
- }
442
- lines.push('');
443
- // Working with This Skill (procedural guidance)
444
- lines.push('## Working with This Skill\n');
445
- lines.push('### Getting Started');
446
- const hasTutorials = sortedCategories.some(c => c.includes('tutorial') || c.includes('getting-started') || c.includes('guide'));
447
- if (hasTutorials) {
448
- lines.push('Start with tutorial or getting-started reference files for foundational concepts.');
449
- }
450
- else {
451
- lines.push('Begin with the first reference file that matches your needs.');
452
- }
453
- lines.push('');
454
- lines.push('### Finding Specific Information');
455
- lines.push('Use the category reference files based on what you need:');
456
- lines.push('- **API/Reference files**: For method signatures, parameters, and return types');
457
- lines.push('- **Guide/Tutorial files**: For step-by-step instructions and examples');
458
- lines.push('- **Other categories**: For domain-specific documentation');
459
- lines.push('');
460
- // Quick Reference (if applicable)
461
- lines.push('## Quick Reference\n');
462
- lines.push('Common patterns and examples are included in the reference files. Load the appropriate reference file to access code examples and detailed patterns.\n');
463
- return lines.join('\n');
464
- }
465
- /**
466
- * Get guidance on when to read a specific category reference file
467
- */
468
- getWhenToReadGuidance(category) {
469
- const categoryLower = category.toLowerCase();
470
- const guidance = {
471
- 'api': 'you need API reference, method signatures, or parameter details',
472
- 'reference': 'you need API reference or technical specifications',
473
- 'tutorials': 'you want step-by-step tutorials or learning guides',
474
- 'tutorial': 'you want step-by-step tutorials or learning guides',
475
- 'guide': 'you need how-to guides or implementation instructions',
476
- 'getting-started': 'you are new to the technology or need foundational concepts',
477
- 'examples': 'you need code examples or sample implementations',
478
- };
479
- // Check for partial matches
480
- for (const [key, value] of Object.entries(guidance)) {
481
- if (categoryLower.includes(key)) {
482
- return value;
483
- }
484
- }
485
- return null;
486
- }
487
- /**
488
- * Save skill to file system in proper directory structure
489
- * Prevents nested skill directories by checking if outputDir is already a skill directory
490
- */
491
- async saveSkill(skill, outputDir, filename) {
492
- // Resolve output directory to absolute path
493
- // If outputDir starts with '.', resolve relative to current working directory (project root)
494
- // Otherwise, use resolve() which resolves relative to current working directory anyway
495
- const resolvedOutputDir = outputDir.startsWith('.')
496
- ? resolve(process.cwd(), outputDir)
497
- : resolve(outputDir);
498
- // Determine expected skill name
499
- const expectedSkillName = filename
500
- ? filename
501
- .toLowerCase()
502
- .replace(/[^a-z0-9-]/g, '-')
503
- .replace(/-+/g, '-')
504
- .replace(/^-|-$/g, '')
505
- .substring(0, 64) || 'documentation-skill'
506
- : skill.skillName.toLowerCase()
507
- .replace(/[^a-z0-9-]/g, '-')
508
- .replace(/-+/g, '-')
509
- .replace(/^-|-$/g, '')
510
- .substring(0, 64);
511
- // Check if outputDir is already a skill directory (contains SKILL.md)
512
- let skillDir;
513
- try {
514
- const skillMdPath = join(resolvedOutputDir, 'SKILL.md');
515
- await access(skillMdPath, constants.F_OK);
516
- // outputDir is already a skill directory, use it directly
517
- skillDir = resolvedOutputDir;
518
- }
519
- catch {
520
- // outputDir is not a skill directory
521
- // Check if the last part of outputDir is already the skill name
522
- // If so, use outputDir directly; otherwise create subdirectory
523
- const outputDirBasename = basename(resolvedOutputDir);
524
- if (outputDirBasename === expectedSkillName) {
525
- // outputDir already ends with skill name, use it directly
526
- skillDir = resolvedOutputDir;
527
- }
528
- else {
529
- // Create new skill directory
530
- skillDir = join(resolvedOutputDir, expectedSkillName);
531
- }
532
- await mkdir(skillDir, { recursive: true });
533
- }
534
- // Create references subdirectory
535
- const referencesDir = join(skillDir, 'references');
536
- await mkdir(referencesDir, { recursive: true });
537
- // Create optional directories
538
- const scriptsDir = join(skillDir, 'scripts');
539
- const assetsDir = join(skillDir, 'assets');
540
- await mkdir(scriptsDir, { recursive: true });
541
- await mkdir(assetsDir, { recursive: true });
542
- // Save SKILL.md
543
- const skillMdPath = join(skillDir, 'SKILL.md');
544
- await writeFile(skillMdPath, skill.skillMd, 'utf-8');
545
- // Save reference files
546
- const referenceFilePaths = [];
547
- for (const [category, content] of skill.referenceFiles.entries()) {
548
- const refFilePath = join(referencesDir, `${category}.md`);
549
- await writeFile(refFilePath, content, 'utf-8');
550
- referenceFilePaths.push(refFilePath);
551
- }
552
- return {
553
- skillDir,
554
- skillMdPath,
555
- referenceFiles: referenceFilePaths,
556
- };
557
- }
558
- /**
559
- * Slugify text for anchor links
560
- */
561
- slugify(text) {
562
- return text
563
- .toLowerCase()
564
- .replace(/[^\w\s-]/g, '')
565
- .replace(/\s+/g, '-')
566
- .replace(/-+/g, '-')
567
- .trim();
568
- }
569
- /**
570
- * Capitalize first letter
571
- */
572
- capitalize(text) {
573
- return text.charAt(0).toUpperCase() + text.slice(1);
574
- }
575
- /**
576
- * Check if crawled content is sufficient for skill generation
577
- */
578
- canGenerateSkill(pages) {
579
- if (pages.length === 0) {
580
- return { canGenerate: false, reason: 'empty_pages' };
581
- }
582
- // Check if any page has sufficient content
583
- let hasSufficientContent = false;
584
- let hasStructuredContent = false;
585
- let hasTextContent = false;
586
- let mediaOnlyCount = 0;
587
- for (const page of pages) {
588
- const contentLength = (page.content || '').trim().length;
589
- const hasHeadings = page.headings && page.headings.length > 0;
590
- const hasText = contentLength > 0;
591
- // Check if page is media-only (has images but no text)
592
- const hasImages = /<img[^>]*>/i.test(page.content || '');
593
- const hasMedia = hasImages || (page.codeSamples && page.codeSamples.length > 0);
594
- if (hasMedia && contentLength < this.MIN_CONTENT_LENGTH) {
595
- mediaOnlyCount++;
596
- }
597
- if (contentLength >= this.MIN_CONTENT_LENGTH) {
598
- hasSufficientContent = true;
599
- }
600
- if (hasHeadings) {
601
- hasStructuredContent = true;
602
- }
603
- if (hasText) {
604
- hasTextContent = true;
605
- }
606
- }
607
- // All pages are media-only
608
- if (mediaOnlyCount === pages.length && !hasTextContent) {
609
- return { canGenerate: false, reason: 'media_only' };
610
- }
611
- // No pages have sufficient content
612
- if (!hasSufficientContent) {
613
- return { canGenerate: false, reason: 'insufficient_content' };
614
- }
615
- // No structured content (headings, sections)
616
- if (!hasStructuredContent) {
617
- return { canGenerate: false, reason: 'no_structured_content' };
618
- }
619
- return { canGenerate: true };
620
- }
621
- /**
622
- * Validate description quality - checks length and presence of trigger keywords
623
- */
624
- validateDescription(description) {
625
- // Check length
626
- if (description.length > 1024) {
627
- throw new Error(`Description exceeds 1024 character limit: ${description.length} characters`);
628
- }
629
- // Check for basic quality indicators (warnings only, don't fail)
630
- const hasUseCase = /for:|when|use|working with/i.test(description);
631
- const hasTriggers = /trigger|keyword|concept/i.test(description) || description.split(',').length >= 3;
632
- const minLength = description.length >= 100; // At least 100 chars for meaningful description
633
- // Log warnings if description seems too generic
634
- if (!hasUseCase && !hasTriggers) {
635
- console.warn('Description may be too generic - consider adding more specific triggers or use cases');
636
- }
637
- if (!minLength) {
638
- console.warn('Description may be too short - consider adding more detail');
639
- }
640
- }
641
- }
642
- //# sourceMappingURL=skill-generator.js.map