@gulibs/safe-coder 0.0.23 → 0.0.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +351 -15
- package/dist/documentation/checkpoint-manager.d.ts +38 -0
- package/dist/documentation/checkpoint-manager.d.ts.map +1 -0
- package/dist/documentation/checkpoint-manager.js +101 -0
- package/dist/documentation/checkpoint-manager.js.map +1 -0
- package/dist/documentation/doc-crawler.d.ts +77 -2
- package/dist/documentation/doc-crawler.d.ts.map +1 -1
- package/dist/documentation/doc-crawler.js +752 -179
- package/dist/documentation/doc-crawler.js.map +1 -1
- package/dist/documentation/llms-txt/detector.d.ts +31 -0
- package/dist/documentation/llms-txt/detector.d.ts.map +1 -0
- package/dist/documentation/llms-txt/detector.js +77 -0
- package/dist/documentation/llms-txt/detector.js.map +1 -0
- package/dist/documentation/llms-txt/downloader.d.ts +30 -0
- package/dist/documentation/llms-txt/downloader.d.ts.map +1 -0
- package/dist/documentation/llms-txt/downloader.js +84 -0
- package/dist/documentation/llms-txt/downloader.js.map +1 -0
- package/dist/documentation/llms-txt/index.d.ts +4 -0
- package/dist/documentation/llms-txt/index.d.ts.map +1 -0
- package/dist/documentation/llms-txt/index.js +4 -0
- package/dist/documentation/llms-txt/index.js.map +1 -0
- package/dist/documentation/llms-txt/parser.d.ts +43 -0
- package/dist/documentation/llms-txt/parser.d.ts.map +1 -0
- package/dist/documentation/llms-txt/parser.js +177 -0
- package/dist/documentation/llms-txt/parser.js.map +1 -0
- package/dist/documentation/skill-generator.d.ts +38 -2
- package/dist/documentation/skill-generator.d.ts.map +1 -1
- package/dist/documentation/skill-generator.js +331 -62
- package/dist/documentation/skill-generator.js.map +1 -1
- package/dist/index.js +0 -0
- package/dist/server/mcp-server.d.ts.map +1 -1
- package/dist/server/mcp-server.js +152 -9
- package/dist/server/mcp-server.js.map +1 -1
- package/package.json +10 -11
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { CrawlResult } from './doc-crawler.js';
|
|
1
|
+
import type { CrawledPage, CrawlResult } from './doc-crawler.js';
|
|
2
2
|
export interface SkillMetadata {
|
|
3
3
|
title: string;
|
|
4
4
|
description: string;
|
|
@@ -22,6 +22,8 @@ export interface SavedSkillFiles {
|
|
|
22
22
|
export declare class SkillGenerator {
|
|
23
23
|
private readonly CONTENT_PREVIEW_LENGTH;
|
|
24
24
|
private readonly MIN_CATEGORIZATION_SCORE;
|
|
25
|
+
private readonly MIN_CONTENT_LENGTH;
|
|
26
|
+
private readonly MIN_HEADINGS_COUNT;
|
|
25
27
|
/**
|
|
26
28
|
* Generate agent skill from crawled documentation
|
|
27
29
|
*/
|
|
@@ -35,9 +37,26 @@ export declare class SkillGenerator {
|
|
|
35
37
|
*/
|
|
36
38
|
private extractTitle;
|
|
37
39
|
/**
|
|
38
|
-
* Generate description
|
|
40
|
+
* Generate description with "when to use" triggers and keywords extracted from pages
|
|
41
|
+
* Description must be comprehensive, include triggers, and stay within 1024 chars
|
|
39
42
|
*/
|
|
40
43
|
private generateDescription;
|
|
44
|
+
/**
|
|
45
|
+
* Extract key topics from page titles, headings, and content
|
|
46
|
+
*/
|
|
47
|
+
private extractKeyTopics;
|
|
48
|
+
/**
|
|
49
|
+
* Extract domain name from URL
|
|
50
|
+
*/
|
|
51
|
+
private extractDomainName;
|
|
52
|
+
/**
|
|
53
|
+
* Extract use cases from pages and categories
|
|
54
|
+
*/
|
|
55
|
+
private extractUseCases;
|
|
56
|
+
/**
|
|
57
|
+
* Generate trigger keywords from categories and topics
|
|
58
|
+
*/
|
|
59
|
+
private generateTriggers;
|
|
41
60
|
/**
|
|
42
61
|
* Categorize pages into categories based on URL patterns and content
|
|
43
62
|
*/
|
|
@@ -49,12 +68,18 @@ export declare class SkillGenerator {
|
|
|
49
68
|
/**
|
|
50
69
|
* Generate reference files for each category
|
|
51
70
|
* Includes full content, code examples, and table of contents from headings
|
|
71
|
+
* Adds table of contents at the top for files longer than 100 lines
|
|
52
72
|
*/
|
|
53
73
|
private generateReferenceFiles;
|
|
54
74
|
/**
|
|
55
75
|
* Generate SKILL.md with YAML frontmatter
|
|
76
|
+
* Optimized for progressive disclosure - concise body focused on procedural instructions
|
|
56
77
|
*/
|
|
57
78
|
private generateSkillMd;
|
|
79
|
+
/**
|
|
80
|
+
* Get guidance on when to read a specific category reference file
|
|
81
|
+
*/
|
|
82
|
+
private getWhenToReadGuidance;
|
|
58
83
|
/**
|
|
59
84
|
* Save skill to file system in proper directory structure
|
|
60
85
|
* Prevents nested skill directories by checking if outputDir is already a skill directory
|
|
@@ -68,5 +93,16 @@ export declare class SkillGenerator {
|
|
|
68
93
|
* Capitalize first letter
|
|
69
94
|
*/
|
|
70
95
|
private capitalize;
|
|
96
|
+
/**
|
|
97
|
+
* Check if crawled content is sufficient for skill generation
|
|
98
|
+
*/
|
|
99
|
+
canGenerateSkill(pages: CrawledPage[]): {
|
|
100
|
+
canGenerate: boolean;
|
|
101
|
+
reason?: string;
|
|
102
|
+
};
|
|
103
|
+
/**
|
|
104
|
+
* Validate description quality - checks length and presence of trigger keywords
|
|
105
|
+
*/
|
|
106
|
+
private validateDescription;
|
|
71
107
|
}
|
|
72
108
|
//# sourceMappingURL=skill-generator.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"skill-generator.d.ts","sourceRoot":"","sources":["../../src/documentation/skill-generator.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"skill-generator.d.ts","sourceRoot":"","sources":["../../src/documentation/skill-generator.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAEjE,MAAM,WAAW,aAAa;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,eAAe,EAAE,MAAM,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,EAAE,CAAC;CACxB;AAED,MAAM,WAAW,cAAc;IAC3B,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpC,QAAQ,EAAE,aAAa,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,eAAe;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,EAAE,CAAC;CAC5B;AAED,qBAAa,cAAc;IACvB,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAO;IAC9C,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAK;IAC9C,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAO;IAC1C,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAK;IAExC;;OAEG;IACH,aAAa,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,cAAc;IA0D5F;;OAEG;IACH,OAAO,CAAC,iBAAiB;IA4BzB;;OAEG;IACH,OAAO,CAAC,YAAY;IAiBpB;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAmD3B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IA2BxB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAczB;;OAEG;IACH,OAAO,CAAC,eAAe;IA6BvB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAkBxB;;OAEG;IACH,OAAO,CAAC,eAAe;IAsDvB;;OAEG;IACH,OAAO,CAAC,eAAe;IA0CvB;;;;OAIG;IACH,OAAO,CAAC,sBAAsB;IA2F9B;;;OAGG;IACH,OAAO,CAAC,eAAe;IAyDvB;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAsB7B;;;OAGG;IACG,SAAS,CACX,KAAK,EAAE,cAAc,EACrB,SAAS,EAAE,MAAM,EACjB,QAAQ,CAAC,EAAE,MAAM,GAClB,OAAO,CAAC,eAAe,CAAC;IAyE3B;;OAEG;IACH,OAAO,CAAC,OAAO;IASf;;OAEG;IACH,OAAO,CAAC,UAAU;IAIlB;;OAEG;IACH,gBAAgB,CAAC,KAAK,EAAE,WAAW,EAAE,GAAG;QAAE,WAAW,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE;IAoDjF;;OAEG;IACH,OAAO,CAAC,mBAAmB;CAoB9B"}
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { writeFile, mkdir, access, constants } from 'fs/promises';
|
|
2
|
-
import { join, resolve } from 'path';
|
|
2
|
+
import { join, resolve, basename } from 'path';
|
|
3
3
|
export class SkillGenerator {
|
|
4
4
|
CONTENT_PREVIEW_LENGTH = 500;
|
|
5
5
|
MIN_CATEGORIZATION_SCORE = 2;
|
|
6
|
+
MIN_CONTENT_LENGTH = 100; // Minimum characters for valid content
|
|
7
|
+
MIN_HEADINGS_COUNT = 1; // Minimum headings for structured content
|
|
6
8
|
/**
|
|
7
9
|
* Generate agent skill from crawled documentation
|
|
8
10
|
*/
|
|
@@ -11,13 +13,22 @@ export class SkillGenerator {
|
|
|
11
13
|
if (pages.length === 0) {
|
|
12
14
|
throw new Error('No pages were successfully crawled');
|
|
13
15
|
}
|
|
16
|
+
// Validate if content is sufficient for skill generation
|
|
17
|
+
const validation = this.canGenerateSkill(pages);
|
|
18
|
+
if (!validation.canGenerate) {
|
|
19
|
+
throw new Error(`Cannot generate skill: ${validation.reason}. ` +
|
|
20
|
+
`Pages crawled: ${totalPages}, but content is insufficient. ` +
|
|
21
|
+
`Consider crawling more pages or a different website.`);
|
|
22
|
+
}
|
|
14
23
|
// Generate skill name
|
|
15
24
|
const name = skillName || this.generateSkillName(rootUrl, pages[0]);
|
|
16
25
|
// Generate title from root URL or first page
|
|
17
26
|
const title = this.extractTitle(rootUrl, pages[0]);
|
|
18
27
|
// Generate description (truncated to 1024 chars)
|
|
19
|
-
const description = this.generateDescription(rootUrl, totalPages, maxDepthReached);
|
|
28
|
+
const description = this.generateDescription(rootUrl, pages, totalPages, maxDepthReached);
|
|
20
29
|
const truncatedDescription = description.length > 1024 ? description.substring(0, 1021) + '...' : description;
|
|
30
|
+
// Validate description quality
|
|
31
|
+
this.validateDescription(truncatedDescription);
|
|
21
32
|
// Categorize pages
|
|
22
33
|
const categories = this.categorizePages(pages);
|
|
23
34
|
// Generate reference files
|
|
@@ -89,10 +100,139 @@ export class SkillGenerator {
|
|
|
89
100
|
}
|
|
90
101
|
}
|
|
91
102
|
/**
|
|
92
|
-
* Generate description
|
|
103
|
+
* Generate description with "when to use" triggers and keywords extracted from pages
|
|
104
|
+
* Description must be comprehensive, include triggers, and stay within 1024 chars
|
|
93
105
|
*/
|
|
94
|
-
generateDescription(rootUrl, totalPages, maxDepth) {
|
|
95
|
-
|
|
106
|
+
generateDescription(rootUrl, pages, totalPages, maxDepth) {
|
|
107
|
+
// Extract key topics and concepts from pages
|
|
108
|
+
const topics = this.extractKeyTopics(pages);
|
|
109
|
+
const categories = this.inferCategories(pages);
|
|
110
|
+
const categoryNames = Array.from(categories.keys());
|
|
111
|
+
// Build description components
|
|
112
|
+
const parts = [];
|
|
113
|
+
// What the skill provides
|
|
114
|
+
const domainName = this.extractDomainName(rootUrl);
|
|
115
|
+
parts.push(`Comprehensive documentation for ${domainName}`);
|
|
116
|
+
// Extract use cases from page titles and categories
|
|
117
|
+
const useCases = this.extractUseCases(pages, categoryNames);
|
|
118
|
+
if (useCases.length > 0) {
|
|
119
|
+
// Format use cases more naturally
|
|
120
|
+
const useCaseText = useCases.length === 1
|
|
121
|
+
? useCases[0]
|
|
122
|
+
: useCases.slice(0, -1).join(', ') + (useCases.length > 1 ? `, and ${useCases[useCases.length - 1]}` : '');
|
|
123
|
+
parts.push(`Use when working with ${domainName} ${useCaseText}`);
|
|
124
|
+
}
|
|
125
|
+
// Add specific triggers/keywords based on categories and topics
|
|
126
|
+
const triggers = this.generateTriggers(categoryNames, topics);
|
|
127
|
+
if (triggers.length > 0) {
|
|
128
|
+
// Include triggers naturally in the description
|
|
129
|
+
const triggerText = triggers.slice(0, 5).join(', ');
|
|
130
|
+
parts.push(`Includes documentation for ${triggerText}`);
|
|
131
|
+
}
|
|
132
|
+
// Add metadata (crawled info) - keep it brief
|
|
133
|
+
parts.push(`Generated from ${rootUrl} (${totalPages} pages, depth ${maxDepth})`);
|
|
134
|
+
// Join and truncate to 1024 chars, preserving key information
|
|
135
|
+
let description = parts.join('. ');
|
|
136
|
+
if (description.length > 1024) {
|
|
137
|
+
// Prioritize: what it does + use cases + triggers, then metadata
|
|
138
|
+
const essential = `${parts[0]}. ${parts[1] || ''}${parts[2] ? `. ${parts[2]}` : ''}`;
|
|
139
|
+
const metadata = parts[3] || '';
|
|
140
|
+
const available = 1024 - essential.length - metadata.length - 10; // 10 for spacing/ellipsis
|
|
141
|
+
if (available > 0 && metadata.length > available) {
|
|
142
|
+
description = `${essential}. ${metadata.substring(0, available)}...`;
|
|
143
|
+
}
|
|
144
|
+
else {
|
|
145
|
+
description = `${essential}. ${metadata}`.substring(0, 1021) + '...';
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
return description;
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Extract key topics from page titles, headings, and content
|
|
152
|
+
*/
|
|
153
|
+
extractKeyTopics(pages) {
|
|
154
|
+
const topicSet = new Set();
|
|
155
|
+
const commonWords = new Set(['the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by', 'from', 'as', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'should', 'could', 'may', 'might', 'must', 'can', 'this', 'that', 'these', 'those', 'what', 'which', 'who', 'when', 'where', 'why', 'how', 'get', 'got', 'go', 'goes', 'went', 'come', 'comes', 'came', 'see', 'saw', 'know', 'knows', 'knew', 'think', 'thinks', 'thought', 'take', 'takes', 'took', 'make', 'makes', 'made', 'use', 'uses', 'used', 'work', 'works', 'worked', 'call', 'calls', 'called']);
|
|
156
|
+
// Extract from titles
|
|
157
|
+
for (const page of pages.slice(0, 20)) { // Limit to first 20 pages for performance
|
|
158
|
+
const titleWords = (page.title || '').toLowerCase().split(/\s+/)
|
|
159
|
+
.filter(w => w.length > 3 && !commonWords.has(w))
|
|
160
|
+
.slice(0, 5);
|
|
161
|
+
titleWords.forEach(w => topicSet.add(w));
|
|
162
|
+
}
|
|
163
|
+
// Extract from headings
|
|
164
|
+
for (const page of pages.slice(0, 20)) {
|
|
165
|
+
if (page.headings) {
|
|
166
|
+
for (const heading of page.headings.slice(0, 10)) {
|
|
167
|
+
const headingWords = (heading.text || '').toLowerCase().split(/\s+/)
|
|
168
|
+
.filter(w => w.length > 3 && !commonWords.has(w))
|
|
169
|
+
.slice(0, 3);
|
|
170
|
+
headingWords.forEach(w => topicSet.add(w));
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
return Array.from(topicSet).slice(0, 10); // Limit to top 10 topics
|
|
175
|
+
}
|
|
176
|
+
/**
|
|
177
|
+
* Extract domain name from URL
|
|
178
|
+
*/
|
|
179
|
+
extractDomainName(rootUrl) {
|
|
180
|
+
try {
|
|
181
|
+
const url = new URL(rootUrl);
|
|
182
|
+
const hostname = url.hostname.replace('www.', '');
|
|
183
|
+
const pathParts = url.pathname.split('/').filter(p => p && !['docs', 'documentation', 'en', 'stable', 'latest'].includes(p));
|
|
184
|
+
if (pathParts.length > 0) {
|
|
185
|
+
return `${hostname} ${pathParts[pathParts.length - 1]}`;
|
|
186
|
+
}
|
|
187
|
+
return hostname;
|
|
188
|
+
}
|
|
189
|
+
catch {
|
|
190
|
+
return 'documentation';
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Extract use cases from pages and categories
|
|
195
|
+
*/
|
|
196
|
+
extractUseCases(pages, categoryNames) {
|
|
197
|
+
const useCases = [];
|
|
198
|
+
// Map categories to use cases
|
|
199
|
+
const categoryToUseCase = {
|
|
200
|
+
'api': 'for API reference and methods',
|
|
201
|
+
'tutorials': 'for tutorials and learning',
|
|
202
|
+
'tutorial': 'for tutorials and learning',
|
|
203
|
+
'guide': 'for guides and how-tos',
|
|
204
|
+
'getting-started': 'for getting started',
|
|
205
|
+
'reference': 'for reference documentation',
|
|
206
|
+
'examples': 'for code examples',
|
|
207
|
+
};
|
|
208
|
+
for (const cat of categoryNames) {
|
|
209
|
+
const useCase = categoryToUseCase[cat.toLowerCase()];
|
|
210
|
+
if (useCase && !useCases.includes(useCase)) {
|
|
211
|
+
useCases.push(useCase);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
// If no specific use cases found, add generic ones
|
|
215
|
+
if (useCases.length === 0) {
|
|
216
|
+
useCases.push('for understanding concepts', 'for implementing features', 'for debugging code');
|
|
217
|
+
}
|
|
218
|
+
return useCases.slice(0, 3); // Limit to 3 use cases
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* Generate trigger keywords from categories and topics
|
|
222
|
+
*/
|
|
223
|
+
generateTriggers(categoryNames, topics) {
|
|
224
|
+
const triggers = [];
|
|
225
|
+
// Add category-based triggers
|
|
226
|
+
for (const cat of categoryNames.slice(0, 3)) {
|
|
227
|
+
triggers.push(cat);
|
|
228
|
+
}
|
|
229
|
+
// Add top topics as triggers
|
|
230
|
+
for (const topic of topics.slice(0, 3)) {
|
|
231
|
+
if (!triggers.includes(topic)) {
|
|
232
|
+
triggers.push(topic);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
return triggers.slice(0, 5); // Limit to 5 triggers
|
|
96
236
|
}
|
|
97
237
|
/**
|
|
98
238
|
* Categorize pages into categories based on URL patterns and content
|
|
@@ -187,6 +327,7 @@ export class SkillGenerator {
|
|
|
187
327
|
/**
|
|
188
328
|
* Generate reference files for each category
|
|
189
329
|
* Includes full content, code examples, and table of contents from headings
|
|
330
|
+
* Adds table of contents at the top for files longer than 100 lines
|
|
190
331
|
*/
|
|
191
332
|
generateReferenceFiles(categories, skillName) {
|
|
192
333
|
const referenceFiles = new Map();
|
|
@@ -197,10 +338,23 @@ export class SkillGenerator {
|
|
|
197
338
|
lines.push(`# ${this.capitalize(skillName)} - ${category.replace('_', ' ').split('-').map(this.capitalize).join(' ')}\n`);
|
|
198
339
|
lines.push(`**Pages:** ${pages.length}\n`);
|
|
199
340
|
lines.push('---\n');
|
|
341
|
+
// Collect all page titles and headings for table of contents
|
|
342
|
+
const tocItems = [];
|
|
343
|
+
for (const page of pages) {
|
|
344
|
+
const headings = [];
|
|
345
|
+
if (page.headings && page.headings.length > 0) {
|
|
346
|
+
for (const heading of page.headings) {
|
|
347
|
+
const level = parseInt(heading.level.replace('h', ''));
|
|
348
|
+
headings.push({ level, text: heading.text });
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
tocItems.push({ title: page.title, url: page.url, headings });
|
|
352
|
+
}
|
|
353
|
+
// Generate content
|
|
200
354
|
for (const page of pages) {
|
|
201
355
|
lines.push(`## ${page.title}\n`);
|
|
202
356
|
lines.push(`**URL:** ${page.url}\n`);
|
|
203
|
-
// Table of contents from headings
|
|
357
|
+
// Table of contents from headings (per page)
|
|
204
358
|
if (page.headings && page.headings.length > 0) {
|
|
205
359
|
lines.push('**Contents:**');
|
|
206
360
|
for (const heading of page.headings.slice(0, 10)) {
|
|
@@ -229,12 +383,35 @@ export class SkillGenerator {
|
|
|
229
383
|
}
|
|
230
384
|
lines.push('---\n');
|
|
231
385
|
}
|
|
232
|
-
|
|
386
|
+
const content = lines.join('\n');
|
|
387
|
+
const lineCount = content.split('\n').length;
|
|
388
|
+
// Add table of contents at the top for files longer than 100 lines
|
|
389
|
+
if (lineCount > 100 && tocItems.length > 0) {
|
|
390
|
+
const tocLines = [];
|
|
391
|
+
tocLines.push('## Table of Contents\n');
|
|
392
|
+
for (const item of tocItems) {
|
|
393
|
+
tocLines.push(`- [${item.title}](#${this.slugify(item.title)})`);
|
|
394
|
+
// Add sub-headings for major sections
|
|
395
|
+
for (const heading of item.headings.slice(0, 3)) {
|
|
396
|
+
if (heading.level <= 3) {
|
|
397
|
+
const indent = ' '.repeat(heading.level - 2);
|
|
398
|
+
tocLines.push(`${indent}- [${heading.text}](#${this.slugify(heading.text)})`);
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
tocLines.push('');
|
|
403
|
+
tocLines.push('---\n');
|
|
404
|
+
referenceFiles.set(category, tocLines.join('\n') + content);
|
|
405
|
+
}
|
|
406
|
+
else {
|
|
407
|
+
referenceFiles.set(category, content);
|
|
408
|
+
}
|
|
233
409
|
}
|
|
234
410
|
return referenceFiles;
|
|
235
411
|
}
|
|
236
412
|
/**
|
|
237
413
|
* Generate SKILL.md with YAML frontmatter
|
|
414
|
+
* Optimized for progressive disclosure - concise body focused on procedural instructions
|
|
238
415
|
*/
|
|
239
416
|
generateSkillMd(skillName, description, categories, allPages) {
|
|
240
417
|
const lines = [];
|
|
@@ -246,61 +423,91 @@ export class SkillGenerator {
|
|
|
246
423
|
lines.push('');
|
|
247
424
|
// Title
|
|
248
425
|
lines.push(`# ${this.capitalize(skillName)} Skill\n`);
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
lines.push('
|
|
252
|
-
|
|
253
|
-
lines.push(`- Working with ${skillName}`);
|
|
254
|
-
lines.push(`- Asking about ${skillName} features or APIs`);
|
|
255
|
-
lines.push(`- Implementing ${skillName} solutions`);
|
|
256
|
-
lines.push(`- Debugging ${skillName} code`);
|
|
257
|
-
lines.push(`- Learning ${skillName} best practices\n`);
|
|
258
|
-
// Quick Reference
|
|
259
|
-
lines.push('## Quick Reference\n');
|
|
260
|
-
lines.push('*Quick reference patterns will be added as you use the skill.*\n');
|
|
261
|
-
// Reference Files
|
|
426
|
+
// Quick Start / Overview
|
|
427
|
+
lines.push('## Overview\n');
|
|
428
|
+
lines.push('This skill provides comprehensive documentation extracted from official sources. Use the reference files below to access detailed information as needed.\n');
|
|
429
|
+
// Reference Files with clear guidance on when to read them
|
|
262
430
|
lines.push('## Reference Files\n');
|
|
263
|
-
lines.push('
|
|
264
|
-
|
|
431
|
+
lines.push('The following reference files contain organized documentation. Read them when you need specific information:\n');
|
|
432
|
+
const sortedCategories = Array.from(categories.keys()).sort();
|
|
433
|
+
for (const category of sortedCategories) {
|
|
265
434
|
const pages = categories.get(category) || [];
|
|
266
|
-
|
|
435
|
+
const categoryDisplay = category.replace('_', ' ').split('-').map(this.capitalize).join(' ');
|
|
436
|
+
const whenToRead = this.getWhenToReadGuidance(category);
|
|
437
|
+
lines.push(`- **[${category}.md](references/${category}.md)** - ${categoryDisplay} documentation (${pages.length} pages)`);
|
|
438
|
+
if (whenToRead) {
|
|
439
|
+
lines.push(` - *Read when: ${whenToRead}*`);
|
|
440
|
+
}
|
|
267
441
|
}
|
|
268
442
|
lines.push('');
|
|
269
|
-
// Working with This Skill
|
|
443
|
+
// Working with This Skill (procedural guidance)
|
|
270
444
|
lines.push('## Working with This Skill\n');
|
|
271
|
-
lines.push('###
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
lines.push('
|
|
280
|
-
lines.push('
|
|
281
|
-
lines.push('
|
|
282
|
-
lines.push('-
|
|
283
|
-
lines.push('-
|
|
284
|
-
lines.push('-
|
|
285
|
-
lines.push('
|
|
286
|
-
|
|
287
|
-
lines.push('
|
|
288
|
-
lines.push('
|
|
289
|
-
// Notes
|
|
290
|
-
lines.push('## Notes\n');
|
|
291
|
-
lines.push('- This skill was automatically generated from official documentation');
|
|
292
|
-
lines.push('- Reference files preserve the structure and examples from source docs');
|
|
293
|
-
lines.push('- Code examples include language detection for better syntax highlighting');
|
|
294
|
-
lines.push('- Quick reference patterns are extracted from common usage examples in the docs\n');
|
|
445
|
+
lines.push('### Getting Started');
|
|
446
|
+
const hasTutorials = sortedCategories.some(c => c.includes('tutorial') || c.includes('getting-started') || c.includes('guide'));
|
|
447
|
+
if (hasTutorials) {
|
|
448
|
+
lines.push('Start with tutorial or getting-started reference files for foundational concepts.');
|
|
449
|
+
}
|
|
450
|
+
else {
|
|
451
|
+
lines.push('Begin with the first reference file that matches your needs.');
|
|
452
|
+
}
|
|
453
|
+
lines.push('');
|
|
454
|
+
lines.push('### Finding Specific Information');
|
|
455
|
+
lines.push('Use the category reference files based on what you need:');
|
|
456
|
+
lines.push('- **API/Reference files**: For method signatures, parameters, and return types');
|
|
457
|
+
lines.push('- **Guide/Tutorial files**: For step-by-step instructions and examples');
|
|
458
|
+
lines.push('- **Other categories**: For domain-specific documentation');
|
|
459
|
+
lines.push('');
|
|
460
|
+
// Quick Reference (if applicable)
|
|
461
|
+
lines.push('## Quick Reference\n');
|
|
462
|
+
lines.push('Common patterns and examples are included in the reference files. Load the appropriate reference file to access code examples and detailed patterns.\n');
|
|
295
463
|
return lines.join('\n');
|
|
296
464
|
}
|
|
465
|
+
/**
|
|
466
|
+
* Get guidance on when to read a specific category reference file
|
|
467
|
+
*/
|
|
468
|
+
getWhenToReadGuidance(category) {
|
|
469
|
+
const categoryLower = category.toLowerCase();
|
|
470
|
+
const guidance = {
|
|
471
|
+
'api': 'you need API reference, method signatures, or parameter details',
|
|
472
|
+
'reference': 'you need API reference or technical specifications',
|
|
473
|
+
'tutorials': 'you want step-by-step tutorials or learning guides',
|
|
474
|
+
'tutorial': 'you want step-by-step tutorials or learning guides',
|
|
475
|
+
'guide': 'you need how-to guides or implementation instructions',
|
|
476
|
+
'getting-started': 'you are new to the technology or need foundational concepts',
|
|
477
|
+
'examples': 'you need code examples or sample implementations',
|
|
478
|
+
};
|
|
479
|
+
// Check for partial matches
|
|
480
|
+
for (const [key, value] of Object.entries(guidance)) {
|
|
481
|
+
if (categoryLower.includes(key)) {
|
|
482
|
+
return value;
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
return null;
|
|
486
|
+
}
|
|
297
487
|
/**
|
|
298
488
|
* Save skill to file system in proper directory structure
|
|
299
489
|
* Prevents nested skill directories by checking if outputDir is already a skill directory
|
|
300
490
|
*/
|
|
301
491
|
async saveSkill(skill, outputDir, filename) {
|
|
302
492
|
// Resolve output directory to absolute path
|
|
303
|
-
|
|
493
|
+
// If outputDir starts with '.', resolve relative to current working directory (project root)
|
|
494
|
+
// Otherwise, use resolve() which resolves relative to current working directory anyway
|
|
495
|
+
const resolvedOutputDir = outputDir.startsWith('.')
|
|
496
|
+
? resolve(process.cwd(), outputDir)
|
|
497
|
+
: resolve(outputDir);
|
|
498
|
+
// Determine expected skill name
|
|
499
|
+
const expectedSkillName = filename
|
|
500
|
+
? filename
|
|
501
|
+
.toLowerCase()
|
|
502
|
+
.replace(/[^a-z0-9-]/g, '-')
|
|
503
|
+
.replace(/-+/g, '-')
|
|
504
|
+
.replace(/^-|-$/g, '')
|
|
505
|
+
.substring(0, 64) || 'documentation-skill'
|
|
506
|
+
: skill.skillName.toLowerCase()
|
|
507
|
+
.replace(/[^a-z0-9-]/g, '-')
|
|
508
|
+
.replace(/-+/g, '-')
|
|
509
|
+
.replace(/^-|-$/g, '')
|
|
510
|
+
.substring(0, 64);
|
|
304
511
|
// Check if outputDir is already a skill directory (contains SKILL.md)
|
|
305
512
|
let skillDir;
|
|
306
513
|
try {
|
|
@@ -310,22 +517,18 @@ export class SkillGenerator {
|
|
|
310
517
|
skillDir = resolvedOutputDir;
|
|
311
518
|
}
|
|
312
519
|
catch {
|
|
313
|
-
// outputDir is not a skill directory
|
|
314
|
-
//
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
.replace(/-+/g, '-')
|
|
321
|
-
.replace(/^-|-$/g, '')
|
|
322
|
-
.substring(0, 64) || 'documentation-skill';
|
|
520
|
+
// outputDir is not a skill directory
|
|
521
|
+
// Check if the last part of outputDir is already the skill name
|
|
522
|
+
// If so, use outputDir directly; otherwise create subdirectory
|
|
523
|
+
const outputDirBasename = basename(resolvedOutputDir);
|
|
524
|
+
if (outputDirBasename === expectedSkillName) {
|
|
525
|
+
// outputDir already ends with skill name, use it directly
|
|
526
|
+
skillDir = resolvedOutputDir;
|
|
323
527
|
}
|
|
324
528
|
else {
|
|
325
|
-
|
|
529
|
+
// Create new skill directory
|
|
530
|
+
skillDir = join(resolvedOutputDir, expectedSkillName);
|
|
326
531
|
}
|
|
327
|
-
// Create skill directory
|
|
328
|
-
skillDir = join(resolvedOutputDir, skillDirName);
|
|
329
532
|
await mkdir(skillDir, { recursive: true });
|
|
330
533
|
}
|
|
331
534
|
// Create references subdirectory
|
|
@@ -369,5 +572,71 @@ export class SkillGenerator {
|
|
|
369
572
|
capitalize(text) {
|
|
370
573
|
return text.charAt(0).toUpperCase() + text.slice(1);
|
|
371
574
|
}
|
|
575
|
+
/**
|
|
576
|
+
* Check if crawled content is sufficient for skill generation
|
|
577
|
+
*/
|
|
578
|
+
canGenerateSkill(pages) {
|
|
579
|
+
if (pages.length === 0) {
|
|
580
|
+
return { canGenerate: false, reason: 'empty_pages' };
|
|
581
|
+
}
|
|
582
|
+
// Check if any page has sufficient content
|
|
583
|
+
let hasSufficientContent = false;
|
|
584
|
+
let hasStructuredContent = false;
|
|
585
|
+
let hasTextContent = false;
|
|
586
|
+
let mediaOnlyCount = 0;
|
|
587
|
+
for (const page of pages) {
|
|
588
|
+
const contentLength = (page.content || '').trim().length;
|
|
589
|
+
const hasHeadings = page.headings && page.headings.length > 0;
|
|
590
|
+
const hasText = contentLength > 0;
|
|
591
|
+
// Check if page is media-only (has images but no text)
|
|
592
|
+
const hasImages = /<img[^>]*>/i.test(page.content || '');
|
|
593
|
+
const hasMedia = hasImages || (page.codeSamples && page.codeSamples.length > 0);
|
|
594
|
+
if (hasMedia && contentLength < this.MIN_CONTENT_LENGTH) {
|
|
595
|
+
mediaOnlyCount++;
|
|
596
|
+
}
|
|
597
|
+
if (contentLength >= this.MIN_CONTENT_LENGTH) {
|
|
598
|
+
hasSufficientContent = true;
|
|
599
|
+
}
|
|
600
|
+
if (hasHeadings) {
|
|
601
|
+
hasStructuredContent = true;
|
|
602
|
+
}
|
|
603
|
+
if (hasText) {
|
|
604
|
+
hasTextContent = true;
|
|
605
|
+
}
|
|
606
|
+
}
|
|
607
|
+
// All pages are media-only
|
|
608
|
+
if (mediaOnlyCount === pages.length && !hasTextContent) {
|
|
609
|
+
return { canGenerate: false, reason: 'media_only' };
|
|
610
|
+
}
|
|
611
|
+
// No pages have sufficient content
|
|
612
|
+
if (!hasSufficientContent) {
|
|
613
|
+
return { canGenerate: false, reason: 'insufficient_content' };
|
|
614
|
+
}
|
|
615
|
+
// No structured content (headings, sections)
|
|
616
|
+
if (!hasStructuredContent) {
|
|
617
|
+
return { canGenerate: false, reason: 'no_structured_content' };
|
|
618
|
+
}
|
|
619
|
+
return { canGenerate: true };
|
|
620
|
+
}
|
|
621
|
+
/**
|
|
622
|
+
* Validate description quality - checks length and presence of trigger keywords
|
|
623
|
+
*/
|
|
624
|
+
validateDescription(description) {
|
|
625
|
+
// Check length
|
|
626
|
+
if (description.length > 1024) {
|
|
627
|
+
throw new Error(`Description exceeds 1024 character limit: ${description.length} characters`);
|
|
628
|
+
}
|
|
629
|
+
// Check for basic quality indicators (warnings only, don't fail)
|
|
630
|
+
const hasUseCase = /for:|when|use|working with/i.test(description);
|
|
631
|
+
const hasTriggers = /trigger|keyword|concept/i.test(description) || description.split(',').length >= 3;
|
|
632
|
+
const minLength = description.length >= 100; // At least 100 chars for meaningful description
|
|
633
|
+
// Log warnings if description seems too generic
|
|
634
|
+
if (!hasUseCase && !hasTriggers) {
|
|
635
|
+
console.warn('Description may be too generic - consider adding more specific triggers or use cases');
|
|
636
|
+
}
|
|
637
|
+
if (!minLength) {
|
|
638
|
+
console.warn('Description may be too short - consider adding more detail');
|
|
639
|
+
}
|
|
640
|
+
}
|
|
372
641
|
}
|
|
373
642
|
//# sourceMappingURL=skill-generator.js.map
|