@gulibs/safe-coder 0.0.25 → 0.0.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +682 -902
- package/dist/cache/cache-manager.d.ts +71 -0
- package/dist/cache/cache-manager.d.ts.map +1 -0
- package/dist/cache/cache-manager.js +244 -0
- package/dist/cache/cache-manager.js.map +1 -0
- package/dist/executor/cli-executor.d.ts +106 -0
- package/dist/executor/cli-executor.d.ts.map +1 -0
- package/dist/executor/cli-executor.js +133 -0
- package/dist/executor/cli-executor.js.map +1 -0
- package/dist/executor/dependency-checker.d.ts +23 -0
- package/dist/executor/dependency-checker.d.ts.map +1 -0
- package/dist/executor/dependency-checker.js +62 -0
- package/dist/executor/dependency-checker.js.map +1 -0
- package/dist/index.js +3 -4
- package/dist/index.js.map +1 -1
- package/dist/processor/content-processor.d.ts +76 -0
- package/dist/processor/content-processor.d.ts.map +1 -0
- package/dist/processor/content-processor.js +182 -0
- package/dist/processor/content-processor.js.map +1 -0
- package/dist/processor/guide-generator.d.ts +68 -0
- package/dist/processor/guide-generator.d.ts.map +1 -0
- package/dist/processor/guide-generator.js +189 -0
- package/dist/processor/guide-generator.js.map +1 -0
- package/dist/server/safe-coder-mcp.d.ts +18 -0
- package/dist/server/safe-coder-mcp.d.ts.map +1 -0
- package/dist/server/safe-coder-mcp.js +164 -0
- package/dist/server/safe-coder-mcp.js.map +1 -0
- package/dist/tools/cache-tools.d.ts +42 -0
- package/dist/tools/cache-tools.d.ts.map +1 -0
- package/dist/tools/cache-tools.js +70 -0
- package/dist/tools/cache-tools.js.map +1 -0
- package/dist/tools/crawl-documentation.d.ts +57 -0
- package/dist/tools/crawl-documentation.d.ts.map +1 -0
- package/dist/tools/crawl-documentation.js +96 -0
- package/dist/tools/crawl-documentation.js.map +1 -0
- package/dist/tools/index.d.ts +4 -0
- package/dist/tools/index.d.ts.map +1 -0
- package/dist/tools/index.js +4 -0
- package/dist/tools/index.js.map +1 -0
- package/dist/tools/save-skill.d.ts +49 -0
- package/dist/tools/save-skill.d.ts.map +1 -0
- package/dist/tools/save-skill.js +207 -0
- package/dist/tools/save-skill.js.map +1 -0
- package/package.json +18 -22
- package/dist/documentation/browser-doc-browser.d.ts +0 -41
- package/dist/documentation/browser-doc-browser.d.ts.map +0 -1
- package/dist/documentation/browser-doc-browser.js +0 -357
- package/dist/documentation/browser-doc-browser.js.map +0 -1
- package/dist/documentation/cache.d.ts +0 -13
- package/dist/documentation/cache.d.ts.map +0 -1
- package/dist/documentation/cache.js +0 -48
- package/dist/documentation/cache.js.map +0 -1
- package/dist/documentation/checkpoint-manager.d.ts +0 -38
- package/dist/documentation/checkpoint-manager.d.ts.map +0 -1
- package/dist/documentation/checkpoint-manager.js +0 -101
- package/dist/documentation/checkpoint-manager.js.map +0 -1
- package/dist/documentation/doc-crawler.d.ts +0 -185
- package/dist/documentation/doc-crawler.d.ts.map +0 -1
- package/dist/documentation/doc-crawler.js +0 -1162
- package/dist/documentation/doc-crawler.js.map +0 -1
- package/dist/documentation/github-client.d.ts +0 -13
- package/dist/documentation/github-client.d.ts.map +0 -1
- package/dist/documentation/github-client.js +0 -90
- package/dist/documentation/github-client.js.map +0 -1
- package/dist/documentation/http-fetcher.d.ts +0 -8
- package/dist/documentation/http-fetcher.d.ts.map +0 -1
- package/dist/documentation/http-fetcher.js +0 -31
- package/dist/documentation/http-fetcher.js.map +0 -1
- package/dist/documentation/index.d.ts +0 -16
- package/dist/documentation/index.d.ts.map +0 -1
- package/dist/documentation/index.js +0 -159
- package/dist/documentation/index.js.map +0 -1
- package/dist/documentation/llms-txt/detector.d.ts +0 -31
- package/dist/documentation/llms-txt/detector.d.ts.map +0 -1
- package/dist/documentation/llms-txt/detector.js +0 -77
- package/dist/documentation/llms-txt/detector.js.map +0 -1
- package/dist/documentation/llms-txt/downloader.d.ts +0 -30
- package/dist/documentation/llms-txt/downloader.d.ts.map +0 -1
- package/dist/documentation/llms-txt/downloader.js +0 -84
- package/dist/documentation/llms-txt/downloader.js.map +0 -1
- package/dist/documentation/llms-txt/index.d.ts +0 -4
- package/dist/documentation/llms-txt/index.d.ts.map +0 -1
- package/dist/documentation/llms-txt/index.js +0 -4
- package/dist/documentation/llms-txt/index.js.map +0 -1
- package/dist/documentation/llms-txt/parser.d.ts +0 -43
- package/dist/documentation/llms-txt/parser.d.ts.map +0 -1
- package/dist/documentation/llms-txt/parser.js +0 -177
- package/dist/documentation/llms-txt/parser.js.map +0 -1
- package/dist/documentation/normalizer.d.ts +0 -6
- package/dist/documentation/normalizer.d.ts.map +0 -1
- package/dist/documentation/normalizer.js +0 -38
- package/dist/documentation/normalizer.js.map +0 -1
- package/dist/documentation/npm-client.d.ts +0 -19
- package/dist/documentation/npm-client.d.ts.map +0 -1
- package/dist/documentation/npm-client.js +0 -182
- package/dist/documentation/npm-client.js.map +0 -1
- package/dist/documentation/skill-generator.d.ts +0 -108
- package/dist/documentation/skill-generator.d.ts.map +0 -1
- package/dist/documentation/skill-generator.js +0 -642
- package/dist/documentation/skill-generator.js.map +0 -1
- package/dist/documentation/web-doc-browser.d.ts +0 -67
- package/dist/documentation/web-doc-browser.d.ts.map +0 -1
- package/dist/documentation/web-doc-browser.js +0 -555
- package/dist/documentation/web-doc-browser.js.map +0 -1
- package/dist/errors/api-validator.d.ts +0 -9
- package/dist/errors/api-validator.d.ts.map +0 -1
- package/dist/errors/api-validator.js +0 -57
- package/dist/errors/api-validator.js.map +0 -1
- package/dist/errors/contextual-analysis.d.ts +0 -14
- package/dist/errors/contextual-analysis.d.ts.map +0 -1
- package/dist/errors/contextual-analysis.js +0 -173
- package/dist/errors/contextual-analysis.js.map +0 -1
- package/dist/errors/cross-file-analyzer.d.ts +0 -16
- package/dist/errors/cross-file-analyzer.d.ts.map +0 -1
- package/dist/errors/cross-file-analyzer.js +0 -172
- package/dist/errors/cross-file-analyzer.js.map +0 -1
- package/dist/errors/eslint-integration.d.ts +0 -9
- package/dist/errors/eslint-integration.d.ts.map +0 -1
- package/dist/errors/eslint-integration.js +0 -131
- package/dist/errors/eslint-integration.js.map +0 -1
- package/dist/errors/framework-detector.d.ts +0 -10
- package/dist/errors/framework-detector.d.ts.map +0 -1
- package/dist/errors/framework-detector.js +0 -126
- package/dist/errors/framework-detector.js.map +0 -1
- package/dist/errors/index.d.ts +0 -18
- package/dist/errors/index.d.ts.map +0 -1
- package/dist/errors/index.js +0 -134
- package/dist/errors/index.js.map +0 -1
- package/dist/errors/pattern-matcher.d.ts +0 -25
- package/dist/errors/pattern-matcher.d.ts.map +0 -1
- package/dist/errors/pattern-matcher.js +0 -44
- package/dist/errors/pattern-matcher.js.map +0 -1
- package/dist/errors/patterns.d.ts +0 -11
- package/dist/errors/patterns.d.ts.map +0 -1
- package/dist/errors/patterns.js +0 -351
- package/dist/errors/patterns.js.map +0 -1
- package/dist/errors/performance-detector.d.ts +0 -11
- package/dist/errors/performance-detector.d.ts.map +0 -1
- package/dist/errors/performance-detector.js +0 -119
- package/dist/errors/performance-detector.js.map +0 -1
- package/dist/errors/runtime-detector.d.ts +0 -7
- package/dist/errors/runtime-detector.d.ts.map +0 -1
- package/dist/errors/runtime-detector.js +0 -86
- package/dist/errors/runtime-detector.js.map +0 -1
- package/dist/errors/security-detector.d.ts +0 -6
- package/dist/errors/security-detector.d.ts.map +0 -1
- package/dist/errors/security-detector.js +0 -75
- package/dist/errors/security-detector.js.map +0 -1
- package/dist/errors/typescript-integration.d.ts +0 -6
- package/dist/errors/typescript-integration.d.ts.map +0 -1
- package/dist/errors/typescript-integration.js +0 -46
- package/dist/errors/typescript-integration.js.map +0 -1
- package/dist/server/mcp-server.d.ts +0 -14
- package/dist/server/mcp-server.d.ts.map +0 -1
- package/dist/server/mcp-server.js +0 -776
- package/dist/server/mcp-server.js.map +0 -1
- package/dist/types/documentation.d.ts +0 -26
- package/dist/types/documentation.d.ts.map +0 -1
- package/dist/types/documentation.js +0 -2
- package/dist/types/documentation.js.map +0 -1
- package/dist/utils/config.d.ts +0 -21
- package/dist/utils/config.d.ts.map +0 -1
- package/dist/utils/config.js +0 -34
- package/dist/utils/config.js.map +0 -1
- package/dist/utils/http-client.d.ts +0 -17
- package/dist/utils/http-client.d.ts.map +0 -1
- package/dist/utils/http-client.js +0 -62
- package/dist/utils/http-client.js.map +0 -1
- package/dist/utils/logger.d.ts +0 -36
- package/dist/utils/logger.d.ts.map +0 -1
- package/dist/utils/logger.js +0 -128
- package/dist/utils/logger.js.map +0 -1
- package/dist/utils/rate-limiter.d.ts +0 -9
- package/dist/utils/rate-limiter.d.ts.map +0 -1
- package/dist/utils/rate-limiter.js +0 -26
- package/dist/utils/rate-limiter.js.map +0 -1
- package/dist/validation/auto-fix.d.ts +0 -15
- package/dist/validation/auto-fix.d.ts.map +0 -1
- package/dist/validation/auto-fix.js +0 -49
- package/dist/validation/auto-fix.js.map +0 -1
- package/dist/validation/index.d.ts +0 -21
- package/dist/validation/index.d.ts.map +0 -1
- package/dist/validation/index.js +0 -45
- package/dist/validation/index.js.map +0 -1
- package/dist/validation/resolution-db.d.ts +0 -15
- package/dist/validation/resolution-db.d.ts.map +0 -1
- package/dist/validation/resolution-db.js +0 -62
- package/dist/validation/resolution-db.js.map +0 -1
|
@@ -1,357 +0,0 @@
|
|
|
1
|
-
import { logger } from '../utils/logger.js';
|
|
2
|
-
/**
|
|
3
|
-
* Browser-based documentation browser using Playwright
|
|
4
|
-
* Handles SPA (Single Page Applications) that require JavaScript rendering
|
|
5
|
-
*/
|
|
6
|
-
export class BrowserDocumentationBrowser {
|
|
7
|
-
browser = null;
|
|
8
|
-
page = null;
|
|
9
|
-
DOCUMENTATION_INDICATORS = [
|
|
10
|
-
/\/docs?\//i,
|
|
11
|
-
/\/documentation/i,
|
|
12
|
-
/\/guide/i,
|
|
13
|
-
/\/tutorial/i,
|
|
14
|
-
/\/api/i,
|
|
15
|
-
/\/reference/i,
|
|
16
|
-
/\/manual/i,
|
|
17
|
-
/\/help/i,
|
|
18
|
-
/\/about/i,
|
|
19
|
-
/\/getting-started/i,
|
|
20
|
-
];
|
|
21
|
-
/**
|
|
22
|
-
* Initialize browser (lazy loading)
|
|
23
|
-
*/
|
|
24
|
-
async ensureBrowser() {
|
|
25
|
-
if (this.browser) {
|
|
26
|
-
return;
|
|
27
|
-
}
|
|
28
|
-
try {
|
|
29
|
-
// Dynamic import to make Playwright optional
|
|
30
|
-
const { chromium } = await import('playwright');
|
|
31
|
-
this.browser = await chromium.launch({
|
|
32
|
-
headless: true,
|
|
33
|
-
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
|
34
|
-
});
|
|
35
|
-
this.page = await this.browser.newPage();
|
|
36
|
-
// Set viewport
|
|
37
|
-
await this.page.setViewportSize({ width: 1920, height: 1080 });
|
|
38
|
-
// Set user agent
|
|
39
|
-
await this.page.setUserAgent('Safe-Coder-MCP/1.0.0 (Documentation Crawler)');
|
|
40
|
-
logger.info('Playwright browser initialized for SPA documentation crawling');
|
|
41
|
-
}
|
|
42
|
-
catch (error) {
|
|
43
|
-
throw new Error(`Failed to initialize Playwright browser. Please install Playwright: npm install playwright && npx playwright install chromium. Error: ${error instanceof Error ? error.message : String(error)}`);
|
|
44
|
-
}
|
|
45
|
-
}
|
|
46
|
-
/**
|
|
47
|
-
* Check if a URL is likely a documentation page
|
|
48
|
-
*/
|
|
49
|
-
async isDocumentationPage(url) {
|
|
50
|
-
// For browser mode, be more lenient - accept URLs that look like documentation
|
|
51
|
-
const hasDocPattern = this.DOCUMENTATION_INDICATORS.some(pattern => pattern.test(url));
|
|
52
|
-
if (hasDocPattern) {
|
|
53
|
-
return true;
|
|
54
|
-
}
|
|
55
|
-
// Also accept if URL contains common documentation keywords
|
|
56
|
-
const urlLower = url.toLowerCase();
|
|
57
|
-
const hasDocKeywords = /(?:doc|guide|tutorial|api|reference|manual|help|about|getting-started)/.test(urlLower);
|
|
58
|
-
// For browser mode, be more permissive - accept most URLs that don't look like non-docs
|
|
59
|
-
const isNonDoc = /(?:login|signup|register|checkout|cart|payment|\.(jpg|jpeg|png|gif|svg|pdf|zip))/.test(urlLower);
|
|
60
|
-
return !isNonDoc && (hasDocKeywords || urlLower.includes('/docs') || urlLower.includes('/documentation'));
|
|
61
|
-
}
|
|
62
|
-
/**
|
|
63
|
-
* Browse a documentation page using browser automation
|
|
64
|
-
* Extracts content from rendered SPA pages
|
|
65
|
-
*/
|
|
66
|
-
async browsePage(url, query) {
|
|
67
|
-
await this.ensureBrowser();
|
|
68
|
-
try {
|
|
69
|
-
const fullUrl = url.startsWith('http') ? url : `https://${url}`;
|
|
70
|
-
const baseUrl = new URL(fullUrl).origin;
|
|
71
|
-
logger.debug('Navigating to page with browser', {
|
|
72
|
-
url: fullUrl,
|
|
73
|
-
method: 'Playwright browser automation',
|
|
74
|
-
});
|
|
75
|
-
// Navigate to page
|
|
76
|
-
await this.page.goto(fullUrl, {
|
|
77
|
-
waitUntil: 'networkidle',
|
|
78
|
-
timeout: 30000,
|
|
79
|
-
});
|
|
80
|
-
// Wait a bit more for SPA to fully render
|
|
81
|
-
await this.page.waitForTimeout(2000);
|
|
82
|
-
// Get rendered HTML
|
|
83
|
-
const html = await this.page.content();
|
|
84
|
-
const title = await this.page.title();
|
|
85
|
-
logger.debug('Page loaded and rendered', {
|
|
86
|
-
url: fullUrl,
|
|
87
|
-
title: title.substring(0, 50),
|
|
88
|
-
htmlLength: html.length,
|
|
89
|
-
});
|
|
90
|
-
// Parse HTML and extract content
|
|
91
|
-
const parsed = this.parseHTML(html, baseUrl, fullUrl);
|
|
92
|
-
// Filter content by query if provided
|
|
93
|
-
let filteredContent = parsed.content;
|
|
94
|
-
let filteredSections = parsed.sections;
|
|
95
|
-
if (query) {
|
|
96
|
-
const queryLower = query.toLowerCase();
|
|
97
|
-
filteredSections = parsed.sections.filter(section => section.title.toLowerCase().includes(queryLower) ||
|
|
98
|
-
section.content.toLowerCase().includes(queryLower));
|
|
99
|
-
if (filteredSections.length > 0) {
|
|
100
|
-
filteredContent = filteredSections.map(s => `## ${s.title}\n\n${s.content}`).join('\n\n');
|
|
101
|
-
}
|
|
102
|
-
else {
|
|
103
|
-
if (parsed.searchableContent.toLowerCase().includes(queryLower)) {
|
|
104
|
-
const paragraphs = parsed.content.split('\n\n');
|
|
105
|
-
const relevant = paragraphs.filter(p => p.toLowerCase().includes(queryLower));
|
|
106
|
-
filteredContent = relevant.join('\n\n');
|
|
107
|
-
}
|
|
108
|
-
else {
|
|
109
|
-
filteredContent = `No content found matching "${query}". Showing full page content:\n\n${parsed.content}`;
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
return {
|
|
114
|
-
url: fullUrl,
|
|
115
|
-
title: title || parsed.title,
|
|
116
|
-
content: filteredContent,
|
|
117
|
-
isDocumentation: true,
|
|
118
|
-
navigationLinks: parsed.navigationLinks,
|
|
119
|
-
sections: filteredSections.length > 0 ? filteredSections : parsed.sections,
|
|
120
|
-
searchableContent: parsed.searchableContent,
|
|
121
|
-
};
|
|
122
|
-
}
|
|
123
|
-
catch (error) {
|
|
124
|
-
throw new Error(`Failed to browse documentation page with browser: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
/**
|
|
128
|
-
* Navigate to a specific page
|
|
129
|
-
*/
|
|
130
|
-
async navigateTo(url, baseUrl) {
|
|
131
|
-
const fullUrl = url.startsWith('http')
|
|
132
|
-
? url
|
|
133
|
-
: baseUrl
|
|
134
|
-
? new URL(url, baseUrl).href
|
|
135
|
-
: `https://${url}`;
|
|
136
|
-
return this.browsePage(fullUrl);
|
|
137
|
-
}
|
|
138
|
-
/**
|
|
139
|
-
* Parse HTML and extract documentation content
|
|
140
|
-
* Enhanced to extract code blocks with language detection
|
|
141
|
-
*/
|
|
142
|
-
parseHTML(html, baseUrl, currentUrl) {
|
|
143
|
-
// Extract title
|
|
144
|
-
const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
|
|
145
|
-
const title = titleMatch ? titleMatch[1].trim() : 'Documentation';
|
|
146
|
-
// Remove scripts and styles
|
|
147
|
-
let cleanHtml = html
|
|
148
|
-
.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
|
|
149
|
-
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
|
|
150
|
-
.replace(/<!--[\s\S]*?-->/g, '');
|
|
151
|
-
// Extract navigation links
|
|
152
|
-
const navigationLinks = [];
|
|
153
|
-
const navigationLinksSet = new Set();
|
|
154
|
-
// Find nav, aside, or sidebar elements
|
|
155
|
-
const navMatches = cleanHtml.match(/<(?:nav|aside)[^>]*>([\s\S]*?)<\/(?:nav|aside)>/gi);
|
|
156
|
-
if (navMatches) {
|
|
157
|
-
navMatches.forEach(navHtml => {
|
|
158
|
-
const linkMatches = navHtml.matchAll(/<a[^>]+href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi);
|
|
159
|
-
for (const match of linkMatches) {
|
|
160
|
-
const href = match[1];
|
|
161
|
-
let text = match[2].replace(/<[^>]+>/g, '').trim();
|
|
162
|
-
if (text && href && !href.startsWith('#')) {
|
|
163
|
-
try {
|
|
164
|
-
const fullUrl = href.startsWith('http')
|
|
165
|
-
? href
|
|
166
|
-
: new URL(href, baseUrl).href;
|
|
167
|
-
const normalizedUrl = fullUrl.split('#')[0];
|
|
168
|
-
const isInternal = new URL(fullUrl).origin === new URL(currentUrl).origin;
|
|
169
|
-
if (!navigationLinksSet.has(normalizedUrl)) {
|
|
170
|
-
navigationLinks.push({ text, url: normalizedUrl, isInternal });
|
|
171
|
-
navigationLinksSet.add(normalizedUrl);
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
catch {
|
|
175
|
-
// Invalid URL, skip
|
|
176
|
-
}
|
|
177
|
-
}
|
|
178
|
-
}
|
|
179
|
-
});
|
|
180
|
-
}
|
|
181
|
-
// Extract main content area
|
|
182
|
-
let mainContent = '';
|
|
183
|
-
const mainMatches = cleanHtml.match(/<main[^>]*>([\s\S]*?)<\/main>/i) ||
|
|
184
|
-
cleanHtml.match(/<article[^>]*>([\s\S]*?)<\/article>/i) ||
|
|
185
|
-
cleanHtml.match(/<div[^>]*class="[^"]*(?:content|main|documentation|doc)[^"]*"[^>]*>([\s\S]*?)<\/div>/i);
|
|
186
|
-
if (mainMatches) {
|
|
187
|
-
mainContent = mainMatches[1];
|
|
188
|
-
}
|
|
189
|
-
else {
|
|
190
|
-
const bodyMatch = cleanHtml.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
|
|
191
|
-
mainContent = bodyMatch ? bodyMatch[1] : cleanHtml;
|
|
192
|
-
}
|
|
193
|
-
// Extract code blocks with language detection
|
|
194
|
-
const codeBlocks = [];
|
|
195
|
-
// Extract <pre><code> blocks
|
|
196
|
-
const preCodeMatches = mainContent.matchAll(/<pre[^>]*><code[^>]*(?:class=["']language-([^"']+)["']|data-language=["']([^"']+)["'])?[^>]*>([\s\S]*?)<\/code><\/pre>/gi);
|
|
197
|
-
for (const match of preCodeMatches) {
|
|
198
|
-
const language = (match[1] || match[2] || this.detectLanguage(match[3])).toLowerCase();
|
|
199
|
-
const code = match[3].trim();
|
|
200
|
-
if (code.length > 10) {
|
|
201
|
-
codeBlocks.push({ code, language });
|
|
202
|
-
}
|
|
203
|
-
}
|
|
204
|
-
// Extract inline <code> blocks (for short snippets)
|
|
205
|
-
const inlineCodeMatches = mainContent.matchAll(/<code[^>]*>([^<]+)<\/code>/gi);
|
|
206
|
-
for (const match of inlineCodeMatches) {
|
|
207
|
-
const code = match[1].trim();
|
|
208
|
-
if (code.length > 5 && code.length < 100) {
|
|
209
|
-
// Only include short inline code, not full blocks
|
|
210
|
-
codeBlocks.push({ code, language: 'text' });
|
|
211
|
-
}
|
|
212
|
-
}
|
|
213
|
-
// Convert HTML to markdown with code blocks preserved
|
|
214
|
-
let textContent = mainContent
|
|
215
|
-
// Preserve code blocks first
|
|
216
|
-
.replace(/<pre[^>]*><code[^>]*(?:class=["']language-([^"']+)["']|data-language=["']([^"']+)["'])?[^>]*>([\s\S]*?)<\/code><\/pre>/gi, (_, lang1, lang2, code) => {
|
|
217
|
-
const lang = (lang1 || lang2 || this.detectLanguage(code)).toLowerCase();
|
|
218
|
-
return `\n\`\`\`${lang}\n${code.trim()}\n\`\`\`\n`;
|
|
219
|
-
})
|
|
220
|
-
// Headers
|
|
221
|
-
.replace(/<h([1-6])[^>]*>([^<]+)<\/h[1-6]>/gi, (_, level, text) => {
|
|
222
|
-
const prefix = '#'.repeat(parseInt(level));
|
|
223
|
-
return `\n${prefix} ${text.trim()}\n`;
|
|
224
|
-
})
|
|
225
|
-
// Links
|
|
226
|
-
.replace(/<a[^>]+href=["']([^"']+)["'][^>]*>([^<]+)<\/a>/gi, '[$2]($1)')
|
|
227
|
-
// Inline code
|
|
228
|
-
.replace(/<code[^>]*>([^<]+)<\/code>/gi, '`$1`')
|
|
229
|
-
// Lists
|
|
230
|
-
.replace(/<li[^>]*>([^<]+)<\/li>/gi, '- $1')
|
|
231
|
-
// Paragraphs
|
|
232
|
-
.replace(/<p[^>]*>([^<]+)<\/p>/gi, '$1\n')
|
|
233
|
-
// Bold/Italic
|
|
234
|
-
.replace(/<(?:strong|b)[^>]*>([^<]+)<\/(?:strong|b)>/gi, '**$1**')
|
|
235
|
-
.replace(/<(?:em|i)[^>]*>([^<]+)<\/(?:em|i)>/gi, '*$1*')
|
|
236
|
-
// Remove remaining HTML tags
|
|
237
|
-
.replace(/<[^>]+>/g, ' ')
|
|
238
|
-
// Clean up whitespace
|
|
239
|
-
.replace(/\n{3,}/g, '\n\n')
|
|
240
|
-
.replace(/[ \t]+/g, ' ')
|
|
241
|
-
.trim();
|
|
242
|
-
// Extract sections
|
|
243
|
-
const sections = [];
|
|
244
|
-
const sectionMatches = textContent.matchAll(/(?:^|\n)(#{1,6})\s+(.+?)(?=\n#{1,6}|\n*$)/gs);
|
|
245
|
-
let lastIndex = 0;
|
|
246
|
-
for (const match of sectionMatches) {
|
|
247
|
-
const headerLevel = match[1].length;
|
|
248
|
-
const headerText = match[2].trim();
|
|
249
|
-
const startIndex = match.index || 0;
|
|
250
|
-
const nextHeaderPattern = new RegExp(`\n#{1,${headerLevel}}\\s+`, 'g');
|
|
251
|
-
nextHeaderPattern.lastIndex = startIndex + match[0].length;
|
|
252
|
-
const nextMatch = nextHeaderPattern.exec(textContent);
|
|
253
|
-
const endIndex = nextMatch ? nextMatch.index : textContent.length;
|
|
254
|
-
const sectionContent = textContent.substring(startIndex + match[0].length, endIndex).trim();
|
|
255
|
-
if (sectionContent) {
|
|
256
|
-
sections.push({
|
|
257
|
-
title: headerText,
|
|
258
|
-
content: sectionContent,
|
|
259
|
-
anchor: `#${headerText.toLowerCase().replace(/\s+/g, '-')}`,
|
|
260
|
-
});
|
|
261
|
-
}
|
|
262
|
-
lastIndex = endIndex;
|
|
263
|
-
}
|
|
264
|
-
// If no sections found, create one from all content
|
|
265
|
-
if (sections.length === 0 && textContent) {
|
|
266
|
-
sections.push({
|
|
267
|
-
title: title,
|
|
268
|
-
content: textContent,
|
|
269
|
-
});
|
|
270
|
-
}
|
|
271
|
-
// Extract links from content area (for discovery)
|
|
272
|
-
const contentLinkMatches = mainContent.matchAll(/<a[^>]+href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi);
|
|
273
|
-
for (const match of contentLinkMatches) {
|
|
274
|
-
const href = match[1];
|
|
275
|
-
let text = match[2].replace(/<[^>]+>/g, '').trim();
|
|
276
|
-
if (text && href && !href.startsWith('#')) {
|
|
277
|
-
try {
|
|
278
|
-
const fullUrl = href.startsWith('http')
|
|
279
|
-
? href
|
|
280
|
-
: new URL(href, baseUrl).href;
|
|
281
|
-
const normalizedUrl = fullUrl.split('#')[0];
|
|
282
|
-
const isInternal = new URL(fullUrl).origin === new URL(currentUrl).origin;
|
|
283
|
-
if (!navigationLinksSet.has(normalizedUrl)) {
|
|
284
|
-
navigationLinks.push({ text, url: normalizedUrl, isInternal });
|
|
285
|
-
navigationLinksSet.add(normalizedUrl);
|
|
286
|
-
}
|
|
287
|
-
}
|
|
288
|
-
catch {
|
|
289
|
-
// Invalid URL, skip
|
|
290
|
-
}
|
|
291
|
-
}
|
|
292
|
-
}
|
|
293
|
-
const finalLinks = [...navigationLinks].slice(0, 200); // Limit to 200 links
|
|
294
|
-
logger.debug('Content extraction completed', {
|
|
295
|
-
url: currentUrl,
|
|
296
|
-
title,
|
|
297
|
-
sectionsCount: sections.length,
|
|
298
|
-
codeBlocksCount: codeBlocks.length,
|
|
299
|
-
linksCount: finalLinks.length,
|
|
300
|
-
contentLength: textContent.length,
|
|
301
|
-
});
|
|
302
|
-
return {
|
|
303
|
-
title,
|
|
304
|
-
content: textContent || 'No content extracted',
|
|
305
|
-
navigationLinks: finalLinks,
|
|
306
|
-
sections,
|
|
307
|
-
searchableContent: textContent.toLowerCase(),
|
|
308
|
-
};
|
|
309
|
-
}
|
|
310
|
-
/**
|
|
311
|
-
* Detect programming language from code
|
|
312
|
-
*/
|
|
313
|
-
detectLanguage(code) {
|
|
314
|
-
const codeLower = code.trim().toLowerCase();
|
|
315
|
-
// Common patterns
|
|
316
|
-
if (/^(import|export|const|let|var|function|class|interface|type)\s/.test(codeLower)) {
|
|
317
|
-
if (/from\s+['"]react['"]/.test(codeLower) || /<[A-Z]\w+/.test(codeLower)) {
|
|
318
|
-
return 'jsx';
|
|
319
|
-
}
|
|
320
|
-
if (/from\s+['"]/.test(codeLower) || /:\s*(string|number|boolean|object)/.test(codeLower)) {
|
|
321
|
-
return 'typescript';
|
|
322
|
-
}
|
|
323
|
-
return 'javascript';
|
|
324
|
-
}
|
|
325
|
-
if (/^(def|class|import|from)\s/.test(codeLower)) {
|
|
326
|
-
return 'python';
|
|
327
|
-
}
|
|
328
|
-
if (/^(package|import|public|private|class)\s/.test(codeLower)) {
|
|
329
|
-
return 'java';
|
|
330
|
-
}
|
|
331
|
-
if (/^(#include|#define|int\s+main|void\s+main)/.test(codeLower)) {
|
|
332
|
-
return 'c';
|
|
333
|
-
}
|
|
334
|
-
if (/^(<!DOCTYPE|<html|<head)/.test(codeLower)) {
|
|
335
|
-
return 'html';
|
|
336
|
-
}
|
|
337
|
-
if (/^(\.|#|\w+\s*\{)/.test(codeLower)) {
|
|
338
|
-
return 'css';
|
|
339
|
-
}
|
|
340
|
-
return 'text';
|
|
341
|
-
}
|
|
342
|
-
/**
|
|
343
|
-
* Close browser
|
|
344
|
-
*/
|
|
345
|
-
async close() {
|
|
346
|
-
if (this.page) {
|
|
347
|
-
await this.page.close();
|
|
348
|
-
this.page = null;
|
|
349
|
-
}
|
|
350
|
-
if (this.browser) {
|
|
351
|
-
await this.browser.close();
|
|
352
|
-
this.browser = null;
|
|
353
|
-
}
|
|
354
|
-
logger.debug('Browser closed');
|
|
355
|
-
}
|
|
356
|
-
}
|
|
357
|
-
//# sourceMappingURL=browser-doc-browser.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"browser-doc-browser.js","sourceRoot":"","sources":["../../src/documentation/browser-doc-browser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAG5C;;;GAGG;AACH,MAAM,OAAO,2BAA2B;IAC9B,OAAO,GAAQ,IAAI,CAAC;IACpB,IAAI,GAAQ,IAAI,CAAC;IACR,wBAAwB,GAAG;QAC1C,YAAY;QACZ,kBAAkB;QAClB,UAAU;QACV,aAAa;QACb,QAAQ;QACR,cAAc;QACd,WAAW;QACX,SAAS;QACT,UAAU;QACV,oBAAoB;KACrB,CAAC;IAEF;;OAEG;IACK,KAAK,CAAC,aAAa;QACzB,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QAED,IAAI,CAAC;YACH,6CAA6C;YAC7C,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;YAChD,IAAI,CAAC,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC;gBACnC,QAAQ,EAAE,IAAI;gBACd,IAAI,EAAE,CAAC,cAAc,EAAE,0BAA0B,CAAC;aACnD,CAAC,CAAC;YACH,IAAI,CAAC,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YAEzC,eAAe;YACf,MAAM,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YAE/D,iBAAiB;YACjB,MAAM,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,8CAA8C,CAAC,CAAC;YAE7E,MAAM,CAAC,IAAI,CAAC,+DAA+D,CAAC,CAAC;QAC/E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,yIAAyI,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAClM,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mBAAmB,CAAC,GAAW;QACnC,+EAA+E;QAC/E,MAAM,aAAa,GAAG,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACvF,IAAI,aAAa,EAAE,CAAC;YAClB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,4DAA4D;QAC5D,MAAM,QAAQ,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QACnC,MAAM,cAAc,GAAG,wEAAwE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE/G,wFAAwF;QACxF,MAAM,QAAQ,GAAG,kFAAkF,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAEnH,OAAO,CAAC,QAAQ,IAAI,CAAC,cAAc,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC;IAC5G,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU,CAAC,GAAW,EAAE,KAAc;QAC1C,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAE3B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,GAAG,EAAE,CAAC;YAChE,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;YAExC,MAAM,CAAC,KAAK,CAAC,iCAAiC,EAAE;gBAC9C,GAAG,EAAE,OAAO;gBACZ,MAAM,EAAE,+BAA+B;aACxC,CAAC,CAAC;YAEH,mBAAmB;YACnB,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;gBAC5B,SAAS,EAAE,aAAa;gBACxB,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;YAEH,0CAA0C;YAC1C,MAAM,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;YAErC,oBAAoB;YACpB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACvC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YAEtC,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE;gBACvC,GAAG,EAAE,OAAO;gBACZ,KAAK,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC;gBAC7B,UAAU,EAAE,IAAI,CAAC,MAAM;aACxB,CAAC,CAAC;YAEH,iCAAiC;YACjC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;YAEtD,sCAAsC;YACtC,IAAI,eAAe,GAAG,MAAM,CAAC,OAAO,CAAC;YACrC,IAAI,gBAAgB,GAAG,MAAM,CAAC,QAAQ,CAAC;YAEvC,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;gBACvC,gBAAgB,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAClD,OAAO,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC;oBAChD,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CACnD,CAAC;gBAEF,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAChC,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CACzC,MAAM,CAAC,CAAC,KAAK,OAAO,CAAC,CAAC,OAAO,EAAE,CAChC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACjB,CAAC;qBAAM,CAAC;oBACN,IAAI,MAAM,CAAC,iBAAiB,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;wBAChE,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;wBAChD,MAAM,QAAQ,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CACrC,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CACrC,CAAC;wBACF,eAAe,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBAC1C,CAAC;yBAAM,CAAC;wBACN,eAAe,GAAG,8BAA8B,KAAK,oCAAoC,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC5G,CAAC;gBACH,CAAC;YACH,CAAC;YAED,OAAO;gBACL,GAAG,EAAE,OAAO;gBACZ,KAAK,EAAE,KAAK,IAAI,MAAM,CAAC,KAAK;gBAC5B,OAAO,EAAE,eAAe;gBACxB,eAAe,EAAE,IAAI;gBACrB,eAAe,EAAE,MAAM,CAAC,eAAe;gBACvC,QAAQ,EAAE,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ;gBAC1E,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;aAC5C,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,qDAAqD,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAChH,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,GAAW,EAAE,OAAgB;QAC5C,MAAM,OAAO,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC;YACpC,CAAC,CAAC,GAAG;YACL,CAAC,CAAC,OAAO;gBACP,CAAC,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,IAAI;gBAC5B,CAAC,CAAC,WAAW,GAAG,EAAE,CAAC;QAEvB,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;IAED;;;OAGG;IACK,SAAS,CAAC,IAAY,EAAE,OAAe,EAAE,UAAkB;QAOjE,gBAAgB;QAChB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;QAC/D,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC;QAElE,4BAA4B;QAC5B,IAAI,SAAS,GAAG,IAAI;aACjB,OAAO,CAAC,mCAAmC,EAAE,EAAE,CAAC;aAChD,OAAO,CAAC,iCAAiC,EAAE,EAAE,CAAC;aAC9C,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC,CAAC;QAEnC,2BAA2B;QAC3B,MAAM,eAAe,GAA8D,EAAE,CAAC;QACtF,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAU,CAAC;QAE7C,uCAAuC;QACvC,MAAM,UAAU,GAAG,SAAS,CAAC,KAAK,CAAC,mDAAmD,CAAC,CAAC;QACxF,IAAI,UAAU,EAAE,CAAC;YACf,UAAU,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;gBAC3B,MAAM,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC,qDAAqD,CAAC,CAAC;gBAC5F,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;oBAChC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;oBACtB,IAAI,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;oBACnD,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;wBAC1C,IAAI,CAAC;4BACH,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;gCACrC,CAAC,CAAC,IAAI;gCACN,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;4BAChC,MAAM,aAAa,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;4BAC5C,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC;4BAE1E,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE,CAAC;gCAC3C,eAAe,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,aAAa,EAAE,UAAU,EAAE,CAAC,CAAC;gCAC/D,kBAAkB,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;4BACxC,CAAC;wBACH,CAAC;wBAAC,MAAM,CAAC;4BACP,oBAAoB;wBACtB,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAED,4BAA4B;QAC5B,IAAI,WAAW,GAAG,EAAE,CAAC;QACrB,MAAM,WAAW,GACf,SAAS,CAAC,KAAK,CAAC,gCAAgC,CAAC;YACjD,SAAS,CAAC,KAAK,CAAC,sCAAsC,CAAC;YACvD,SAAS,CAAC,KAAK,CAAC,uFAAuF,CAAC,CAAC;QAE3G,IAAI,WAAW,EAAE,CAAC;YAChB,WAAW,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;QAC/B,CAAC;aAAM,CAAC;YACN,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;YACpE,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACrD,CAAC;QAED,8CAA8C;QAC9C,MAAM,UAAU,GAA8C,EAAE,CAAC;QAEjE,6BAA6B;QAC7B,MAAM,cAAc,GAAG,WAAW,CAAC,QAAQ,CAAC,0HAA0H,CAAC,CAAC;QACxK,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YACvF,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;gBACrB,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QAED,oDAAoD;QACpD,MAAM,iBAAiB,GAAG,WAAW,CAAC,QAAQ,CAAC,8BAA8B,CAAC,CAAC;QAC/E,KAAK,MAAM,KAAK,IAAI,iBAAiB,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACzC,kDAAkD;gBAClD,UAAU,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;QAED,sDAAsD;QACtD,IAAI,WAAW,GAAG,WAAW;YAC3B,6BAA6B;aAC5B,OAAO,CAAC,0HAA0H,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YAC7J,MAAM,IAAI,GAAG,CAAC,KAAK,IAAI,KAAK,IAAI,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YACzE,OAAO,WAAW,IAAI,KAAK,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC;QACrD,CAAC,CAAC;YACF,UAAU;aACT,OAAO,CAAC,oCAAoC,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;YAChE,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;YAC3C,OAAO,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC;QACxC,CAAC,CAAC;YACF,QAAQ;aACP,OAAO,CAAC,kDAAkD,EAAE,UAAU,CAAC;YACxE,cAAc;aACb,OAAO,CAAC,8BAA8B,EAAE,MAAM,CAAC;YAChD,QAAQ;aACP,OAAO,CAAC,0BAA0B,EAAE,MAAM,CAAC;YAC5C,aAAa;aACZ,OAAO,CAAC,wBAAwB,EAAE,MAAM,CAAC;YAC1C,cAAc;aACb,OAAO,CAAC,8CAA8C,EAAE,QAAQ,CAAC;aACjE,OAAO,CAAC,sCAAsC,EAAE,MAAM,CAAC;YACxD,6BAA6B;aAC5B,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;YACzB,sBAAsB;aACrB,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;aAC1B,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;aACvB,IAAI,EAAE,CAAC;QAEV,mBAAmB;QACnB,MAAM,QAAQ,GAA+D,EAAE,CAAC;QAChF,MAAM,cAAc,GAAG,WAAW,CAAC,QAAQ,CAAC,6CAA6C,CAAC,CAAC;QAE3F,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YACpC,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC;YAEpC,MAAM,iBAAiB,GAAG,IAAI,MAAM,CAAC,SAAS,WAAW,OAAO,EAAE,GAAG,CAAC,CAAC;YACvE,iBAAiB,CAAC,SAAS,GAAG,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YAC3D,MAAM,SAAS,GAAG,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACtD,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC;YAElE,MAAM,cAAc,GAAG,WAAW,CAAC,SAAS,CAAC,UAAU,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;YAE5F,IAAI,cAAc,EAAE,CAAC;gBACnB,QAAQ,CAAC,IAAI,CAAC;oBACZ,KAAK,EAAE,UAAU;oBACjB,OAAO,EAAE,cAAc;oBACvB,MAAM,EAAE,IAAI,UAAU,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE;iBAC5D,CAAC,CAAC;YACL,CAAC;YAED,SAAS,GAAG,QAAQ,CAAC;QACvB,CAAC;QAED,oDAAoD;QACpD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,WAAW,EAAE,CAAC;YACzC,QAAQ,CAAC,IAAI,CAAC;gBACZ,KAAK,EAAE,KAAK;gBACZ,OAAO,EAAE,WAAW;aACrB,CAAC,CAAC;QACL,CAAC;QAED,kDAAkD;QAClD,MAAM,kBAAkB,GAAG,WAAW,CAAC,QAAQ,CAAC,qDAAqD,CAAC,CAAC;QACvG,KAAK,MAAM,KAAK,IAAI,kBAAkB,EAAE,CAAC;YACvC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YACnD,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC1C,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;wBACrC,CAAC,CAAC,IAAI;wBACN,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC;oBAChC,MAAM,aAAa,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC5C,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC;oBAE1E,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE,CAAC;wBAC3C,eAAe,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,aAAa,EAAE,UAAU,EAAE,CAAC,CAAC;wBAC/D,kBAAkB,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;oBACxC,CAAC;gBACH,CAAC;gBAAC,MAAM,CAAC;oBACP,oBAAoB;gBACtB,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,UAAU,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,qBAAqB;QAE5E,MAAM,CAAC,KAAK,CAAC,8BAA8B,EAAE;YAC3C,GAAG,EAAE,UAAU;YACf,KAAK;YACL,aAAa,EAAE,QAAQ,CAAC,MAAM;YAC9B,eAAe,EAAE,UAAU,CAAC,MAAM;YAClC,UAAU,EAAE,UAAU,CAAC,MAAM;YAC7B,aAAa,EAAE,WAAW,CAAC,MAAM;SAClC,CAAC,CAAC;QAEH,OAAO;YACL,KAAK;YACL,OAAO,EAAE,WAAW,IAAI,sBAAsB;YAC9C,eAAe,EAAE,UAAU;YAC3B,QAAQ;YACR,iBAAiB,EAAE,WAAW,CAAC,WAAW,EAAE;SAC7C,CAAC;IACJ,CAAC;IAED;;OAEG;IACK,cAAc,CAAC,IAAY;QACjC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAE5C,kBAAkB;QAClB,IAAI,gEAAgE,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACrF,IAAI,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC1E,OAAO,KAAK,CAAC;YACf,CAAC;YACD,IAAI,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,oCAAoC,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC1F,OAAO,YAAY,CAAC;YACtB,CAAC;YACD,OAAO,YAAY,CAAC;QACtB,CAAC;QAED,IAAI,4BAA4B,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACjD,OAAO,QAAQ,CAAC;QAClB,CAAC;QAED,IAAI,0CAA0C,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YAC/D,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,IAAI,4CAA4C,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACjE,OAAO,GAAG,CAAC;QACb,CAAC;QAED,IAAI,0BAA0B,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YAC/C,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,IAAI,kBAAkB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACvC,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,MAAM,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACxB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACnB,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YAC3B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;QACD,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;IACjC,CAAC;CACF"}
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
import type { Documentation } from '../types/documentation.js';
|
|
2
|
-
export declare class DocumentationCache {
|
|
3
|
-
private cache;
|
|
4
|
-
private readonly defaultTTL;
|
|
5
|
-
constructor(defaultTTLMinutes?: number);
|
|
6
|
-
get(key: string): Documentation | null;
|
|
7
|
-
set(key: string, documentation: Documentation, ttl?: number): void;
|
|
8
|
-
generateKey(packageName: string, version?: string): string;
|
|
9
|
-
clear(): void;
|
|
10
|
-
size(): number;
|
|
11
|
-
cleanup(): number;
|
|
12
|
-
}
|
|
13
|
-
//# sourceMappingURL=cache.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../src/documentation/cache.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAc,MAAM,2BAA2B,CAAC;AAE3E,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,KAAK,CAAsC;IACnD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;gBAExB,iBAAiB,GAAE,MAAW;IAI1C,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAetC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,aAAa,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI;IAUlE,WAAW,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM;IAI1D,KAAK,IAAI,IAAI;IAIb,IAAI,IAAI,MAAM;IAKd,OAAO,IAAI,MAAM;CAalB"}
|
|
@@ -1,48 +0,0 @@
|
|
|
1
|
-
export class DocumentationCache {
|
|
2
|
-
cache = new Map();
|
|
3
|
-
defaultTTL; // in milliseconds
|
|
4
|
-
constructor(defaultTTLMinutes = 60) {
|
|
5
|
-
this.defaultTTL = defaultTTLMinutes * 60 * 1000;
|
|
6
|
-
}
|
|
7
|
-
get(key) {
|
|
8
|
-
const entry = this.cache.get(key);
|
|
9
|
-
if (!entry) {
|
|
10
|
-
return null;
|
|
11
|
-
}
|
|
12
|
-
if (new Date() > entry.expiresAt) {
|
|
13
|
-
this.cache.delete(key);
|
|
14
|
-
return null;
|
|
15
|
-
}
|
|
16
|
-
return entry.documentation;
|
|
17
|
-
}
|
|
18
|
-
set(key, documentation, ttl) {
|
|
19
|
-
const expiresAt = new Date(Date.now() + (ttl || this.defaultTTL));
|
|
20
|
-
this.cache.set(key, {
|
|
21
|
-
documentation,
|
|
22
|
-
expiresAt,
|
|
23
|
-
version: documentation.version || 'latest',
|
|
24
|
-
});
|
|
25
|
-
}
|
|
26
|
-
generateKey(packageName, version) {
|
|
27
|
-
return version ? `${packageName}@${version}` : `${packageName}@latest`;
|
|
28
|
-
}
|
|
29
|
-
clear() {
|
|
30
|
-
this.cache.clear();
|
|
31
|
-
}
|
|
32
|
-
size() {
|
|
33
|
-
return this.cache.size;
|
|
34
|
-
}
|
|
35
|
-
// Clean expired entries
|
|
36
|
-
cleanup() {
|
|
37
|
-
const now = new Date();
|
|
38
|
-
let cleaned = 0;
|
|
39
|
-
for (const [key, entry] of this.cache.entries()) {
|
|
40
|
-
if (now > entry.expiresAt) {
|
|
41
|
-
this.cache.delete(key);
|
|
42
|
-
cleaned++;
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
return cleaned;
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
//# sourceMappingURL=cache.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../src/documentation/cache.ts"],"names":[],"mappings":"AAEA,MAAM,OAAO,kBAAkB;IACrB,KAAK,GAA4B,IAAI,GAAG,EAAE,CAAC;IAClC,UAAU,CAAS,CAAC,kBAAkB;IAEvD,YAAY,oBAA4B,EAAE;QACxC,IAAI,CAAC,UAAU,GAAG,iBAAiB,GAAG,EAAE,GAAG,IAAI,CAAC;IAClD,CAAC;IAED,GAAG,CAAC,GAAW;QACb,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAElC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,IAAI,IAAI,EAAE,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACvB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,KAAK,CAAC,aAAa,CAAC;IAC7B,CAAC;IAED,GAAG,CAAC,GAAW,EAAE,aAA4B,EAAE,GAAY;QACzD,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QAElE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE;YAClB,aAAa;YACb,SAAS;YACT,OAAO,EAAE,aAAa,CAAC,OAAO,IAAI,QAAQ;SAC3C,CAAC,CAAC;IACL,CAAC;IAED,WAAW,CAAC,WAAmB,EAAE,OAAgB;QAC/C,OAAO,OAAO,CAAC,CAAC,CAAC,GAAG,WAAW,IAAI,OAAO,EAAE,CAAC,CAAC,CAAC,GAAG,WAAW,SAAS,CAAC;IACzE,CAAC;IAED,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,wBAAwB;IACxB,OAAO;QACL,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YAChD,IAAI,GAAG,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;gBAC1B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACvB,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
import type { CrawlOptions } from './doc-crawler.js';
|
|
2
|
-
export interface CheckpointData {
|
|
3
|
-
config: CrawlOptions;
|
|
4
|
-
visitedUrls: string[];
|
|
5
|
-
pendingUrls: Array<{
|
|
6
|
-
url: string;
|
|
7
|
-
depth: number;
|
|
8
|
-
}>;
|
|
9
|
-
pagesCrawled: number;
|
|
10
|
-
lastUpdated: string;
|
|
11
|
-
baseUrl: string;
|
|
12
|
-
}
|
|
13
|
-
/**
|
|
14
|
-
* Manager for crawl checkpoint/resume functionality
|
|
15
|
-
* Based on Skill_Seekers implementation
|
|
16
|
-
*/
|
|
17
|
-
export declare class CheckpointManager {
|
|
18
|
-
private checkpointFile;
|
|
19
|
-
constructor(checkpointFile: string);
|
|
20
|
-
/**
|
|
21
|
-
* Save checkpoint data to file
|
|
22
|
-
*/
|
|
23
|
-
saveCheckpoint(data: CheckpointData): Promise<void>;
|
|
24
|
-
/**
|
|
25
|
-
* Load checkpoint data from file
|
|
26
|
-
* Returns null if file doesn't exist or is invalid
|
|
27
|
-
*/
|
|
28
|
-
loadCheckpoint(): Promise<CheckpointData | null>;
|
|
29
|
-
/**
|
|
30
|
-
* Clear checkpoint file
|
|
31
|
-
*/
|
|
32
|
-
clearCheckpoint(): Promise<void>;
|
|
33
|
-
/**
|
|
34
|
-
* Check if checkpoint exists
|
|
35
|
-
*/
|
|
36
|
-
exists(): Promise<boolean>;
|
|
37
|
-
}
|
|
38
|
-
//# sourceMappingURL=checkpoint-manager.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"checkpoint-manager.d.ts","sourceRoot":"","sources":["../../src/documentation/checkpoint-manager.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAErD,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,YAAY,CAAC;IACrB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,WAAW,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACnD,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;GAGG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,cAAc,CAAS;gBAEnB,cAAc,EAAE,MAAM;IAIlC;;OAEG;IACG,cAAc,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBzD;;;OAGG;IACG,cAAc,IAAI,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAiCtD;;OAEG;IACG,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC;IAkBtC;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;CAQjC"}
|
|
@@ -1,101 +0,0 @@
|
|
|
1
|
-
import { writeFile, readFile, unlink, access } from 'fs/promises';
|
|
2
|
-
import { logger } from '../utils/logger.js';
|
|
3
|
-
/**
|
|
4
|
-
* Manager for crawl checkpoint/resume functionality
|
|
5
|
-
* Based on Skill_Seekers implementation
|
|
6
|
-
*/
|
|
7
|
-
export class CheckpointManager {
|
|
8
|
-
checkpointFile;
|
|
9
|
-
constructor(checkpointFile) {
|
|
10
|
-
this.checkpointFile = checkpointFile;
|
|
11
|
-
}
|
|
12
|
-
/**
|
|
13
|
-
* Save checkpoint data to file
|
|
14
|
-
*/
|
|
15
|
-
async saveCheckpoint(data) {
|
|
16
|
-
try {
|
|
17
|
-
const json = JSON.stringify(data, null, 2);
|
|
18
|
-
await writeFile(this.checkpointFile, json, 'utf-8');
|
|
19
|
-
logger.info('Checkpoint saved', {
|
|
20
|
-
file: this.checkpointFile,
|
|
21
|
-
pagesCrawled: data.pagesCrawled,
|
|
22
|
-
pendingUrls: data.pendingUrls.length,
|
|
23
|
-
visitedUrls: data.visitedUrls.length,
|
|
24
|
-
});
|
|
25
|
-
}
|
|
26
|
-
catch (error) {
|
|
27
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
28
|
-
logger.error('Failed to save checkpoint', {
|
|
29
|
-
file: this.checkpointFile,
|
|
30
|
-
error: errorMessage,
|
|
31
|
-
});
|
|
32
|
-
throw new Error(`Failed to save checkpoint: ${errorMessage}`);
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
/**
|
|
36
|
-
* Load checkpoint data from file
|
|
37
|
-
* Returns null if file doesn't exist or is invalid
|
|
38
|
-
*/
|
|
39
|
-
async loadCheckpoint() {
|
|
40
|
-
try {
|
|
41
|
-
// Check if file exists
|
|
42
|
-
await access(this.checkpointFile);
|
|
43
|
-
// Read and parse checkpoint
|
|
44
|
-
const content = await readFile(this.checkpointFile, 'utf-8');
|
|
45
|
-
const data = JSON.parse(content);
|
|
46
|
-
logger.info('Checkpoint loaded', {
|
|
47
|
-
file: this.checkpointFile,
|
|
48
|
-
pagesCrawled: data.pagesCrawled,
|
|
49
|
-
pendingUrls: data.pendingUrls.length,
|
|
50
|
-
visitedUrls: data.visitedUrls.length,
|
|
51
|
-
lastUpdated: data.lastUpdated,
|
|
52
|
-
});
|
|
53
|
-
return data;
|
|
54
|
-
}
|
|
55
|
-
catch (error) {
|
|
56
|
-
if (error.code === 'ENOENT') {
|
|
57
|
-
logger.debug('No checkpoint file found', { file: this.checkpointFile });
|
|
58
|
-
return null;
|
|
59
|
-
}
|
|
60
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
61
|
-
logger.warn('Failed to load checkpoint', {
|
|
62
|
-
file: this.checkpointFile,
|
|
63
|
-
error: errorMessage,
|
|
64
|
-
});
|
|
65
|
-
return null;
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
/**
|
|
69
|
-
* Clear checkpoint file
|
|
70
|
-
*/
|
|
71
|
-
async clearCheckpoint() {
|
|
72
|
-
try {
|
|
73
|
-
await unlink(this.checkpointFile);
|
|
74
|
-
logger.info('Checkpoint cleared', { file: this.checkpointFile });
|
|
75
|
-
}
|
|
76
|
-
catch (error) {
|
|
77
|
-
if (error.code === 'ENOENT') {
|
|
78
|
-
// File doesn't exist, that's fine
|
|
79
|
-
return;
|
|
80
|
-
}
|
|
81
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
82
|
-
logger.warn('Failed to clear checkpoint', {
|
|
83
|
-
file: this.checkpointFile,
|
|
84
|
-
error: errorMessage,
|
|
85
|
-
});
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
/**
|
|
89
|
-
* Check if checkpoint exists
|
|
90
|
-
*/
|
|
91
|
-
async exists() {
|
|
92
|
-
try {
|
|
93
|
-
await access(this.checkpointFile);
|
|
94
|
-
return true;
|
|
95
|
-
}
|
|
96
|
-
catch {
|
|
97
|
-
return false;
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
//# sourceMappingURL=checkpoint-manager.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"checkpoint-manager.js","sourceRoot":"","sources":["../../src/documentation/checkpoint-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAY5C;;;GAGG;AACH,MAAM,OAAO,iBAAiB;IACpB,cAAc,CAAS;IAE/B,YAAY,cAAsB;QAChC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,IAAoB;QACvC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAC3C,MAAM,SAAS,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;YAEpD,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE;gBAC9B,IAAI,EAAE,IAAI,CAAC,cAAc;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM;gBACpC,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM;aACrC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5E,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE;gBACxC,IAAI,EAAE,IAAI,CAAC,cAAc;gBACzB,KAAK,EAAE,YAAY;aACpB,CAAC,CAAC;YACH,MAAM,IAAI,KAAK,CAAC,8BAA8B,YAAY,EAAE,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc;QAClB,IAAI,CAAC;YACH,uBAAuB;YACvB,MAAM,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAElC,4BAA4B;YAC5B,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;YAC7D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAmB,CAAC;YAEnD,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE;gBAC/B,IAAI,EAAE,IAAI,CAAC,cAAc;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM;gBACpC,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM;gBACpC,WAAW,EAAE,IAAI,CAAC,WAAW;aAC9B,CAAC,CAAC;YAEH,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACvD,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC;gBACxE,OAAO,IAAI,CAAC;YACd,CAAC;YAED,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5E,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE;gBACvC,IAAI,EAAE,IAAI,CAAC,cAAc;gBACzB,KAAK,EAAE,YAAY;aACpB,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe;QACnB,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAClC,MAAM,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC;QACnE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACvD,kCAAkC;gBAClC,OAAO;YACT,CAAC;YAED,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5E,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE;gBACxC,IAAI,EAAE,IAAI,CAAC,cAAc;gBACzB,KAAK,EAAE,YAAY;aACpB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM;QACV,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAClC,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;CACF"}
|