@gulibs/safe-coder 0.0.26 → 0.0.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +678 -994
- package/dist/cache/cache-manager.d.ts +71 -0
- package/dist/cache/cache-manager.d.ts.map +1 -0
- package/dist/cache/cache-manager.js +244 -0
- package/dist/cache/cache-manager.js.map +1 -0
- package/dist/executor/cli-executor.d.ts +106 -0
- package/dist/executor/cli-executor.d.ts.map +1 -0
- package/dist/executor/cli-executor.js +133 -0
- package/dist/executor/cli-executor.js.map +1 -0
- package/dist/executor/dependency-checker.d.ts +23 -0
- package/dist/executor/dependency-checker.d.ts.map +1 -0
- package/dist/executor/dependency-checker.js +62 -0
- package/dist/executor/dependency-checker.js.map +1 -0
- package/dist/index.js +3 -4
- package/dist/index.js.map +1 -1
- package/dist/processor/content-processor.d.ts +76 -0
- package/dist/processor/content-processor.d.ts.map +1 -0
- package/dist/processor/content-processor.js +182 -0
- package/dist/processor/content-processor.js.map +1 -0
- package/dist/processor/guide-generator.d.ts +68 -0
- package/dist/processor/guide-generator.d.ts.map +1 -0
- package/dist/processor/guide-generator.js +189 -0
- package/dist/processor/guide-generator.js.map +1 -0
- package/dist/server/safe-coder-mcp.d.ts +18 -0
- package/dist/server/safe-coder-mcp.d.ts.map +1 -0
- package/dist/server/safe-coder-mcp.js +164 -0
- package/dist/server/safe-coder-mcp.js.map +1 -0
- package/dist/tools/cache-tools.d.ts +42 -0
- package/dist/tools/cache-tools.d.ts.map +1 -0
- package/dist/tools/cache-tools.js +70 -0
- package/dist/tools/cache-tools.js.map +1 -0
- package/dist/tools/crawl-documentation.d.ts +57 -0
- package/dist/tools/crawl-documentation.d.ts.map +1 -0
- package/dist/tools/crawl-documentation.js +96 -0
- package/dist/tools/crawl-documentation.js.map +1 -0
- package/dist/tools/index.d.ts +4 -0
- package/dist/tools/index.d.ts.map +1 -0
- package/dist/tools/index.js +4 -0
- package/dist/tools/index.js.map +1 -0
- package/dist/tools/save-skill.d.ts +49 -0
- package/dist/tools/save-skill.d.ts.map +1 -0
- package/dist/tools/save-skill.js +207 -0
- package/dist/tools/save-skill.js.map +1 -0
- package/package.json +18 -28
- package/dist/documentation/browser-doc-browser.d.ts +0 -41
- package/dist/documentation/browser-doc-browser.d.ts.map +0 -1
- package/dist/documentation/browser-doc-browser.js +0 -357
- package/dist/documentation/browser-doc-browser.js.map +0 -1
- package/dist/documentation/browser-manager.d.ts +0 -51
- package/dist/documentation/browser-manager.d.ts.map +0 -1
- package/dist/documentation/browser-manager.js +0 -260
- package/dist/documentation/browser-manager.js.map +0 -1
- package/dist/documentation/cache.d.ts +0 -13
- package/dist/documentation/cache.d.ts.map +0 -1
- package/dist/documentation/cache.js +0 -48
- package/dist/documentation/cache.js.map +0 -1
- package/dist/documentation/checkpoint-manager.d.ts +0 -38
- package/dist/documentation/checkpoint-manager.d.ts.map +0 -1
- package/dist/documentation/checkpoint-manager.js +0 -101
- package/dist/documentation/checkpoint-manager.js.map +0 -1
- package/dist/documentation/doc-crawler.d.ts +0 -221
- package/dist/documentation/doc-crawler.d.ts.map +0 -1
- package/dist/documentation/doc-crawler.js +0 -1415
- package/dist/documentation/doc-crawler.js.map +0 -1
- package/dist/documentation/github-client.d.ts +0 -13
- package/dist/documentation/github-client.d.ts.map +0 -1
- package/dist/documentation/github-client.js +0 -90
- package/dist/documentation/github-client.js.map +0 -1
- package/dist/documentation/http-fetcher.d.ts +0 -8
- package/dist/documentation/http-fetcher.d.ts.map +0 -1
- package/dist/documentation/http-fetcher.js +0 -31
- package/dist/documentation/http-fetcher.js.map +0 -1
- package/dist/documentation/index.d.ts +0 -16
- package/dist/documentation/index.d.ts.map +0 -1
- package/dist/documentation/index.js +0 -159
- package/dist/documentation/index.js.map +0 -1
- package/dist/documentation/llms-txt/detector.d.ts +0 -31
- package/dist/documentation/llms-txt/detector.d.ts.map +0 -1
- package/dist/documentation/llms-txt/detector.js +0 -77
- package/dist/documentation/llms-txt/detector.js.map +0 -1
- package/dist/documentation/llms-txt/downloader.d.ts +0 -30
- package/dist/documentation/llms-txt/downloader.d.ts.map +0 -1
- package/dist/documentation/llms-txt/downloader.js +0 -84
- package/dist/documentation/llms-txt/downloader.js.map +0 -1
- package/dist/documentation/llms-txt/index.d.ts +0 -4
- package/dist/documentation/llms-txt/index.d.ts.map +0 -1
- package/dist/documentation/llms-txt/index.js +0 -4
- package/dist/documentation/llms-txt/index.js.map +0 -1
- package/dist/documentation/llms-txt/parser.d.ts +0 -43
- package/dist/documentation/llms-txt/parser.d.ts.map +0 -1
- package/dist/documentation/llms-txt/parser.js +0 -177
- package/dist/documentation/llms-txt/parser.js.map +0 -1
- package/dist/documentation/normalizer.d.ts +0 -6
- package/dist/documentation/normalizer.d.ts.map +0 -1
- package/dist/documentation/normalizer.js +0 -38
- package/dist/documentation/normalizer.js.map +0 -1
- package/dist/documentation/npm-client.d.ts +0 -19
- package/dist/documentation/npm-client.d.ts.map +0 -1
- package/dist/documentation/npm-client.js +0 -182
- package/dist/documentation/npm-client.js.map +0 -1
- package/dist/documentation/skill-generator.d.ts +0 -108
- package/dist/documentation/skill-generator.d.ts.map +0 -1
- package/dist/documentation/skill-generator.js +0 -642
- package/dist/documentation/skill-generator.js.map +0 -1
- package/dist/documentation/web-doc-browser.d.ts +0 -67
- package/dist/documentation/web-doc-browser.d.ts.map +0 -1
- package/dist/documentation/web-doc-browser.js +0 -555
- package/dist/documentation/web-doc-browser.js.map +0 -1
- package/dist/errors/api-validator.d.ts +0 -9
- package/dist/errors/api-validator.d.ts.map +0 -1
- package/dist/errors/api-validator.js +0 -57
- package/dist/errors/api-validator.js.map +0 -1
- package/dist/errors/contextual-analysis.d.ts +0 -14
- package/dist/errors/contextual-analysis.d.ts.map +0 -1
- package/dist/errors/contextual-analysis.js +0 -173
- package/dist/errors/contextual-analysis.js.map +0 -1
- package/dist/errors/cross-file-analyzer.d.ts +0 -16
- package/dist/errors/cross-file-analyzer.d.ts.map +0 -1
- package/dist/errors/cross-file-analyzer.js +0 -172
- package/dist/errors/cross-file-analyzer.js.map +0 -1
- package/dist/errors/eslint-integration.d.ts +0 -9
- package/dist/errors/eslint-integration.d.ts.map +0 -1
- package/dist/errors/eslint-integration.js +0 -131
- package/dist/errors/eslint-integration.js.map +0 -1
- package/dist/errors/framework-detector.d.ts +0 -10
- package/dist/errors/framework-detector.d.ts.map +0 -1
- package/dist/errors/framework-detector.js +0 -126
- package/dist/errors/framework-detector.js.map +0 -1
- package/dist/errors/index.d.ts +0 -18
- package/dist/errors/index.d.ts.map +0 -1
- package/dist/errors/index.js +0 -134
- package/dist/errors/index.js.map +0 -1
- package/dist/errors/pattern-matcher.d.ts +0 -25
- package/dist/errors/pattern-matcher.d.ts.map +0 -1
- package/dist/errors/pattern-matcher.js +0 -44
- package/dist/errors/pattern-matcher.js.map +0 -1
- package/dist/errors/patterns.d.ts +0 -11
- package/dist/errors/patterns.d.ts.map +0 -1
- package/dist/errors/patterns.js +0 -351
- package/dist/errors/patterns.js.map +0 -1
- package/dist/errors/performance-detector.d.ts +0 -11
- package/dist/errors/performance-detector.d.ts.map +0 -1
- package/dist/errors/performance-detector.js +0 -119
- package/dist/errors/performance-detector.js.map +0 -1
- package/dist/errors/runtime-detector.d.ts +0 -7
- package/dist/errors/runtime-detector.d.ts.map +0 -1
- package/dist/errors/runtime-detector.js +0 -86
- package/dist/errors/runtime-detector.js.map +0 -1
- package/dist/errors/security-detector.d.ts +0 -6
- package/dist/errors/security-detector.d.ts.map +0 -1
- package/dist/errors/security-detector.js +0 -75
- package/dist/errors/security-detector.js.map +0 -1
- package/dist/errors/typescript-integration.d.ts +0 -6
- package/dist/errors/typescript-integration.d.ts.map +0 -1
- package/dist/errors/typescript-integration.js +0 -46
- package/dist/errors/typescript-integration.js.map +0 -1
- package/dist/server/mcp-server.d.ts +0 -14
- package/dist/server/mcp-server.d.ts.map +0 -1
- package/dist/server/mcp-server.js +0 -793
- package/dist/server/mcp-server.js.map +0 -1
- package/dist/types/documentation.d.ts +0 -26
- package/dist/types/documentation.d.ts.map +0 -1
- package/dist/types/documentation.js +0 -2
- package/dist/types/documentation.js.map +0 -1
- package/dist/utils/config.d.ts +0 -21
- package/dist/utils/config.d.ts.map +0 -1
- package/dist/utils/config.js +0 -34
- package/dist/utils/config.js.map +0 -1
- package/dist/utils/http-client.d.ts +0 -17
- package/dist/utils/http-client.d.ts.map +0 -1
- package/dist/utils/http-client.js +0 -62
- package/dist/utils/http-client.js.map +0 -1
- package/dist/utils/logger.d.ts +0 -36
- package/dist/utils/logger.d.ts.map +0 -1
- package/dist/utils/logger.js +0 -128
- package/dist/utils/logger.js.map +0 -1
- package/dist/utils/rate-limiter.d.ts +0 -9
- package/dist/utils/rate-limiter.d.ts.map +0 -1
- package/dist/utils/rate-limiter.js +0 -26
- package/dist/utils/rate-limiter.js.map +0 -1
- package/dist/validation/auto-fix.d.ts +0 -15
- package/dist/validation/auto-fix.d.ts.map +0 -1
- package/dist/validation/auto-fix.js +0 -49
- package/dist/validation/auto-fix.js.map +0 -1
- package/dist/validation/index.d.ts +0 -21
- package/dist/validation/index.d.ts.map +0 -1
- package/dist/validation/index.js +0 -45
- package/dist/validation/index.js.map +0 -1
- package/dist/validation/resolution-db.d.ts +0 -15
- package/dist/validation/resolution-db.d.ts.map +0 -1
- package/dist/validation/resolution-db.js +0 -62
- package/dist/validation/resolution-db.js.map +0 -1
|
@@ -1,260 +0,0 @@
|
|
|
1
|
-
import { launch } from 'puppeteer-core';
|
|
2
|
-
import * as chromeLauncher from 'chrome-launcher';
|
|
3
|
-
import { logger } from '../utils/logger.js';
|
|
4
|
-
import { platform } from 'os';
|
|
5
|
-
import { existsSync } from 'fs';
|
|
6
|
-
import { access } from 'fs/promises';
|
|
7
|
-
/**
|
|
8
|
-
* Browser manager for SPA rendering using puppeteer-core
|
|
9
|
-
* Detects and uses system Chrome/Chromium/Edge
|
|
10
|
-
*/
|
|
11
|
-
export class BrowserManager {
|
|
12
|
-
browser;
|
|
13
|
-
config;
|
|
14
|
-
// Browser paths by platform
|
|
15
|
-
BROWSER_PATHS = {
|
|
16
|
-
darwin: [
|
|
17
|
-
'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
|
|
18
|
-
'/Applications/Chromium.app/Contents/MacOS/Chromium',
|
|
19
|
-
'/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge',
|
|
20
|
-
],
|
|
21
|
-
win32: [
|
|
22
|
-
'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe',
|
|
23
|
-
'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
|
|
24
|
-
'C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe',
|
|
25
|
-
'C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe',
|
|
26
|
-
],
|
|
27
|
-
linux: [
|
|
28
|
-
'/usr/bin/google-chrome',
|
|
29
|
-
'/usr/bin/google-chrome-stable',
|
|
30
|
-
'/usr/bin/chromium-browser',
|
|
31
|
-
'/usr/bin/chromium',
|
|
32
|
-
'/snap/bin/chromium',
|
|
33
|
-
'/usr/bin/microsoft-edge',
|
|
34
|
-
],
|
|
35
|
-
};
|
|
36
|
-
constructor() {
|
|
37
|
-
this.config = {
|
|
38
|
-
executablePath: undefined,
|
|
39
|
-
headless: true,
|
|
40
|
-
timeout: 30000,
|
|
41
|
-
waitForTimeout: 3000,
|
|
42
|
-
networkIdleTimeout: 500,
|
|
43
|
-
};
|
|
44
|
-
}
|
|
45
|
-
/**
|
|
46
|
-
* Detect system browser
|
|
47
|
-
* Returns path to Chrome/Chromium/Edge or null
|
|
48
|
-
*/
|
|
49
|
-
async detectBrowser() {
|
|
50
|
-
logger.info('Detecting system browser');
|
|
51
|
-
// 1. Check environment variables
|
|
52
|
-
const envPaths = [
|
|
53
|
-
process.env.CHROME_PATH,
|
|
54
|
-
process.env.PUPPETEER_EXECUTABLE_PATH,
|
|
55
|
-
process.env.CHROMIUM_PATH,
|
|
56
|
-
];
|
|
57
|
-
for (const envPath of envPaths) {
|
|
58
|
-
if (envPath && existsSync(envPath)) {
|
|
59
|
-
logger.info('Found browser via environment variable', { path: envPath });
|
|
60
|
-
return envPath;
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
// 2. Use chrome-launcher to find Chrome
|
|
64
|
-
try {
|
|
65
|
-
const installations = await chromeLauncher.Launcher.getInstallations();
|
|
66
|
-
if (installations.length > 0) {
|
|
67
|
-
logger.info('Found browser via chrome-launcher', { path: installations[0] });
|
|
68
|
-
return installations[0];
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
catch (error) {
|
|
72
|
-
logger.debug('chrome-launcher detection failed', {
|
|
73
|
-
error: error instanceof Error ? error.message : String(error),
|
|
74
|
-
});
|
|
75
|
-
}
|
|
76
|
-
// 3. Check common paths for current platform
|
|
77
|
-
const platformKey = platform();
|
|
78
|
-
const paths = this.BROWSER_PATHS[platformKey] || [];
|
|
79
|
-
for (const path of paths) {
|
|
80
|
-
try {
|
|
81
|
-
await access(path);
|
|
82
|
-
logger.info('Found browser at common path', { path, platform: platformKey });
|
|
83
|
-
return path;
|
|
84
|
-
}
|
|
85
|
-
catch {
|
|
86
|
-
// Path doesn't exist, continue
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
// 4. Try to use bundled puppeteer if installed
|
|
90
|
-
try {
|
|
91
|
-
const puppeteer = await import('puppeteer');
|
|
92
|
-
if (puppeteer.executablePath) {
|
|
93
|
-
const bundledPath = puppeteer.executablePath();
|
|
94
|
-
logger.info('Found bundled Chromium from puppeteer', { path: bundledPath });
|
|
95
|
-
return bundledPath;
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
catch {
|
|
99
|
-
// puppeteer not installed
|
|
100
|
-
logger.debug('Bundled puppeteer not available');
|
|
101
|
-
}
|
|
102
|
-
logger.warn('No browser found on system');
|
|
103
|
-
return null;
|
|
104
|
-
}
|
|
105
|
-
/**
|
|
106
|
-
* Launch browser instance
|
|
107
|
-
*/
|
|
108
|
-
async launch(config) {
|
|
109
|
-
if (this.browser) {
|
|
110
|
-
logger.debug('Browser already launched');
|
|
111
|
-
return this.browser;
|
|
112
|
-
}
|
|
113
|
-
// Merge config
|
|
114
|
-
this.config = {
|
|
115
|
-
...this.config,
|
|
116
|
-
...config,
|
|
117
|
-
};
|
|
118
|
-
// Detect browser if not specified
|
|
119
|
-
if (!this.config.executablePath) {
|
|
120
|
-
const detected = await this.detectBrowser();
|
|
121
|
-
if (!detected) {
|
|
122
|
-
throw new Error('No browser found. Install Chrome or set CHROME_PATH environment variable. ' +
|
|
123
|
-
'See docs/SPA_BROWSER_SETUP.md for details.');
|
|
124
|
-
}
|
|
125
|
-
this.config.executablePath = detected;
|
|
126
|
-
}
|
|
127
|
-
logger.info('Launching browser', {
|
|
128
|
-
executablePath: this.config.executablePath,
|
|
129
|
-
headless: this.config.headless,
|
|
130
|
-
});
|
|
131
|
-
try {
|
|
132
|
-
this.browser = await launch({
|
|
133
|
-
executablePath: this.config.executablePath,
|
|
134
|
-
headless: this.config.headless,
|
|
135
|
-
args: [
|
|
136
|
-
'--no-sandbox',
|
|
137
|
-
'--disable-setuid-sandbox',
|
|
138
|
-
'--disable-dev-shm-usage',
|
|
139
|
-
'--disable-accelerated-2d-canvas',
|
|
140
|
-
'--disable-gpu',
|
|
141
|
-
'--no-first-run',
|
|
142
|
-
'--no-zygote',
|
|
143
|
-
'--disable-background-networking',
|
|
144
|
-
'--disable-background-timer-throttling',
|
|
145
|
-
'--disable-backgrounding-occluded-windows',
|
|
146
|
-
'--disable-renderer-backgrounding',
|
|
147
|
-
],
|
|
148
|
-
});
|
|
149
|
-
logger.info('Browser launched successfully');
|
|
150
|
-
return this.browser;
|
|
151
|
-
}
|
|
152
|
-
catch (error) {
|
|
153
|
-
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
154
|
-
logger.error('Failed to launch browser', { error: errorMsg });
|
|
155
|
-
throw new Error(`Failed to launch browser: ${errorMsg}`);
|
|
156
|
-
}
|
|
157
|
-
}
|
|
158
|
-
/**
|
|
159
|
-
* Render a page and extract content
|
|
160
|
-
* Uses balanced performance mode
|
|
161
|
-
*/
|
|
162
|
-
async renderPage(url) {
|
|
163
|
-
if (!this.browser) {
|
|
164
|
-
throw new Error('Browser not launched. Call launch() first.');
|
|
165
|
-
}
|
|
166
|
-
logger.info('Rendering page with browser', { url });
|
|
167
|
-
let page;
|
|
168
|
-
try {
|
|
169
|
-
page = await this.browser.newPage();
|
|
170
|
-
// Set viewport
|
|
171
|
-
await page.setViewport({ width: 1280, height: 800 });
|
|
172
|
-
// Optimize: disable unnecessary resources (balanced mode)
|
|
173
|
-
await page.setRequestInterception(true);
|
|
174
|
-
page.on('request', (req) => {
|
|
175
|
-
const resourceType = req.resourceType();
|
|
176
|
-
if (['image', 'stylesheet', 'font', 'media'].includes(resourceType)) {
|
|
177
|
-
req.abort();
|
|
178
|
-
}
|
|
179
|
-
else {
|
|
180
|
-
req.continue();
|
|
181
|
-
}
|
|
182
|
-
});
|
|
183
|
-
// Navigate to page
|
|
184
|
-
await page.goto(url, {
|
|
185
|
-
waitUntil: 'domcontentloaded', // Wait for DOM
|
|
186
|
-
timeout: this.config.timeout,
|
|
187
|
-
});
|
|
188
|
-
// Wait for main content (balanced mode)
|
|
189
|
-
await page.waitForTimeout(this.config.waitForTimeout);
|
|
190
|
-
// Try to wait for network idle (with fallback)
|
|
191
|
-
try {
|
|
192
|
-
await page.waitForNetworkIdle({
|
|
193
|
-
timeout: this.config.networkIdleTimeout,
|
|
194
|
-
});
|
|
195
|
-
}
|
|
196
|
-
catch (error) {
|
|
197
|
-
// Network idle timeout is not critical, continue
|
|
198
|
-
logger.debug('Network idle timeout, continuing', { url });
|
|
199
|
-
}
|
|
200
|
-
// Extract content using page.evaluate
|
|
201
|
-
// Using string evaluation to avoid DOM type errors in Node.js context
|
|
202
|
-
const result = await page.evaluate(`
|
|
203
|
-
(() => {
|
|
204
|
-
return {
|
|
205
|
-
html: document.documentElement.outerHTML,
|
|
206
|
-
title: document.title,
|
|
207
|
-
url: window.location.href,
|
|
208
|
-
links: Array.from(document.querySelectorAll('a')).map(a => ({
|
|
209
|
-
text: a.textContent?.trim() || '',
|
|
210
|
-
url: a.href,
|
|
211
|
-
})),
|
|
212
|
-
};
|
|
213
|
-
})()
|
|
214
|
-
`);
|
|
215
|
-
logger.info('Page rendered successfully', {
|
|
216
|
-
url,
|
|
217
|
-
title: result.title.substring(0, 50),
|
|
218
|
-
linksFound: result.links.length,
|
|
219
|
-
htmlSize: result.html.length,
|
|
220
|
-
});
|
|
221
|
-
return result;
|
|
222
|
-
}
|
|
223
|
-
catch (error) {
|
|
224
|
-
const errorMsg = error instanceof Error ? error.message : String(error);
|
|
225
|
-
logger.error('Failed to render page', { url, error: errorMsg });
|
|
226
|
-
throw error;
|
|
227
|
-
}
|
|
228
|
-
finally {
|
|
229
|
-
// Always close the page to free resources
|
|
230
|
-
if (page) {
|
|
231
|
-
await page.close();
|
|
232
|
-
}
|
|
233
|
-
}
|
|
234
|
-
}
|
|
235
|
-
/**
|
|
236
|
-
* Close browser instance and cleanup resources
|
|
237
|
-
*/
|
|
238
|
-
async close() {
|
|
239
|
-
if (this.browser) {
|
|
240
|
-
logger.info('Closing browser');
|
|
241
|
-
try {
|
|
242
|
-
await this.browser.close();
|
|
243
|
-
this.browser = undefined;
|
|
244
|
-
logger.info('Browser closed successfully');
|
|
245
|
-
}
|
|
246
|
-
catch (error) {
|
|
247
|
-
logger.warn('Error closing browser', {
|
|
248
|
-
error: error instanceof Error ? error.message : String(error),
|
|
249
|
-
});
|
|
250
|
-
}
|
|
251
|
-
}
|
|
252
|
-
}
|
|
253
|
-
/**
|
|
254
|
-
* Check if browser is currently launched
|
|
255
|
-
*/
|
|
256
|
-
isLaunched() {
|
|
257
|
-
return this.browser !== undefined;
|
|
258
|
-
}
|
|
259
|
-
}
|
|
260
|
-
//# sourceMappingURL=browser-manager.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"browser-manager.js","sourceRoot":"","sources":["../../src/documentation/browser-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAA2B,MAAM,gBAAgB,CAAC;AACjE,OAAO,KAAK,cAAc,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EAAE,QAAQ,EAAE,MAAM,IAAI,CAAC;AAC9B,OAAO,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAChC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AA0BrC;;;GAGG;AACH,MAAM,OAAO,cAAc;IACjB,OAAO,CAAW;IAClB,MAAM,CAAwB;IAEtC,4BAA4B;IACX,aAAa,GAAG;QAC/B,MAAM,EAAE;YACN,8DAA8D;YAC9D,oDAAoD;YACpD,gEAAgE;SACjE;QACD,KAAK,EAAE;YACL,4DAA4D;YAC5D,kEAAkE;YAClE,6DAA6D;YAC7D,mEAAmE;SACpE;QACD,KAAK,EAAE;YACL,wBAAwB;YACxB,+BAA+B;YAC/B,2BAA2B;YAC3B,mBAAmB;YACnB,oBAAoB;YACpB,yBAAyB;SAC1B;KACF,CAAC;IAEF;QACE,IAAI,CAAC,MAAM,GAAG;YACZ,cAAc,EAAE,SAAS;YACzB,QAAQ,EAAE,IAAI;YACd,OAAO,EAAE,KAAK;YACd,cAAc,EAAE,IAAI;YACpB,kBAAkB,EAAE,GAAG;SACxB,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,aAAa;QACjB,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAExC,iCAAiC;QACjC,MAAM,QAAQ,GAAG;YACf,OAAO,CAAC,GAAG,CAAC,WAAW;YACvB,OAAO,CAAC,GAAG,CAAC,yBAAyB;YACrC,OAAO,CAAC,GAAG,CAAC,aAAa;SAC1B,CAAC;QAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,OAAO,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;gBACnC,MAAM,CAAC,IAAI,CAAC,wCAAwC,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;gBACzE,OAAO,OAAO,CAAC;YACjB,CAAC;QACH,CAAC;QAED,wCAAwC;QACxC,IAAI,CAAC;YACH,MAAM,aAAa,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,gBAAgB,EAAE,CAAC;YACvE,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC7B,MAAM,CAAC,IAAI,CAAC,mCAAmC,EAAE,EAAE,IAAI,EAAE,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC7E,OAAO,aAAa,CAAC,CAAC,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE;gBAC/C,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC9D,CAAC,CAAC;QACL,CAAC;QAED,6CAA6C;QAC7C,MAAM,WAAW,GAAG,QAAQ,EAAqC,CAAC;QAClE,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC;QAEpD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;gBACnB,MAAM,CAAC,IAAI,CAAC,8BAA8B,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC;gBAC7E,OAAO,IAAI,CAAC;YACd,CAAC;YAAC,MAAM,CAAC;gBACP,+BAA+B;YACjC,CAAC;QACH,CAAC;QAED,+CAA+C;QAC/C,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;YAC5C,IAAI,SAAS,CAAC,cAAc,EAAE,CAAC;gBAC7B,MAAM,WAAW,GAAG,SAAS,CAAC,cAAc,EAAE,CAAC;gBAC/C,MAAM,CAAC,IAAI,CAAC,uCAAuC,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;gBAC5E,OAAO,WAAW,CAAC;YACrB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,0BAA0B;YAC1B,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;QAC1C,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,MAA+B;QAC1C,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;YACzC,OAAO,IAAI,CAAC,OAAO,CAAC;QACtB,CAAC;QAED,eAAe;QACf,IAAI,CAAC,MAAM,GAAG;YACZ,GAAG,IAAI,CAAC,MAAM;YACd,GAAG,MAAM;SACV,CAAC;QAEF,kCAAkC;QAClC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YAChC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;YAC5C,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,IAAI,KAAK,CACb,4EAA4E;oBAC5E,4CAA4C,CAC7C,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,cAAc,GAAG,QAAQ,CAAC;QACxC,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE;YAC/B,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc;YAC1C,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ;SAC/B,CAAC,CAAC;QAEH,IAAI,CAAC;YACH,IAAI,CAAC,OAAO,GAAG,MAAM,MAAM,CAAC;gBAC1B,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC,cAAc;gBAC1C,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ;gBAC9B,IAAI,EAAE;oBACJ,cAAc;oBACd,0BAA0B;oBAC1B,yBAAyB;oBACzB,iCAAiC;oBACjC,eAAe;oBACf,gBAAgB;oBAChB,aAAa;oBACb,iCAAiC;oBACjC,uCAAuC;oBACvC,0CAA0C;oBAC1C,kCAAkC;iBACnC;aACF,CAAC,CAAC;YAEH,MAAM,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;YAC7C,OAAO,IAAI,CAAC,OAAO,CAAC;QACtB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,QAAQ,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACxE,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,6BAA6B,QAAQ,EAAE,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU,CAAC,GAAW;QAC1B,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAChE,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,6BAA6B,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;QAEpD,IAAI,IAAsB,CAAC;QAC3B,IAAI,CAAC;YACH,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YAEpC,eAAe;YACf,MAAM,IAAI,CAAC,WAAW,CAAC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;YAErD,0DAA0D;YAC1D,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;YACxC,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,EAAE;gBACzB,MAAM,YAAY,GAAG,GAAG,CAAC,YAAY,EAAE,CAAC;gBACxC,IAAI,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;oBACpE,GAAG,CAAC,KAAK,EAAE,CAAC;gBACd,CAAC;qBAAM,CAAC;oBACN,GAAG,CAAC,QAAQ,EAAE,CAAC;gBACjB,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,mBAAmB;YACnB,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE;gBACnB,SAAS,EAAE,kBAAkB,EAAE,eAAe;gBAC9C,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO;aAC7B,CAAC,CAAC;YAEH,wCAAwC;YACxC,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;YAEtD,+CAA+C;YAC/C,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,kBAAkB,CAAC;oBAC5B,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,kBAAkB;iBACxC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,iDAAiD;gBACjD,MAAM,CAAC,KAAK,CAAC,kCAAkC,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;YAC5D,CAAC;YAED,sCAAsC;YACtC,sEAAsE;YACtE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC;;;;;;;;;;;;OAYlC,CAAwB,CAAC;YAE1B,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE;gBACxC,GAAG;gBACH,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC;gBACpC,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM;gBAC/B,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM;aAC7B,CAAC,CAAC;YAEH,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,QAAQ,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACxE,MAAM,CAAC,KAAK,CAAC,uBAAuB,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;YAChE,MAAM,KAAK,CAAC;QACd,CAAC;gBAAS,CAAC;YACT,0CAA0C;YAC1C,IAAI,IAAI,EAAE,CAAC;gBACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;YAC/B,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;gBAC3B,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC;gBACzB,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;YAC7C,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,IAAI,CAAC,uBAAuB,EAAE;oBACnC,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;iBAC9D,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,UAAU;QACR,OAAO,IAAI,CAAC,OAAO,KAAK,SAAS,CAAC;IACpC,CAAC;CACF"}
|
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
import type { Documentation } from '../types/documentation.js';
|
|
2
|
-
export declare class DocumentationCache {
|
|
3
|
-
private cache;
|
|
4
|
-
private readonly defaultTTL;
|
|
5
|
-
constructor(defaultTTLMinutes?: number);
|
|
6
|
-
get(key: string): Documentation | null;
|
|
7
|
-
set(key: string, documentation: Documentation, ttl?: number): void;
|
|
8
|
-
generateKey(packageName: string, version?: string): string;
|
|
9
|
-
clear(): void;
|
|
10
|
-
size(): number;
|
|
11
|
-
cleanup(): number;
|
|
12
|
-
}
|
|
13
|
-
//# sourceMappingURL=cache.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../src/documentation/cache.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAc,MAAM,2BAA2B,CAAC;AAE3E,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,KAAK,CAAsC;IACnD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;gBAExB,iBAAiB,GAAE,MAAW;IAI1C,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI;IAetC,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,aAAa,EAAE,aAAa,EAAE,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI;IAUlE,WAAW,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM;IAI1D,KAAK,IAAI,IAAI;IAIb,IAAI,IAAI,MAAM;IAKd,OAAO,IAAI,MAAM;CAalB"}
|
|
@@ -1,48 +0,0 @@
|
|
|
1
|
-
export class DocumentationCache {
|
|
2
|
-
cache = new Map();
|
|
3
|
-
defaultTTL; // in milliseconds
|
|
4
|
-
constructor(defaultTTLMinutes = 60) {
|
|
5
|
-
this.defaultTTL = defaultTTLMinutes * 60 * 1000;
|
|
6
|
-
}
|
|
7
|
-
get(key) {
|
|
8
|
-
const entry = this.cache.get(key);
|
|
9
|
-
if (!entry) {
|
|
10
|
-
return null;
|
|
11
|
-
}
|
|
12
|
-
if (new Date() > entry.expiresAt) {
|
|
13
|
-
this.cache.delete(key);
|
|
14
|
-
return null;
|
|
15
|
-
}
|
|
16
|
-
return entry.documentation;
|
|
17
|
-
}
|
|
18
|
-
set(key, documentation, ttl) {
|
|
19
|
-
const expiresAt = new Date(Date.now() + (ttl || this.defaultTTL));
|
|
20
|
-
this.cache.set(key, {
|
|
21
|
-
documentation,
|
|
22
|
-
expiresAt,
|
|
23
|
-
version: documentation.version || 'latest',
|
|
24
|
-
});
|
|
25
|
-
}
|
|
26
|
-
generateKey(packageName, version) {
|
|
27
|
-
return version ? `${packageName}@${version}` : `${packageName}@latest`;
|
|
28
|
-
}
|
|
29
|
-
clear() {
|
|
30
|
-
this.cache.clear();
|
|
31
|
-
}
|
|
32
|
-
size() {
|
|
33
|
-
return this.cache.size;
|
|
34
|
-
}
|
|
35
|
-
// Clean expired entries
|
|
36
|
-
cleanup() {
|
|
37
|
-
const now = new Date();
|
|
38
|
-
let cleaned = 0;
|
|
39
|
-
for (const [key, entry] of this.cache.entries()) {
|
|
40
|
-
if (now > entry.expiresAt) {
|
|
41
|
-
this.cache.delete(key);
|
|
42
|
-
cleaned++;
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
return cleaned;
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
//# sourceMappingURL=cache.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../src/documentation/cache.ts"],"names":[],"mappings":"AAEA,MAAM,OAAO,kBAAkB;IACrB,KAAK,GAA4B,IAAI,GAAG,EAAE,CAAC;IAClC,UAAU,CAAS,CAAC,kBAAkB;IAEvD,YAAY,oBAA4B,EAAE;QACxC,IAAI,CAAC,UAAU,GAAG,iBAAiB,GAAG,EAAE,GAAG,IAAI,CAAC;IAClD,CAAC;IAED,GAAG,CAAC,GAAW;QACb,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAElC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,IAAI,IAAI,EAAE,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACvB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,KAAK,CAAC,aAAa,CAAC;IAC7B,CAAC;IAED,GAAG,CAAC,GAAW,EAAE,aAA4B,EAAE,GAAY;QACzD,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;QAElE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE;YAClB,aAAa;YACb,SAAS;YACT,OAAO,EAAE,aAAa,CAAC,OAAO,IAAI,QAAQ;SAC3C,CAAC,CAAC;IACL,CAAC;IAED,WAAW,CAAC,WAAmB,EAAE,OAAgB;QAC/C,OAAO,OAAO,CAAC,CAAC,CAAC,GAAG,WAAW,IAAI,OAAO,EAAE,CAAC,CAAC,CAAC,GAAG,WAAW,SAAS,CAAC;IACzE,CAAC;IAED,KAAK;QACH,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IACzB,CAAC;IAED,wBAAwB;IACxB,OAAO;QACL,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,OAAO,GAAG,CAAC,CAAC;QAEhB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YAChD,IAAI,GAAG,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;gBAC1B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACvB,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;CACF"}
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
import type { CrawlOptions } from './doc-crawler.js';
|
|
2
|
-
export interface CheckpointData {
|
|
3
|
-
config: CrawlOptions;
|
|
4
|
-
visitedUrls: string[];
|
|
5
|
-
pendingUrls: Array<{
|
|
6
|
-
url: string;
|
|
7
|
-
depth: number;
|
|
8
|
-
}>;
|
|
9
|
-
pagesCrawled: number;
|
|
10
|
-
lastUpdated: string;
|
|
11
|
-
baseUrl: string;
|
|
12
|
-
}
|
|
13
|
-
/**
|
|
14
|
-
* Manager for crawl checkpoint/resume functionality
|
|
15
|
-
* Based on Skill_Seekers implementation
|
|
16
|
-
*/
|
|
17
|
-
export declare class CheckpointManager {
|
|
18
|
-
private checkpointFile;
|
|
19
|
-
constructor(checkpointFile: string);
|
|
20
|
-
/**
|
|
21
|
-
* Save checkpoint data to file
|
|
22
|
-
*/
|
|
23
|
-
saveCheckpoint(data: CheckpointData): Promise<void>;
|
|
24
|
-
/**
|
|
25
|
-
* Load checkpoint data from file
|
|
26
|
-
* Returns null if file doesn't exist or is invalid
|
|
27
|
-
*/
|
|
28
|
-
loadCheckpoint(): Promise<CheckpointData | null>;
|
|
29
|
-
/**
|
|
30
|
-
* Clear checkpoint file
|
|
31
|
-
*/
|
|
32
|
-
clearCheckpoint(): Promise<void>;
|
|
33
|
-
/**
|
|
34
|
-
* Check if checkpoint exists
|
|
35
|
-
*/
|
|
36
|
-
exists(): Promise<boolean>;
|
|
37
|
-
}
|
|
38
|
-
//# sourceMappingURL=checkpoint-manager.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"checkpoint-manager.d.ts","sourceRoot":"","sources":["../../src/documentation/checkpoint-manager.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAErD,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,YAAY,CAAC;IACrB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,WAAW,EAAE,KAAK,CAAC;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACnD,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED;;;GAGG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,cAAc,CAAS;gBAEnB,cAAc,EAAE,MAAM;IAIlC;;OAEG;IACG,cAAc,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBzD;;;OAGG;IACG,cAAc,IAAI,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAiCtD;;OAEG;IACG,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC;IAkBtC;;OAEG;IACG,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC;CAQjC"}
|
|
@@ -1,101 +0,0 @@
|
|
|
1
|
-
import { writeFile, readFile, unlink, access } from 'fs/promises';
|
|
2
|
-
import { logger } from '../utils/logger.js';
|
|
3
|
-
/**
|
|
4
|
-
* Manager for crawl checkpoint/resume functionality
|
|
5
|
-
* Based on Skill_Seekers implementation
|
|
6
|
-
*/
|
|
7
|
-
export class CheckpointManager {
|
|
8
|
-
checkpointFile;
|
|
9
|
-
constructor(checkpointFile) {
|
|
10
|
-
this.checkpointFile = checkpointFile;
|
|
11
|
-
}
|
|
12
|
-
/**
|
|
13
|
-
* Save checkpoint data to file
|
|
14
|
-
*/
|
|
15
|
-
async saveCheckpoint(data) {
|
|
16
|
-
try {
|
|
17
|
-
const json = JSON.stringify(data, null, 2);
|
|
18
|
-
await writeFile(this.checkpointFile, json, 'utf-8');
|
|
19
|
-
logger.info('Checkpoint saved', {
|
|
20
|
-
file: this.checkpointFile,
|
|
21
|
-
pagesCrawled: data.pagesCrawled,
|
|
22
|
-
pendingUrls: data.pendingUrls.length,
|
|
23
|
-
visitedUrls: data.visitedUrls.length,
|
|
24
|
-
});
|
|
25
|
-
}
|
|
26
|
-
catch (error) {
|
|
27
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
28
|
-
logger.error('Failed to save checkpoint', {
|
|
29
|
-
file: this.checkpointFile,
|
|
30
|
-
error: errorMessage,
|
|
31
|
-
});
|
|
32
|
-
throw new Error(`Failed to save checkpoint: ${errorMessage}`);
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
/**
|
|
36
|
-
* Load checkpoint data from file
|
|
37
|
-
* Returns null if file doesn't exist or is invalid
|
|
38
|
-
*/
|
|
39
|
-
async loadCheckpoint() {
|
|
40
|
-
try {
|
|
41
|
-
// Check if file exists
|
|
42
|
-
await access(this.checkpointFile);
|
|
43
|
-
// Read and parse checkpoint
|
|
44
|
-
const content = await readFile(this.checkpointFile, 'utf-8');
|
|
45
|
-
const data = JSON.parse(content);
|
|
46
|
-
logger.info('Checkpoint loaded', {
|
|
47
|
-
file: this.checkpointFile,
|
|
48
|
-
pagesCrawled: data.pagesCrawled,
|
|
49
|
-
pendingUrls: data.pendingUrls.length,
|
|
50
|
-
visitedUrls: data.visitedUrls.length,
|
|
51
|
-
lastUpdated: data.lastUpdated,
|
|
52
|
-
});
|
|
53
|
-
return data;
|
|
54
|
-
}
|
|
55
|
-
catch (error) {
|
|
56
|
-
if (error.code === 'ENOENT') {
|
|
57
|
-
logger.debug('No checkpoint file found', { file: this.checkpointFile });
|
|
58
|
-
return null;
|
|
59
|
-
}
|
|
60
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
61
|
-
logger.warn('Failed to load checkpoint', {
|
|
62
|
-
file: this.checkpointFile,
|
|
63
|
-
error: errorMessage,
|
|
64
|
-
});
|
|
65
|
-
return null;
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
/**
|
|
69
|
-
* Clear checkpoint file
|
|
70
|
-
*/
|
|
71
|
-
async clearCheckpoint() {
|
|
72
|
-
try {
|
|
73
|
-
await unlink(this.checkpointFile);
|
|
74
|
-
logger.info('Checkpoint cleared', { file: this.checkpointFile });
|
|
75
|
-
}
|
|
76
|
-
catch (error) {
|
|
77
|
-
if (error.code === 'ENOENT') {
|
|
78
|
-
// File doesn't exist, that's fine
|
|
79
|
-
return;
|
|
80
|
-
}
|
|
81
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
82
|
-
logger.warn('Failed to clear checkpoint', {
|
|
83
|
-
file: this.checkpointFile,
|
|
84
|
-
error: errorMessage,
|
|
85
|
-
});
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
/**
|
|
89
|
-
* Check if checkpoint exists
|
|
90
|
-
*/
|
|
91
|
-
async exists() {
|
|
92
|
-
try {
|
|
93
|
-
await access(this.checkpointFile);
|
|
94
|
-
return true;
|
|
95
|
-
}
|
|
96
|
-
catch {
|
|
97
|
-
return false;
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
//# sourceMappingURL=checkpoint-manager.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"checkpoint-manager.js","sourceRoot":"","sources":["../../src/documentation/checkpoint-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAClE,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAY5C;;;GAGG;AACH,MAAM,OAAO,iBAAiB;IACpB,cAAc,CAAS;IAE/B,YAAY,cAAsB;QAChC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;IACvC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,IAAoB;QACvC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAC3C,MAAM,SAAS,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;YAEpD,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE;gBAC9B,IAAI,EAAE,IAAI,CAAC,cAAc;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM;gBACpC,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM;aACrC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5E,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE;gBACxC,IAAI,EAAE,IAAI,CAAC,cAAc;gBACzB,KAAK,EAAE,YAAY;aACpB,CAAC,CAAC;YACH,MAAM,IAAI,KAAK,CAAC,8BAA8B,YAAY,EAAE,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc;QAClB,IAAI,CAAC;YACH,uBAAuB;YACvB,MAAM,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAElC,4BAA4B;YAC5B,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC;YAC7D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAmB,CAAC;YAEnD,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE;gBAC/B,IAAI,EAAE,IAAI,CAAC,cAAc;gBACzB,YAAY,EAAE,IAAI,CAAC,YAAY;gBAC/B,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM;gBACpC,WAAW,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM;gBACpC,WAAW,EAAE,IAAI,CAAC,WAAW;aAC9B,CAAC,CAAC;YAEH,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACvD,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC;gBACxE,OAAO,IAAI,CAAC;YACd,CAAC;YAED,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5E,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE;gBACvC,IAAI,EAAE,IAAI,CAAC,cAAc;gBACzB,KAAK,EAAE,YAAY;aACpB,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe;QACnB,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAClC,MAAM,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC;QACnE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACvD,kCAAkC;gBAClC,OAAO;YACT,CAAC;YAED,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5E,MAAM,CAAC,IAAI,CAAC,4BAA4B,EAAE;gBACxC,IAAI,EAAE,IAAI,CAAC,cAAc;gBACzB,KAAK,EAAE,YAAY;aACpB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM;QACV,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAClC,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;CACF"}
|