@tikoci/rosetta 0.5.2 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -1
- package/package.json +1 -1
- package/src/browse.ts +90 -0
- package/src/db.ts +130 -7
- package/src/extract-commands.ts +55 -10
- package/src/extract-dude.ts +409 -0
- package/src/mcp-http.test.ts +7 -5
- package/src/mcp.ts +287 -2
- package/src/paths.ts +1 -1
- package/src/query.test.ts +124 -6
- package/src/query.ts +93 -2
- package/src/release.test.ts +32 -0
- package/src/setup.ts +2 -2
|
@@ -0,0 +1,409 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
/**
|
|
3
|
+
* extract-dude.ts — Extract "The Dude" documentation from Wayback Machine.
|
|
4
|
+
*
|
|
5
|
+
* One-time extractor: fetches archived wiki pages from web.archive.org,
|
|
6
|
+
* parses HTML with linkedom, downloads screenshots to dude/images/,
|
|
7
|
+
* caches raw HTML to dude/pages/, and populates dude_pages + dude_images tables.
|
|
8
|
+
*
|
|
9
|
+
* Usage:
|
|
10
|
+
* bun run src/extract-dude.ts # Fetch from Wayback Machine + download images
|
|
11
|
+
* bun run src/extract-dude.ts --from-cache # Re-extract from cached dude/pages/ HTML
|
|
12
|
+
* bun run src/extract-dude.ts --from-cache --skip-images # CI path: no image download
|
|
13
|
+
* bun run src/extract-dude.ts --force # Force re-download even if cached
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
17
|
+
import { join } from "node:path";
|
|
18
|
+
import { parseHTML } from "linkedom";
|
|
19
|
+
import { db, initDb } from "./db.ts";
|
|
20
|
+
|
|
21
|
+
// ── Configuration ──
|
|
22
|
+
|
|
23
|
+
const PROJECT_ROOT = join(import.meta.dirname, "..");
|
|
24
|
+
const PAGES_DIR = join(PROJECT_ROOT, "dude", "pages");
|
|
25
|
+
const IMAGES_DIR = join(PROJECT_ROOT, "dude", "images");
|
|
26
|
+
const FETCH_DELAY_MS = 500;
|
|
27
|
+
|
|
28
|
+
const FROM_CACHE = process.argv.includes("--from-cache");
|
|
29
|
+
const FORCE = process.argv.includes("--force");
|
|
30
|
+
const SKIP_IMAGES = process.argv.includes("--skip-images");
|
|
31
|
+
|
|
32
|
+
/** Page definition: wiki path suffix → metadata */
|
|
33
|
+
interface PageDef {
|
|
34
|
+
/** Wiki path after /wiki/ (e.g. "Manual:The_Dude_v6/Probes") */
|
|
35
|
+
wikiPath: string;
|
|
36
|
+
/** Short slug for DB (e.g. "Probes", "Device_discovery") */
|
|
37
|
+
slug: string;
|
|
38
|
+
/** Human-readable breadcrumb path */
|
|
39
|
+
path: string;
|
|
40
|
+
/** Version tag: "v6" or "v3" */
|
|
41
|
+
version: "v6" | "v3";
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// ── Page list (from CDX API enumeration) ──
|
|
45
|
+
|
|
46
|
+
const V6_PAGES: PageDef[] = [
|
|
47
|
+
{ wikiPath: "Manual:The_Dude", slug: "The_Dude", path: "The Dude", version: "v6" },
|
|
48
|
+
{ wikiPath: "Manual:The_Dude_v6/Installation", slug: "Installation", path: "The Dude > v6 > Installation", version: "v6" },
|
|
49
|
+
{ wikiPath: "Manual:The_Dude_v6/First_use", slug: "First_use", path: "The Dude > v6 > First Use", version: "v6" },
|
|
50
|
+
{ wikiPath: "Manual:The_Dude_v6/Interface", slug: "Interface", path: "The Dude > v6 > Interface", version: "v6" },
|
|
51
|
+
{ wikiPath: "Manual:The_Dude_v6/Device_settings", slug: "Device_settings", path: "The Dude > v6 > Device Settings", version: "v6" },
|
|
52
|
+
{ wikiPath: "Manual:The_Dude_v6/Device_discovery", slug: "Device_discovery", path: "The Dude > v6 > Device Discovery", version: "v6" },
|
|
53
|
+
{ wikiPath: "Manual:The_Dude_v6/Device_map", slug: "Device_map", path: "The Dude > v6 > Device Map", version: "v6" },
|
|
54
|
+
{ wikiPath: "Manual:The_Dude_v6/Device_list", slug: "Device_list", path: "The Dude > v6 > Device List", version: "v6" },
|
|
55
|
+
{ wikiPath: "Manual:The_Dude_v6/Networks", slug: "Networks", path: "The Dude > v6 > Networks", version: "v6" },
|
|
56
|
+
{ wikiPath: "Manual:The_Dude_v6/Links", slug: "Links", path: "The Dude > v6 > Links", version: "v6" },
|
|
57
|
+
{ wikiPath: "Manual:The_Dude_v6/Agents", slug: "Agents", path: "The Dude > v6 > Agents", version: "v6" },
|
|
58
|
+
{ wikiPath: "Manual:The_Dude_v6/Charts", slug: "Charts", path: "The Dude > v6 > Charts", version: "v6" },
|
|
59
|
+
{ wikiPath: "Manual:The_Dude_v6/Functions", slug: "Functions", path: "The Dude > v6 > Functions", version: "v6" },
|
|
60
|
+
{ wikiPath: "Manual:The_Dude_v6/Probes", slug: "Probes", path: "The Dude > v6 > Probes", version: "v6" },
|
|
61
|
+
{ wikiPath: "Manual:The_Dude_v6/MIB_Nodes", slug: "MIB_Nodes", path: "The Dude > v6 > MIB Nodes", version: "v6" },
|
|
62
|
+
{ wikiPath: "Manual:The_Dude_v6/Notifications", slug: "Notifications", path: "The Dude > v6 > Notifications", version: "v6" },
|
|
63
|
+
{ wikiPath: "Manual:The_Dude_v6/Syslog", slug: "Syslog", path: "The Dude > v6 > Syslog", version: "v6" },
|
|
64
|
+
{ wikiPath: "Manual:The_Dude_v6/Services", slug: "Services", path: "The Dude > v6 > Services", version: "v6" },
|
|
65
|
+
{ wikiPath: "Manual:The_Dude_v6/Logs", slug: "Logs", path: "The Dude > v6 > Logs", version: "v6" },
|
|
66
|
+
{ wikiPath: "Manual:The_Dude_v6/History", slug: "History", path: "The Dude > v6 > History", version: "v6" },
|
|
67
|
+
{ wikiPath: "Manual:The_Dude_v6/Tools", slug: "Tools", path: "The Dude > v6 > Tools", version: "v6" },
|
|
68
|
+
{ wikiPath: "Manual:The_Dude_v6/Panels", slug: "Panels", path: "The Dude > v6 > Panels", version: "v6" },
|
|
69
|
+
{ wikiPath: "Manual:The_Dude_v6/Server_settings", slug: "Server_settings", path: "The Dude > v6 > Server Settings", version: "v6" },
|
|
70
|
+
{ wikiPath: "Manual:The_Dude_v6/Files", slug: "Files", path: "The Dude > v6 > Files", version: "v6" },
|
|
71
|
+
{ wikiPath: "Manual:The_Dude_v6/Exporting", slug: "Exporting", path: "The Dude > v6 > Search and Export", version: "v6" },
|
|
72
|
+
{ wikiPath: "Manual:The_Dude_v6/Address_lists", slug: "Address_lists", path: "The Dude > v6 > Address Lists", version: "v6" },
|
|
73
|
+
{ wikiPath: "Manual:The_Dude_v6/Admins", slug: "Admins", path: "The Dude > v6 > Admins", version: "v6" },
|
|
74
|
+
{ wikiPath: "Manual:The_Dude_v6/DB_import_export", slug: "DB_import_export", path: "The Dude > v6 > Database Import and Export", version: "v6" },
|
|
75
|
+
{ wikiPath: "Manual:The_Dude_v6/Change_DB_path", slug: "Change_DB_path", path: "The Dude > v6 > Change DB Path", version: "v6" },
|
|
76
|
+
{ wikiPath: "Manual:The_Dude_v6/db_vacuum", slug: "db_vacuum", path: "The Dude > v6 > SQLite3 DB Vacuum", version: "v6" },
|
|
77
|
+
{ wikiPath: "Manual:The_Dude_v6/MigrationToNewDude", slug: "MigrationToNewDude", path: "The Dude > v6 > Migration from v3/v4 to v6", version: "v6" },
|
|
78
|
+
{ wikiPath: "Manual:The_Dude_v6/Malformed_db_repair", slug: "Malformed_db_repair", path: "The Dude > v6 > Malformed DB Repair", version: "v6" },
|
|
79
|
+
{ wikiPath: "Manual:The_Dude_v6/The_dude_server_on_VM_CHR", slug: "Server_on_CHR", path: "The Dude > v6 > Server on CHR", version: "v6" },
|
|
80
|
+
{ wikiPath: "Manual:The_Dude_v6/The_Dude_server_on_hEX_RB750Gr3", slug: "Server_on_hEX", path: "The Dude > v6 > Server on hEX RB750Gr3", version: "v6" },
|
|
81
|
+
{ wikiPath: "Manual:The_Dude_v6/Dude_Telegram_Example", slug: "Telegram_Example", path: "The Dude > v6 > Telegram Notification Example", version: "v6" },
|
|
82
|
+
{ wikiPath: "Manual:The_Dude_v6/client_shortcut_arguments", slug: "Client_shortcuts", path: "The Dude > v6 > Client Shortcut Arguments", version: "v6" },
|
|
83
|
+
{ wikiPath: "Manual:The_Dude_v6/dude_v6.xx_changelog", slug: "Changelog_v6", path: "The Dude > v6 > Version Changelog", version: "v6" },
|
|
84
|
+
];
|
|
85
|
+
|
|
86
|
+
const V3_PAGES: PageDef[] = [
|
|
87
|
+
{ wikiPath: "Manual:The_Dude4", slug: "v3_Overview", path: "The Dude > v3/v4 > Overview", version: "v3" },
|
|
88
|
+
{ wikiPath: "Manual:The_Dude/Installation", slug: "v3_Installation", path: "The Dude > v3/v4 > Installation", version: "v3" },
|
|
89
|
+
{ wikiPath: "Manual:The_Dude/First_use", slug: "v3_First_use", path: "The Dude > v3/v4 > First Use", version: "v3" },
|
|
90
|
+
{ wikiPath: "Manual:The_Dude/Interface", slug: "v3_Interface", path: "The Dude > v3/v4 > Interface", version: "v3" },
|
|
91
|
+
{ wikiPath: "Manual:The_Dude/Device_settings", slug: "v3_Device_settings", path: "The Dude > v3/v4 > Device Settings", version: "v3" },
|
|
92
|
+
{ wikiPath: "Manual:The_Dude/Device_discovery", slug: "v3_Device_discovery", path: "The Dude > v3/v4 > Device Discovery", version: "v3" },
|
|
93
|
+
{ wikiPath: "Manual:The_Dude/Device_map", slug: "v3_Device_map", path: "The Dude > v3/v4 > Device Map", version: "v3" },
|
|
94
|
+
{ wikiPath: "Manual:The_Dude/Device_list", slug: "v3_Device_list", path: "The Dude > v3/v4 > Device List", version: "v3" },
|
|
95
|
+
{ wikiPath: "Manual:The_Dude/Networks", slug: "v3_Networks", path: "The Dude > v3/v4 > Networks", version: "v3" },
|
|
96
|
+
{ wikiPath: "Manual:The_Dude/Links", slug: "v3_Links", path: "The Dude > v3/v4 > Links", version: "v3" },
|
|
97
|
+
{ wikiPath: "Manual:The_Dude/Agents", slug: "v3_Agents", path: "The Dude > v3/v4 > Agents", version: "v3" },
|
|
98
|
+
{ wikiPath: "Manual:The_Dude/Charts", slug: "v3_Charts", path: "The Dude > v3/v4 > Charts", version: "v3" },
|
|
99
|
+
{ wikiPath: "Manual:The_Dude/Functions", slug: "v3_Functions", path: "The Dude > v3/v4 > Functions", version: "v3" },
|
|
100
|
+
{ wikiPath: "Manual:The_Dude/Probes", slug: "v3_Probes", path: "The Dude > v3/v4 > Probes", version: "v3" },
|
|
101
|
+
{ wikiPath: "Manual:The_Dude/MIB_Nodes", slug: "v3_MIB_Nodes", path: "The Dude > v3/v4 > MIB Nodes", version: "v3" },
|
|
102
|
+
{ wikiPath: "Manual:The_Dude/Notifications", slug: "v3_Notifications", path: "The Dude > v3/v4 > Notifications", version: "v3" },
|
|
103
|
+
{ wikiPath: "Manual:The_Dude/Syslog", slug: "v3_Syslog", path: "The Dude > v3/v4 > Syslog", version: "v3" },
|
|
104
|
+
{ wikiPath: "Manual:The_Dude/Services", slug: "v3_Services", path: "The Dude > v3/v4 > Services", version: "v3" },
|
|
105
|
+
{ wikiPath: "Manual:The_Dude/Logs", slug: "v3_Logs", path: "The Dude > v3/v4 > Logs", version: "v3" },
|
|
106
|
+
{ wikiPath: "Manual:The_Dude/History", slug: "v3_History", path: "The Dude > v3/v4 > History", version: "v3" },
|
|
107
|
+
{ wikiPath: "Manual:The_Dude/Tools", slug: "v3_Tools", path: "The Dude > v3/v4 > Tools", version: "v3" },
|
|
108
|
+
{ wikiPath: "Manual:The_Dude/Panels", slug: "v3_Panels", path: "The Dude > v3/v4 > Panels", version: "v3" },
|
|
109
|
+
{ wikiPath: "Manual:The_Dude/Server_settings", slug: "v3_Server_settings", path: "The Dude > v3/v4 > Server Settings", version: "v3" },
|
|
110
|
+
{ wikiPath: "Manual:The_Dude/Files", slug: "v3_Files", path: "The Dude > v3/v4 > Files", version: "v3" },
|
|
111
|
+
{ wikiPath: "Manual:The_Dude/Exporting", slug: "v3_Exporting", path: "The Dude > v3/v4 > Exporting", version: "v3" },
|
|
112
|
+
{ wikiPath: "Manual:The_Dude/Address_lists", slug: "v3_Address_lists", path: "The Dude > v3/v4 > Address Lists", version: "v3" },
|
|
113
|
+
{ wikiPath: "Manual:The_Dude/Admins", slug: "v3_Admins", path: "The Dude > v3/v4 > Admins", version: "v3" },
|
|
114
|
+
{ wikiPath: "Manual:The_Dude/Web_interface", slug: "v3_Web_interface", path: "The Dude > v3/v4 > Web Interface", version: "v3" },
|
|
115
|
+
{ wikiPath: "Manual:The_Dude/Changelog", slug: "v3_Changelog", path: "The Dude > v3/v4 > Changelog", version: "v3" },
|
|
116
|
+
];
|
|
117
|
+
|
|
118
|
+
const ALL_PAGES: PageDef[] = [...V6_PAGES, ...V3_PAGES];
|
|
119
|
+
|
|
120
|
+
// ── Wayback Machine helpers ──
|
|
121
|
+
|
|
122
|
+
const WIKI_BASE = "https://wiki.mikrotik.com/wiki/";
|
|
123
|
+
|
|
124
|
+
/** Build a Wayback Machine URL. Uses wildcard timestamp to get latest snapshot. */
|
|
125
|
+
function waybackUrl(wikiPath: string): string {
|
|
126
|
+
return `https://web.archive.org/web/2024/${WIKI_BASE}${encodeURIComponent(wikiPath).replace(/%2F/g, "/").replace(/%3A/g, ":")}`;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function delay(ms: number): Promise<void> {
|
|
130
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** Fetch with retry for Wayback Machine rate limits. */
|
|
134
|
+
async function fetchWithRetry(url: string, retries = 3): Promise<Response> {
|
|
135
|
+
for (let attempt = 0; attempt < retries; attempt++) {
|
|
136
|
+
const response = await fetch(url, {
|
|
137
|
+
headers: { "User-Agent": "rosetta-dude-extractor/1.0 (MikroTik documentation tool)" },
|
|
138
|
+
});
|
|
139
|
+
if (response.ok) return response;
|
|
140
|
+
if (response.status === 429 || response.status === 503) {
|
|
141
|
+
const waitMs = (attempt + 1) * 2000;
|
|
142
|
+
console.log(` Rate limited (${response.status}), waiting ${waitMs}ms...`);
|
|
143
|
+
await delay(waitMs);
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
if (response.status === 404) {
|
|
147
|
+
throw new Error(`404 Not Found: ${url}`);
|
|
148
|
+
}
|
|
149
|
+
throw new Error(`HTTP ${response.status}: ${url}`);
|
|
150
|
+
}
|
|
151
|
+
throw new Error(`Failed after ${retries} retries: ${url}`);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// ── HTML parsing ──
|
|
155
|
+
|
|
156
|
+
interface ParsedPage {
|
|
157
|
+
title: string;
|
|
158
|
+
text: string;
|
|
159
|
+
code: string;
|
|
160
|
+
lastEdited: string | null;
|
|
161
|
+
images: Array<{ filename: string; altText: string | null; originalUrl: string; waybackImageUrl: string }>;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
function parsePage(html: string, _waybackBaseUrl: string): ParsedPage {
|
|
165
|
+
const { document } = parseHTML(html);
|
|
166
|
+
|
|
167
|
+
// Title from first h1 or page heading
|
|
168
|
+
const h1 = document.querySelector("#firstHeading, h1.firstHeading, h1");
|
|
169
|
+
const title = h1?.textContent?.trim()?.replace(/^Manual:The Dude v6\//, "")
|
|
170
|
+
.replace(/^Manual:The Dude\//, "")
|
|
171
|
+
.replace(/^Manual:The Dude4?$/, "The Dude")
|
|
172
|
+
.replace(/_/g, " ") ?? "Unknown";
|
|
173
|
+
|
|
174
|
+
// Main content area
|
|
175
|
+
const content = document.querySelector("#mw-content-text, .mw-parser-output, #bodyContent");
|
|
176
|
+
if (!content) return { title, text: "", code: "", lastEdited: null, images: [] };
|
|
177
|
+
|
|
178
|
+
// Remove navigation, TOC, edit links, Wayback Machine toolbar
|
|
179
|
+
for (const sel of ["#toc", ".toc", ".mw-editsection", ".navbox", "#catlinks",
|
|
180
|
+
".printfooter", "#wm-ipp-base", "#wm-ipp", "#donato", ".wb-autocomplete-suggestions",
|
|
181
|
+
"#mw-navigation", "#footer", ".noprint"]) {
|
|
182
|
+
for (const el of content.querySelectorAll(sel)) el.remove();
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
// Extract code blocks
|
|
186
|
+
const codeBlocks: string[] = [];
|
|
187
|
+
for (const pre of content.querySelectorAll("pre, code.ros")) {
|
|
188
|
+
const text = pre.textContent?.trim();
|
|
189
|
+
if (text && text.length > 10) codeBlocks.push(text);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// Extract images
|
|
193
|
+
const images: ParsedPage["images"] = [];
|
|
194
|
+
const seenFilenames = new Set<string>();
|
|
195
|
+
for (const img of content.querySelectorAll("img")) {
|
|
196
|
+
const src = img.getAttribute("src") ?? "";
|
|
197
|
+
const alt = img.getAttribute("alt") ?? null;
|
|
198
|
+
|
|
199
|
+
// Skip tiny icons, navigation icons, and Wayback Machine UI images
|
|
200
|
+
const width = Number.parseInt(img.getAttribute("width") ?? "0", 10);
|
|
201
|
+
if (width > 0 && width < 30) continue;
|
|
202
|
+
if (src.includes("web.archive.org/static/") || src.includes("/_static/")) continue;
|
|
203
|
+
|
|
204
|
+
// Extract filename from MediaWiki image URLs
|
|
205
|
+
// Patterns: /images/X/XX/Filename.JPG or /wiki/File:Filename.JPG
|
|
206
|
+
let filename: string | null = null;
|
|
207
|
+
const srcMatch = src.match(/\/images\/[0-9a-f]\/[0-9a-f]{2}\/([^/?]+)/i)
|
|
208
|
+
?? src.match(/\/([^/]+\.(?:jpg|jpeg|png|gif|svg))(?:\?|$)/i);
|
|
209
|
+
if (srcMatch) {
|
|
210
|
+
filename = decodeURIComponent(srcMatch[1]);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// Also check parent <a> links for File: references
|
|
214
|
+
if (!filename) {
|
|
215
|
+
const parentA = img.closest("a");
|
|
216
|
+
const href = parentA?.getAttribute("href") ?? "";
|
|
217
|
+
const fileMatch = href.match(/File:([^&"]+)/);
|
|
218
|
+
if (fileMatch) filename = decodeURIComponent(fileMatch[1]);
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
if (!filename) continue;
|
|
222
|
+
if (seenFilenames.has(filename)) continue;
|
|
223
|
+
// Skip common wiki icons
|
|
224
|
+
if (/^Icon-\w+\.png$/i.test(filename)) continue;
|
|
225
|
+
if (/^Version\.png$/i.test(filename)) continue;
|
|
226
|
+
seenFilenames.add(filename);
|
|
227
|
+
|
|
228
|
+
// Build a usable image URL from the Wayback Machine
|
|
229
|
+
let waybackImageUrl = src;
|
|
230
|
+
if (src.startsWith("/")) {
|
|
231
|
+
waybackImageUrl = `https://web.archive.org${src}`;
|
|
232
|
+
}
|
|
233
|
+
// Ensure we use the raw image URL (id_ prefix bypasses Wayback rewriting)
|
|
234
|
+
waybackImageUrl = waybackImageUrl.replace(
|
|
235
|
+
/\/web\/\d+\//,
|
|
236
|
+
"/web/2024id_/",
|
|
237
|
+
);
|
|
238
|
+
|
|
239
|
+
const originalUrl = `${WIKI_BASE}File:${encodeURIComponent(filename)}`;
|
|
240
|
+
|
|
241
|
+
images.push({ filename, altText: alt, originalUrl, waybackImageUrl });
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// Extract body text
|
|
245
|
+
const textContent = content.textContent ?? "";
|
|
246
|
+
// Clean up whitespace: collapse multiple blank lines, trim
|
|
247
|
+
const text = textContent.replace(/\n{3,}/g, "\n\n").trim();
|
|
248
|
+
|
|
249
|
+
// Last edited date from footer
|
|
250
|
+
const lastEditedEl = document.querySelector("#footer-info-lastmod, .lastmod");
|
|
251
|
+
const lastEdited = lastEditedEl?.textContent?.replace(/.*last edited on\s*/i, "").trim() ?? null;
|
|
252
|
+
|
|
253
|
+
return { title, text, code: codeBlocks.join("\n\n"), lastEdited, images };
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
// ── Image download ──
|
|
257
|
+
|
|
258
|
+
async function downloadImage(url: string, dest: string): Promise<boolean> {
|
|
259
|
+
if (existsSync(dest) && !FORCE) return true;
|
|
260
|
+
try {
|
|
261
|
+
const response = await fetchWithRetry(url);
|
|
262
|
+
const buffer = await response.arrayBuffer();
|
|
263
|
+
if (buffer.byteLength < 100) return false; // too small, probably an error page
|
|
264
|
+
writeFileSync(dest, Buffer.from(buffer));
|
|
265
|
+
return true;
|
|
266
|
+
} catch (e) {
|
|
267
|
+
console.log(` Warning: failed to download image: ${e}`);
|
|
268
|
+
return false;
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
// ── Main extraction ──
|
|
273
|
+
|
|
274
|
+
async function main() {
|
|
275
|
+
initDb();
|
|
276
|
+
|
|
277
|
+
mkdirSync(PAGES_DIR, { recursive: true });
|
|
278
|
+
mkdirSync(IMAGES_DIR, { recursive: true });
|
|
279
|
+
|
|
280
|
+
// Idempotent: clear existing data
|
|
281
|
+
db.run("DELETE FROM dude_images");
|
|
282
|
+
db.run("DELETE FROM dude_pages");
|
|
283
|
+
|
|
284
|
+
const insertPage = db.prepare(`
|
|
285
|
+
INSERT INTO dude_pages (slug, title, path, version, url, wayback_url, text, code, last_edited, word_count)
|
|
286
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
287
|
+
`);
|
|
288
|
+
const insertImage = db.prepare(`
|
|
289
|
+
INSERT INTO dude_images (page_id, filename, alt_text, caption, local_path, original_url, wayback_url, sort_order)
|
|
290
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
291
|
+
`);
|
|
292
|
+
|
|
293
|
+
let pageCount = 0;
|
|
294
|
+
let imageCount = 0;
|
|
295
|
+
let errorCount = 0;
|
|
296
|
+
|
|
297
|
+
for (const pageDef of ALL_PAGES) {
|
|
298
|
+
const cacheFile = join(PAGES_DIR, `${pageDef.slug}.html`);
|
|
299
|
+
const wbUrl = waybackUrl(pageDef.wikiPath);
|
|
300
|
+
const originalUrl = `${WIKI_BASE}${pageDef.wikiPath}`;
|
|
301
|
+
|
|
302
|
+
let html: string;
|
|
303
|
+
|
|
304
|
+
if (FROM_CACHE || (existsSync(cacheFile) && !FORCE)) {
|
|
305
|
+
// Read from cache
|
|
306
|
+
if (!existsSync(cacheFile)) {
|
|
307
|
+
console.log(` SKIP (no cache): ${pageDef.slug}`);
|
|
308
|
+
continue;
|
|
309
|
+
}
|
|
310
|
+
html = readFileSync(cacheFile, "utf-8");
|
|
311
|
+
console.log(` [cache] ${pageDef.slug}`);
|
|
312
|
+
} else {
|
|
313
|
+
// Fetch from Wayback Machine
|
|
314
|
+
console.log(` [fetch] ${pageDef.slug} ...`);
|
|
315
|
+
try {
|
|
316
|
+
const response = await fetchWithRetry(wbUrl);
|
|
317
|
+
html = await response.text();
|
|
318
|
+
// Cache the raw HTML
|
|
319
|
+
writeFileSync(cacheFile, html);
|
|
320
|
+
await delay(FETCH_DELAY_MS);
|
|
321
|
+
} catch (e) {
|
|
322
|
+
console.log(` ERROR: ${pageDef.slug}: ${e}`);
|
|
323
|
+
errorCount++;
|
|
324
|
+
continue;
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
// Parse HTML
|
|
329
|
+
const parsed = parsePage(html, wbUrl);
|
|
330
|
+
if (!parsed.text && !parsed.code) {
|
|
331
|
+
console.log(` WARN: empty content for ${pageDef.slug}`);
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
const wordCount = parsed.text.split(/\s+/).filter(Boolean).length;
|
|
335
|
+
|
|
336
|
+
// Insert page
|
|
337
|
+
db.transaction(() => {
|
|
338
|
+
insertPage.run(
|
|
339
|
+
pageDef.slug,
|
|
340
|
+
parsed.title,
|
|
341
|
+
pageDef.path,
|
|
342
|
+
pageDef.version,
|
|
343
|
+
originalUrl,
|
|
344
|
+
wbUrl,
|
|
345
|
+
parsed.text,
|
|
346
|
+
parsed.code || null,
|
|
347
|
+
parsed.lastEdited,
|
|
348
|
+
wordCount,
|
|
349
|
+
);
|
|
350
|
+
|
|
351
|
+
const pageRow = db.prepare("SELECT id FROM dude_pages WHERE slug = ?").get(pageDef.slug) as { id: number };
|
|
352
|
+
const pageId = pageRow.id;
|
|
353
|
+
|
|
354
|
+
// Download and insert images
|
|
355
|
+
for (let i = 0; i < parsed.images.length; i++) {
|
|
356
|
+
const img = parsed.images[i];
|
|
357
|
+
const localPath = `dude/images/${img.filename}`;
|
|
358
|
+
|
|
359
|
+
insertImage.run(
|
|
360
|
+
pageId,
|
|
361
|
+
img.filename,
|
|
362
|
+
img.altText,
|
|
363
|
+
null, // caption — could be inferred from surrounding text
|
|
364
|
+
localPath,
|
|
365
|
+
img.originalUrl,
|
|
366
|
+
img.waybackImageUrl,
|
|
367
|
+
i,
|
|
368
|
+
);
|
|
369
|
+
imageCount++;
|
|
370
|
+
}
|
|
371
|
+
})();
|
|
372
|
+
|
|
373
|
+
pageCount++;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
// Download images outside the DB transaction loop (allows partial success)
|
|
377
|
+
if (SKIP_IMAGES) {
|
|
378
|
+
console.log("\nSkipping image download (--skip-images).");
|
|
379
|
+
} else {
|
|
380
|
+
console.log("\nDownloading images...");
|
|
381
|
+
const allImages = db.prepare("SELECT DISTINCT filename, wayback_url FROM dude_images").all() as Array<{ filename: string; wayback_url: string }>;
|
|
382
|
+
let downloadedCount = 0;
|
|
383
|
+
let skipCount = 0;
|
|
384
|
+
for (const img of allImages) {
|
|
385
|
+
const destPath = join(IMAGES_DIR, img.filename);
|
|
386
|
+
if (existsSync(destPath) && !FORCE) {
|
|
387
|
+
skipCount++;
|
|
388
|
+
continue;
|
|
389
|
+
}
|
|
390
|
+
const ok = await downloadImage(img.wayback_url, destPath);
|
|
391
|
+
if (ok) {
|
|
392
|
+
downloadedCount++;
|
|
393
|
+
console.log(` ✓ ${img.filename}`);
|
|
394
|
+
}
|
|
395
|
+
await delay(FETCH_DELAY_MS);
|
|
396
|
+
}
|
|
397
|
+
console.log(`\nDone: ${pageCount} pages, ${imageCount} image refs, ${downloadedCount} downloaded, ${skipCount} cached, ${errorCount} errors`);
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
// Summary
|
|
401
|
+
const stats = db.prepare("SELECT COUNT(*) AS c FROM dude_pages").get() as { c: number };
|
|
402
|
+
const imgStats = db.prepare("SELECT COUNT(*) AS c FROM dude_images").get() as { c: number };
|
|
403
|
+
console.log(`DB: ${stats.c} dude_pages, ${imgStats.c} dude_images`);
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
main().catch((e) => {
|
|
407
|
+
console.error("Fatal:", e);
|
|
408
|
+
process.exit(1);
|
|
409
|
+
});
|
package/src/mcp-http.test.ts
CHANGED
|
@@ -108,6 +108,8 @@ function createFixtureDb(dbPath: string): void {
|
|
|
108
108
|
1, '/system', 'system', 'dir', NULL, 1, 'fixture command', '7.22'
|
|
109
109
|
);`);
|
|
110
110
|
|
|
111
|
+
// Stamp schema version so mcp.ts doesn't trigger a 50MB auto-download
|
|
112
|
+
fixture.run("PRAGMA user_version = 1;");
|
|
111
113
|
fixture.close();
|
|
112
114
|
}
|
|
113
115
|
|
|
@@ -315,7 +317,7 @@ describe("HTTP transport: session lifecycle", () => {
|
|
|
315
317
|
expect(serverInfo.name).toBe("rosetta");
|
|
316
318
|
});
|
|
317
319
|
|
|
318
|
-
test("tools/list returns all
|
|
320
|
+
test("tools/list returns all 16 tools after initialization", async () => {
|
|
319
321
|
const { sessionId } = await mcpInitialize(server.url);
|
|
320
322
|
|
|
321
323
|
// Send initialized notification first (required by protocol)
|
|
@@ -326,7 +328,7 @@ describe("HTTP transport: session lifecycle", () => {
|
|
|
326
328
|
|
|
327
329
|
const result = (messages[0] as Record<string, unknown>).result as Record<string, unknown>;
|
|
328
330
|
const tools = result.tools as Array<{ name: string }>;
|
|
329
|
-
expect(tools.length).toBe(
|
|
331
|
+
expect(tools.length).toBe(16);
|
|
330
332
|
|
|
331
333
|
const toolNames = tools.map((t) => t.name).sort();
|
|
332
334
|
expect(toolNames).toContain("routeros_search");
|
|
@@ -547,8 +549,8 @@ describe("HTTP transport: multi-session", () => {
|
|
|
547
549
|
const tools1 = ((msgs1[0] as Record<string, unknown>).result as Record<string, unknown>).tools as unknown[];
|
|
548
550
|
const tools2 = ((msgs2[0] as Record<string, unknown>).result as Record<string, unknown>).tools as unknown[];
|
|
549
551
|
|
|
550
|
-
expect(tools1.length).toBe(
|
|
551
|
-
expect(tools2.length).toBe(
|
|
552
|
+
expect(tools1.length).toBe(16);
|
|
553
|
+
expect(tools2.length).toBe(16);
|
|
552
554
|
});
|
|
553
555
|
|
|
554
556
|
test("deleting one session does not affect another", async () => {
|
|
@@ -570,7 +572,7 @@ describe("HTTP transport: multi-session", () => {
|
|
|
570
572
|
// Client2 still works
|
|
571
573
|
const msgs = await mcpRequest(server.url, client2.sessionId, "tools/list", 2);
|
|
572
574
|
const tools = ((msgs[0] as Record<string, unknown>).result as Record<string, unknown>).tools as unknown[];
|
|
573
|
-
expect(tools.length).toBe(
|
|
575
|
+
expect(tools.length).toBe(16);
|
|
574
576
|
|
|
575
577
|
// Client1 is gone
|
|
576
578
|
const resp = await fetch(server.url, {
|