@tikoci/rosetta 0.5.2 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,409 @@
1
+ #!/usr/bin/env bun
2
+ /**
3
+ * extract-dude.ts — Extract "The Dude" documentation from Wayback Machine.
4
+ *
5
+ * One-time extractor: fetches archived wiki pages from web.archive.org,
6
+ * parses HTML with linkedom, downloads screenshots to dude/images/,
7
+ * caches raw HTML to dude/pages/, and populates dude_pages + dude_images tables.
8
+ *
9
+ * Usage:
10
+ * bun run src/extract-dude.ts # Fetch from Wayback Machine + download images
11
+ * bun run src/extract-dude.ts --from-cache # Re-extract from cached dude/pages/ HTML
12
+ * bun run src/extract-dude.ts --from-cache --skip-images # CI path: no image download
13
+ * bun run src/extract-dude.ts --force # Force re-download even if cached
14
+ */
15
+
16
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
17
+ import { join } from "node:path";
18
+ import { parseHTML } from "linkedom";
19
+ import { db, initDb } from "./db.ts";
20
+
21
+ // ── Configuration ──
22
+
23
+ const PROJECT_ROOT = join(import.meta.dirname, "..");
24
+ const PAGES_DIR = join(PROJECT_ROOT, "dude", "pages");
25
+ const IMAGES_DIR = join(PROJECT_ROOT, "dude", "images");
26
+ const FETCH_DELAY_MS = 500;
27
+
28
+ const FROM_CACHE = process.argv.includes("--from-cache");
29
+ const FORCE = process.argv.includes("--force");
30
+ const SKIP_IMAGES = process.argv.includes("--skip-images");
31
+
32
+ /** Page definition: wiki path suffix → metadata */
33
+ interface PageDef {
34
+ /** Wiki path after /wiki/ (e.g. "Manual:The_Dude_v6/Probes") */
35
+ wikiPath: string;
36
+ /** Short slug for DB (e.g. "Probes", "Device_discovery") */
37
+ slug: string;
38
+ /** Human-readable breadcrumb path */
39
+ path: string;
40
+ /** Version tag: "v6" or "v3" */
41
+ version: "v6" | "v3";
42
+ }
43
+
44
+ // ── Page list (from CDX API enumeration) ──
45
+
46
+ const V6_PAGES: PageDef[] = [
47
+ { wikiPath: "Manual:The_Dude", slug: "The_Dude", path: "The Dude", version: "v6" },
48
+ { wikiPath: "Manual:The_Dude_v6/Installation", slug: "Installation", path: "The Dude > v6 > Installation", version: "v6" },
49
+ { wikiPath: "Manual:The_Dude_v6/First_use", slug: "First_use", path: "The Dude > v6 > First Use", version: "v6" },
50
+ { wikiPath: "Manual:The_Dude_v6/Interface", slug: "Interface", path: "The Dude > v6 > Interface", version: "v6" },
51
+ { wikiPath: "Manual:The_Dude_v6/Device_settings", slug: "Device_settings", path: "The Dude > v6 > Device Settings", version: "v6" },
52
+ { wikiPath: "Manual:The_Dude_v6/Device_discovery", slug: "Device_discovery", path: "The Dude > v6 > Device Discovery", version: "v6" },
53
+ { wikiPath: "Manual:The_Dude_v6/Device_map", slug: "Device_map", path: "The Dude > v6 > Device Map", version: "v6" },
54
+ { wikiPath: "Manual:The_Dude_v6/Device_list", slug: "Device_list", path: "The Dude > v6 > Device List", version: "v6" },
55
+ { wikiPath: "Manual:The_Dude_v6/Networks", slug: "Networks", path: "The Dude > v6 > Networks", version: "v6" },
56
+ { wikiPath: "Manual:The_Dude_v6/Links", slug: "Links", path: "The Dude > v6 > Links", version: "v6" },
57
+ { wikiPath: "Manual:The_Dude_v6/Agents", slug: "Agents", path: "The Dude > v6 > Agents", version: "v6" },
58
+ { wikiPath: "Manual:The_Dude_v6/Charts", slug: "Charts", path: "The Dude > v6 > Charts", version: "v6" },
59
+ { wikiPath: "Manual:The_Dude_v6/Functions", slug: "Functions", path: "The Dude > v6 > Functions", version: "v6" },
60
+ { wikiPath: "Manual:The_Dude_v6/Probes", slug: "Probes", path: "The Dude > v6 > Probes", version: "v6" },
61
+ { wikiPath: "Manual:The_Dude_v6/MIB_Nodes", slug: "MIB_Nodes", path: "The Dude > v6 > MIB Nodes", version: "v6" },
62
+ { wikiPath: "Manual:The_Dude_v6/Notifications", slug: "Notifications", path: "The Dude > v6 > Notifications", version: "v6" },
63
+ { wikiPath: "Manual:The_Dude_v6/Syslog", slug: "Syslog", path: "The Dude > v6 > Syslog", version: "v6" },
64
+ { wikiPath: "Manual:The_Dude_v6/Services", slug: "Services", path: "The Dude > v6 > Services", version: "v6" },
65
+ { wikiPath: "Manual:The_Dude_v6/Logs", slug: "Logs", path: "The Dude > v6 > Logs", version: "v6" },
66
+ { wikiPath: "Manual:The_Dude_v6/History", slug: "History", path: "The Dude > v6 > History", version: "v6" },
67
+ { wikiPath: "Manual:The_Dude_v6/Tools", slug: "Tools", path: "The Dude > v6 > Tools", version: "v6" },
68
+ { wikiPath: "Manual:The_Dude_v6/Panels", slug: "Panels", path: "The Dude > v6 > Panels", version: "v6" },
69
+ { wikiPath: "Manual:The_Dude_v6/Server_settings", slug: "Server_settings", path: "The Dude > v6 > Server Settings", version: "v6" },
70
+ { wikiPath: "Manual:The_Dude_v6/Files", slug: "Files", path: "The Dude > v6 > Files", version: "v6" },
71
+ { wikiPath: "Manual:The_Dude_v6/Exporting", slug: "Exporting", path: "The Dude > v6 > Search and Export", version: "v6" },
72
+ { wikiPath: "Manual:The_Dude_v6/Address_lists", slug: "Address_lists", path: "The Dude > v6 > Address Lists", version: "v6" },
73
+ { wikiPath: "Manual:The_Dude_v6/Admins", slug: "Admins", path: "The Dude > v6 > Admins", version: "v6" },
74
+ { wikiPath: "Manual:The_Dude_v6/DB_import_export", slug: "DB_import_export", path: "The Dude > v6 > Database Import and Export", version: "v6" },
75
+ { wikiPath: "Manual:The_Dude_v6/Change_DB_path", slug: "Change_DB_path", path: "The Dude > v6 > Change DB Path", version: "v6" },
76
+ { wikiPath: "Manual:The_Dude_v6/db_vacuum", slug: "db_vacuum", path: "The Dude > v6 > SQLite3 DB Vacuum", version: "v6" },
77
+ { wikiPath: "Manual:The_Dude_v6/MigrationToNewDude", slug: "MigrationToNewDude", path: "The Dude > v6 > Migration from v3/v4 to v6", version: "v6" },
78
+ { wikiPath: "Manual:The_Dude_v6/Malformed_db_repair", slug: "Malformed_db_repair", path: "The Dude > v6 > Malformed DB Repair", version: "v6" },
79
+ { wikiPath: "Manual:The_Dude_v6/The_dude_server_on_VM_CHR", slug: "Server_on_CHR", path: "The Dude > v6 > Server on CHR", version: "v6" },
80
+ { wikiPath: "Manual:The_Dude_v6/The_Dude_server_on_hEX_RB750Gr3", slug: "Server_on_hEX", path: "The Dude > v6 > Server on hEX RB750Gr3", version: "v6" },
81
+ { wikiPath: "Manual:The_Dude_v6/Dude_Telegram_Example", slug: "Telegram_Example", path: "The Dude > v6 > Telegram Notification Example", version: "v6" },
82
+ { wikiPath: "Manual:The_Dude_v6/client_shortcut_arguments", slug: "Client_shortcuts", path: "The Dude > v6 > Client Shortcut Arguments", version: "v6" },
83
+ { wikiPath: "Manual:The_Dude_v6/dude_v6.xx_changelog", slug: "Changelog_v6", path: "The Dude > v6 > Version Changelog", version: "v6" },
84
+ ];
85
+
86
+ const V3_PAGES: PageDef[] = [
87
+ { wikiPath: "Manual:The_Dude4", slug: "v3_Overview", path: "The Dude > v3/v4 > Overview", version: "v3" },
88
+ { wikiPath: "Manual:The_Dude/Installation", slug: "v3_Installation", path: "The Dude > v3/v4 > Installation", version: "v3" },
89
+ { wikiPath: "Manual:The_Dude/First_use", slug: "v3_First_use", path: "The Dude > v3/v4 > First Use", version: "v3" },
90
+ { wikiPath: "Manual:The_Dude/Interface", slug: "v3_Interface", path: "The Dude > v3/v4 > Interface", version: "v3" },
91
+ { wikiPath: "Manual:The_Dude/Device_settings", slug: "v3_Device_settings", path: "The Dude > v3/v4 > Device Settings", version: "v3" },
92
+ { wikiPath: "Manual:The_Dude/Device_discovery", slug: "v3_Device_discovery", path: "The Dude > v3/v4 > Device Discovery", version: "v3" },
93
+ { wikiPath: "Manual:The_Dude/Device_map", slug: "v3_Device_map", path: "The Dude > v3/v4 > Device Map", version: "v3" },
94
+ { wikiPath: "Manual:The_Dude/Device_list", slug: "v3_Device_list", path: "The Dude > v3/v4 > Device List", version: "v3" },
95
+ { wikiPath: "Manual:The_Dude/Networks", slug: "v3_Networks", path: "The Dude > v3/v4 > Networks", version: "v3" },
96
+ { wikiPath: "Manual:The_Dude/Links", slug: "v3_Links", path: "The Dude > v3/v4 > Links", version: "v3" },
97
+ { wikiPath: "Manual:The_Dude/Agents", slug: "v3_Agents", path: "The Dude > v3/v4 > Agents", version: "v3" },
98
+ { wikiPath: "Manual:The_Dude/Charts", slug: "v3_Charts", path: "The Dude > v3/v4 > Charts", version: "v3" },
99
+ { wikiPath: "Manual:The_Dude/Functions", slug: "v3_Functions", path: "The Dude > v3/v4 > Functions", version: "v3" },
100
+ { wikiPath: "Manual:The_Dude/Probes", slug: "v3_Probes", path: "The Dude > v3/v4 > Probes", version: "v3" },
101
+ { wikiPath: "Manual:The_Dude/MIB_Nodes", slug: "v3_MIB_Nodes", path: "The Dude > v3/v4 > MIB Nodes", version: "v3" },
102
+ { wikiPath: "Manual:The_Dude/Notifications", slug: "v3_Notifications", path: "The Dude > v3/v4 > Notifications", version: "v3" },
103
+ { wikiPath: "Manual:The_Dude/Syslog", slug: "v3_Syslog", path: "The Dude > v3/v4 > Syslog", version: "v3" },
104
+ { wikiPath: "Manual:The_Dude/Services", slug: "v3_Services", path: "The Dude > v3/v4 > Services", version: "v3" },
105
+ { wikiPath: "Manual:The_Dude/Logs", slug: "v3_Logs", path: "The Dude > v3/v4 > Logs", version: "v3" },
106
+ { wikiPath: "Manual:The_Dude/History", slug: "v3_History", path: "The Dude > v3/v4 > History", version: "v3" },
107
+ { wikiPath: "Manual:The_Dude/Tools", slug: "v3_Tools", path: "The Dude > v3/v4 > Tools", version: "v3" },
108
+ { wikiPath: "Manual:The_Dude/Panels", slug: "v3_Panels", path: "The Dude > v3/v4 > Panels", version: "v3" },
109
+ { wikiPath: "Manual:The_Dude/Server_settings", slug: "v3_Server_settings", path: "The Dude > v3/v4 > Server Settings", version: "v3" },
110
+ { wikiPath: "Manual:The_Dude/Files", slug: "v3_Files", path: "The Dude > v3/v4 > Files", version: "v3" },
111
+ { wikiPath: "Manual:The_Dude/Exporting", slug: "v3_Exporting", path: "The Dude > v3/v4 > Exporting", version: "v3" },
112
+ { wikiPath: "Manual:The_Dude/Address_lists", slug: "v3_Address_lists", path: "The Dude > v3/v4 > Address Lists", version: "v3" },
113
+ { wikiPath: "Manual:The_Dude/Admins", slug: "v3_Admins", path: "The Dude > v3/v4 > Admins", version: "v3" },
114
+ { wikiPath: "Manual:The_Dude/Web_interface", slug: "v3_Web_interface", path: "The Dude > v3/v4 > Web Interface", version: "v3" },
115
+ { wikiPath: "Manual:The_Dude/Changelog", slug: "v3_Changelog", path: "The Dude > v3/v4 > Changelog", version: "v3" },
116
+ ];
117
+
118
+ const ALL_PAGES: PageDef[] = [...V6_PAGES, ...V3_PAGES];
119
+
120
+ // ── Wayback Machine helpers ──
121
+
122
+ const WIKI_BASE = "https://wiki.mikrotik.com/wiki/";
123
+
124
+ /** Build a Wayback Machine URL. Uses wildcard timestamp to get latest snapshot. */
125
+ function waybackUrl(wikiPath: string): string {
126
+ return `https://web.archive.org/web/2024/${WIKI_BASE}${encodeURIComponent(wikiPath).replace(/%2F/g, "/").replace(/%3A/g, ":")}`;
127
+ }
128
+
129
+ function delay(ms: number): Promise<void> {
130
+ return new Promise((resolve) => setTimeout(resolve, ms));
131
+ }
132
+
133
+ /** Fetch with retry for Wayback Machine rate limits. */
134
+ async function fetchWithRetry(url: string, retries = 3): Promise<Response> {
135
+ for (let attempt = 0; attempt < retries; attempt++) {
136
+ const response = await fetch(url, {
137
+ headers: { "User-Agent": "rosetta-dude-extractor/1.0 (MikroTik documentation tool)" },
138
+ });
139
+ if (response.ok) return response;
140
+ if (response.status === 429 || response.status === 503) {
141
+ const waitMs = (attempt + 1) * 2000;
142
+ console.log(` Rate limited (${response.status}), waiting ${waitMs}ms...`);
143
+ await delay(waitMs);
144
+ continue;
145
+ }
146
+ if (response.status === 404) {
147
+ throw new Error(`404 Not Found: ${url}`);
148
+ }
149
+ throw new Error(`HTTP ${response.status}: ${url}`);
150
+ }
151
+ throw new Error(`Failed after ${retries} retries: ${url}`);
152
+ }
153
+
154
+ // ── HTML parsing ──
155
+
156
+ interface ParsedPage {
157
+ title: string;
158
+ text: string;
159
+ code: string;
160
+ lastEdited: string | null;
161
+ images: Array<{ filename: string; altText: string | null; originalUrl: string; waybackImageUrl: string }>;
162
+ }
163
+
164
+ function parsePage(html: string, _waybackBaseUrl: string): ParsedPage {
165
+ const { document } = parseHTML(html);
166
+
167
+ // Title from first h1 or page heading
168
+ const h1 = document.querySelector("#firstHeading, h1.firstHeading, h1");
169
+ const title = h1?.textContent?.trim()?.replace(/^Manual:The Dude v6\//, "")
170
+ .replace(/^Manual:The Dude\//, "")
171
+ .replace(/^Manual:The Dude4?$/, "The Dude")
172
+ .replace(/_/g, " ") ?? "Unknown";
173
+
174
+ // Main content area
175
+ const content = document.querySelector("#mw-content-text, .mw-parser-output, #bodyContent");
176
+ if (!content) return { title, text: "", code: "", lastEdited: null, images: [] };
177
+
178
+ // Remove navigation, TOC, edit links, Wayback Machine toolbar
179
+ for (const sel of ["#toc", ".toc", ".mw-editsection", ".navbox", "#catlinks",
180
+ ".printfooter", "#wm-ipp-base", "#wm-ipp", "#donato", ".wb-autocomplete-suggestions",
181
+ "#mw-navigation", "#footer", ".noprint"]) {
182
+ for (const el of content.querySelectorAll(sel)) el.remove();
183
+ }
184
+
185
+ // Extract code blocks
186
+ const codeBlocks: string[] = [];
187
+ for (const pre of content.querySelectorAll("pre, code.ros")) {
188
+ const text = pre.textContent?.trim();
189
+ if (text && text.length > 10) codeBlocks.push(text);
190
+ }
191
+
192
+ // Extract images
193
+ const images: ParsedPage["images"] = [];
194
+ const seenFilenames = new Set<string>();
195
+ for (const img of content.querySelectorAll("img")) {
196
+ const src = img.getAttribute("src") ?? "";
197
+ const alt = img.getAttribute("alt") ?? null;
198
+
199
+ // Skip tiny icons, navigation icons, and Wayback Machine UI images
200
+ const width = Number.parseInt(img.getAttribute("width") ?? "0", 10);
201
+ if (width > 0 && width < 30) continue;
202
+ if (src.includes("web.archive.org/static/") || src.includes("/_static/")) continue;
203
+
204
+ // Extract filename from MediaWiki image URLs
205
+ // Patterns: /images/X/XX/Filename.JPG or /wiki/File:Filename.JPG
206
+ let filename: string | null = null;
207
+ const srcMatch = src.match(/\/images\/[0-9a-f]\/[0-9a-f]{2}\/([^/?]+)/i)
208
+ ?? src.match(/\/([^/]+\.(?:jpg|jpeg|png|gif|svg))(?:\?|$)/i);
209
+ if (srcMatch) {
210
+ filename = decodeURIComponent(srcMatch[1]);
211
+ }
212
+
213
+ // Also check parent <a> links for File: references
214
+ if (!filename) {
215
+ const parentA = img.closest("a");
216
+ const href = parentA?.getAttribute("href") ?? "";
217
+ const fileMatch = href.match(/File:([^&"]+)/);
218
+ if (fileMatch) filename = decodeURIComponent(fileMatch[1]);
219
+ }
220
+
221
+ if (!filename) continue;
222
+ if (seenFilenames.has(filename)) continue;
223
+ // Skip common wiki icons
224
+ if (/^Icon-\w+\.png$/i.test(filename)) continue;
225
+ if (/^Version\.png$/i.test(filename)) continue;
226
+ seenFilenames.add(filename);
227
+
228
+ // Build a usable image URL from the Wayback Machine
229
+ let waybackImageUrl = src;
230
+ if (src.startsWith("/")) {
231
+ waybackImageUrl = `https://web.archive.org${src}`;
232
+ }
233
+ // Ensure we use the raw image URL (id_ prefix bypasses Wayback rewriting)
234
+ waybackImageUrl = waybackImageUrl.replace(
235
+ /\/web\/\d+\//,
236
+ "/web/2024id_/",
237
+ );
238
+
239
+ const originalUrl = `${WIKI_BASE}File:${encodeURIComponent(filename)}`;
240
+
241
+ images.push({ filename, altText: alt, originalUrl, waybackImageUrl });
242
+ }
243
+
244
+ // Extract body text
245
+ const textContent = content.textContent ?? "";
246
+ // Clean up whitespace: collapse multiple blank lines, trim
247
+ const text = textContent.replace(/\n{3,}/g, "\n\n").trim();
248
+
249
+ // Last edited date from footer
250
+ const lastEditedEl = document.querySelector("#footer-info-lastmod, .lastmod");
251
+ const lastEdited = lastEditedEl?.textContent?.replace(/.*last edited on\s*/i, "").trim() ?? null;
252
+
253
+ return { title, text, code: codeBlocks.join("\n\n"), lastEdited, images };
254
+ }
255
+
256
+ // ── Image download ──
257
+
258
+ async function downloadImage(url: string, dest: string): Promise<boolean> {
259
+ if (existsSync(dest) && !FORCE) return true;
260
+ try {
261
+ const response = await fetchWithRetry(url);
262
+ const buffer = await response.arrayBuffer();
263
+ if (buffer.byteLength < 100) return false; // too small, probably an error page
264
+ writeFileSync(dest, Buffer.from(buffer));
265
+ return true;
266
+ } catch (e) {
267
+ console.log(` Warning: failed to download image: ${e}`);
268
+ return false;
269
+ }
270
+ }
271
+
272
+ // ── Main extraction ──
273
+
274
+ async function main() {
275
+ initDb();
276
+
277
+ mkdirSync(PAGES_DIR, { recursive: true });
278
+ mkdirSync(IMAGES_DIR, { recursive: true });
279
+
280
+ // Idempotent: clear existing data
281
+ db.run("DELETE FROM dude_images");
282
+ db.run("DELETE FROM dude_pages");
283
+
284
+ const insertPage = db.prepare(`
285
+ INSERT INTO dude_pages (slug, title, path, version, url, wayback_url, text, code, last_edited, word_count)
286
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
287
+ `);
288
+ const insertImage = db.prepare(`
289
+ INSERT INTO dude_images (page_id, filename, alt_text, caption, local_path, original_url, wayback_url, sort_order)
290
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)
291
+ `);
292
+
293
+ let pageCount = 0;
294
+ let imageCount = 0;
295
+ let errorCount = 0;
296
+
297
+ for (const pageDef of ALL_PAGES) {
298
+ const cacheFile = join(PAGES_DIR, `${pageDef.slug}.html`);
299
+ const wbUrl = waybackUrl(pageDef.wikiPath);
300
+ const originalUrl = `${WIKI_BASE}${pageDef.wikiPath}`;
301
+
302
+ let html: string;
303
+
304
+ if (FROM_CACHE || (existsSync(cacheFile) && !FORCE)) {
305
+ // Read from cache
306
+ if (!existsSync(cacheFile)) {
307
+ console.log(` SKIP (no cache): ${pageDef.slug}`);
308
+ continue;
309
+ }
310
+ html = readFileSync(cacheFile, "utf-8");
311
+ console.log(` [cache] ${pageDef.slug}`);
312
+ } else {
313
+ // Fetch from Wayback Machine
314
+ console.log(` [fetch] ${pageDef.slug} ...`);
315
+ try {
316
+ const response = await fetchWithRetry(wbUrl);
317
+ html = await response.text();
318
+ // Cache the raw HTML
319
+ writeFileSync(cacheFile, html);
320
+ await delay(FETCH_DELAY_MS);
321
+ } catch (e) {
322
+ console.log(` ERROR: ${pageDef.slug}: ${e}`);
323
+ errorCount++;
324
+ continue;
325
+ }
326
+ }
327
+
328
+ // Parse HTML
329
+ const parsed = parsePage(html, wbUrl);
330
+ if (!parsed.text && !parsed.code) {
331
+ console.log(` WARN: empty content for ${pageDef.slug}`);
332
+ }
333
+
334
+ const wordCount = parsed.text.split(/\s+/).filter(Boolean).length;
335
+
336
+ // Insert page
337
+ db.transaction(() => {
338
+ insertPage.run(
339
+ pageDef.slug,
340
+ parsed.title,
341
+ pageDef.path,
342
+ pageDef.version,
343
+ originalUrl,
344
+ wbUrl,
345
+ parsed.text,
346
+ parsed.code || null,
347
+ parsed.lastEdited,
348
+ wordCount,
349
+ );
350
+
351
+ const pageRow = db.prepare("SELECT id FROM dude_pages WHERE slug = ?").get(pageDef.slug) as { id: number };
352
+ const pageId = pageRow.id;
353
+
354
+ // Download and insert images
355
+ for (let i = 0; i < parsed.images.length; i++) {
356
+ const img = parsed.images[i];
357
+ const localPath = `dude/images/${img.filename}`;
358
+
359
+ insertImage.run(
360
+ pageId,
361
+ img.filename,
362
+ img.altText,
363
+ null, // caption — could be inferred from surrounding text
364
+ localPath,
365
+ img.originalUrl,
366
+ img.waybackImageUrl,
367
+ i,
368
+ );
369
+ imageCount++;
370
+ }
371
+ })();
372
+
373
+ pageCount++;
374
+ }
375
+
376
+ // Download images outside the DB transaction loop (allows partial success)
377
+ if (SKIP_IMAGES) {
378
+ console.log("\nSkipping image download (--skip-images).");
379
+ } else {
380
+ console.log("\nDownloading images...");
381
+ const allImages = db.prepare("SELECT DISTINCT filename, wayback_url FROM dude_images").all() as Array<{ filename: string; wayback_url: string }>;
382
+ let downloadedCount = 0;
383
+ let skipCount = 0;
384
+ for (const img of allImages) {
385
+ const destPath = join(IMAGES_DIR, img.filename);
386
+ if (existsSync(destPath) && !FORCE) {
387
+ skipCount++;
388
+ continue;
389
+ }
390
+ const ok = await downloadImage(img.wayback_url, destPath);
391
+ if (ok) {
392
+ downloadedCount++;
393
+ console.log(` ✓ ${img.filename}`);
394
+ }
395
+ await delay(FETCH_DELAY_MS);
396
+ }
397
+ console.log(`\nDone: ${pageCount} pages, ${imageCount} image refs, ${downloadedCount} downloaded, ${skipCount} cached, ${errorCount} errors`);
398
+ }
399
+
400
+ // Summary
401
+ const stats = db.prepare("SELECT COUNT(*) AS c FROM dude_pages").get() as { c: number };
402
+ const imgStats = db.prepare("SELECT COUNT(*) AS c FROM dude_images").get() as { c: number };
403
+ console.log(`DB: ${stats.c} dude_pages, ${imgStats.c} dude_images`);
404
+ }
405
+
406
+ main().catch((e) => {
407
+ console.error("Fatal:", e);
408
+ process.exit(1);
409
+ });
@@ -108,6 +108,8 @@ function createFixtureDb(dbPath: string): void {
108
108
  1, '/system', 'system', 'dir', NULL, 1, 'fixture command', '7.22'
109
109
  );`);
110
110
 
111
+ // Stamp schema version so mcp.ts doesn't trigger a 50MB auto-download
112
+ fixture.run("PRAGMA user_version = 1;");
111
113
  fixture.close();
112
114
  }
113
115
 
@@ -315,7 +317,7 @@ describe("HTTP transport: session lifecycle", () => {
315
317
  expect(serverInfo.name).toBe("rosetta");
316
318
  });
317
319
 
318
- test("tools/list returns all 14 tools after initialization", async () => {
320
+ test("tools/list returns all 16 tools after initialization", async () => {
319
321
  const { sessionId } = await mcpInitialize(server.url);
320
322
 
321
323
  // Send initialized notification first (required by protocol)
@@ -326,7 +328,7 @@ describe("HTTP transport: session lifecycle", () => {
326
328
 
327
329
  const result = (messages[0] as Record<string, unknown>).result as Record<string, unknown>;
328
330
  const tools = result.tools as Array<{ name: string }>;
329
- expect(tools.length).toBe(14);
331
+ expect(tools.length).toBe(16);
330
332
 
331
333
  const toolNames = tools.map((t) => t.name).sort();
332
334
  expect(toolNames).toContain("routeros_search");
@@ -547,8 +549,8 @@ describe("HTTP transport: multi-session", () => {
547
549
  const tools1 = ((msgs1[0] as Record<string, unknown>).result as Record<string, unknown>).tools as unknown[];
548
550
  const tools2 = ((msgs2[0] as Record<string, unknown>).result as Record<string, unknown>).tools as unknown[];
549
551
 
550
- expect(tools1.length).toBe(14);
551
- expect(tools2.length).toBe(14);
552
+ expect(tools1.length).toBe(16);
553
+ expect(tools2.length).toBe(16);
552
554
  });
553
555
 
554
556
  test("deleting one session does not affect another", async () => {
@@ -570,7 +572,7 @@ describe("HTTP transport: multi-session", () => {
570
572
  // Client2 still works
571
573
  const msgs = await mcpRequest(server.url, client2.sessionId, "tools/list", 2);
572
574
  const tools = ((msgs[0] as Record<string, unknown>).result as Record<string, unknown>).tools as unknown[];
573
- expect(tools.length).toBe(14);
575
+ expect(tools.length).toBe(16);
574
576
 
575
577
  // Client1 is gone
576
578
  const resp = await fetch(server.url, {