@tikoci/rosetta 0.8.5 → 0.8.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/extract-test-results.test.ts +193 -0
- package/src/extract-test-results.ts +117 -109
package/package.json
CHANGED
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
// Set BEFORE importing extract-test-results.ts (which transitively imports
|
|
2
|
+
// db.ts). Prevents this test file from pinning the DB singleton to a real
|
|
3
|
+
// ros-help.db path before query.test.ts can enforce its in-memory guard.
|
|
4
|
+
process.env.DB_PATH = ":memory:";
|
|
5
|
+
|
|
6
|
+
import { describe, expect, it } from "bun:test";
|
|
7
|
+
import { parseHTML } from "linkedom";
|
|
8
|
+
|
|
9
|
+
// Dynamic import so the DB_PATH assignment above is visible before db.ts loads.
|
|
10
|
+
const { parsePerformanceTable } = await import("./extract-test-results.ts");
|
|
11
|
+
|
|
12
|
+
// ── Fixture helpers ──────────────────────────────────────────────────────────
|
|
13
|
+
|
|
14
|
+
/** Build a minimal performance-table HTML matching the real MikroTik page structure. */
|
|
15
|
+
function makeEthernetTable(rows: string[]): Element {
|
|
16
|
+
const html = `
|
|
17
|
+
<table class="performance-table">
|
|
18
|
+
<thead>
|
|
19
|
+
<tr>
|
|
20
|
+
<td>RB1100Dx4</td>
|
|
21
|
+
<td>AL21400 1G all port test</td>
|
|
22
|
+
</tr>
|
|
23
|
+
<tr>
|
|
24
|
+
<td>Mode</td>
|
|
25
|
+
<td>Configuration</td>
|
|
26
|
+
<td>1518 byte</td>
|
|
27
|
+
<td></td>
|
|
28
|
+
<td>512 byte</td>
|
|
29
|
+
<td></td>
|
|
30
|
+
<td>64 byte</td>
|
|
31
|
+
<td></td>
|
|
32
|
+
</tr>
|
|
33
|
+
<tr>
|
|
34
|
+
<td></td><td></td>
|
|
35
|
+
<td>kpps</td><td>Mbps</td>
|
|
36
|
+
<td>kpps</td><td>Mbps</td>
|
|
37
|
+
<td>kpps</td><td>Mbps</td>
|
|
38
|
+
</tr>
|
|
39
|
+
</thead>
|
|
40
|
+
<tbody>
|
|
41
|
+
${rows.join("\n ")}
|
|
42
|
+
</tbody>
|
|
43
|
+
</table>`;
|
|
44
|
+
const { document } = parseHTML(`<html><body>${html}</body></html>`);
|
|
45
|
+
const table = document.querySelector("table");
|
|
46
|
+
if (!table) throw new Error("fixture build failed");
|
|
47
|
+
return table;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function makeIpsecTable(rows: string[]): Element {
|
|
51
|
+
const html = `
|
|
52
|
+
<table class="performance-table">
|
|
53
|
+
<thead>
|
|
54
|
+
<tr>
|
|
55
|
+
<td>RB1100Dx4</td>
|
|
56
|
+
<td>IPsec AES hardware acceleration</td>
|
|
57
|
+
</tr>
|
|
58
|
+
<tr>
|
|
59
|
+
<td>Mode</td>
|
|
60
|
+
<td>Configuration</td>
|
|
61
|
+
<td>1400 byte</td>
|
|
62
|
+
<td></td>
|
|
63
|
+
<td>512 byte</td>
|
|
64
|
+
<td></td>
|
|
65
|
+
<td>64 byte</td>
|
|
66
|
+
<td></td>
|
|
67
|
+
</tr>
|
|
68
|
+
<tr>
|
|
69
|
+
<td></td><td></td>
|
|
70
|
+
<td>kpps</td><td>Mbps</td>
|
|
71
|
+
<td>kpps</td><td>Mbps</td>
|
|
72
|
+
<td>kpps</td><td>Mbps</td>
|
|
73
|
+
</tr>
|
|
74
|
+
</thead>
|
|
75
|
+
<tbody>
|
|
76
|
+
${rows.join("\n ")}
|
|
77
|
+
</tbody>
|
|
78
|
+
</table>`;
|
|
79
|
+
const { document } = parseHTML(`<html><body>${html}</body></html>`);
|
|
80
|
+
const table = document.querySelector("table");
|
|
81
|
+
if (!table) throw new Error("fixture build failed");
|
|
82
|
+
return table;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// ── Tests ────────────────────────────────────────────────────────────────────
|
|
86
|
+
|
|
87
|
+
describe("parsePerformanceTable", () => {
|
|
88
|
+
describe("type detection", () => {
|
|
89
|
+
it("detects ethernet type from header", () => {
|
|
90
|
+
const table = makeEthernetTable([
|
|
91
|
+
"<tr><td>Bridging</td><td>none (fast path)</td><td>606.5</td><td>7,365.3</td><td>1,736.4</td><td>7,112.3</td><td>5,509.7</td><td>2,821.0</td></tr>",
|
|
92
|
+
]);
|
|
93
|
+
const { testType } = parsePerformanceTable(table);
|
|
94
|
+
expect(testType).toBe("ethernet");
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
it("detects ipsec type from header", () => {
|
|
98
|
+
const table = makeIpsecTable([
|
|
99
|
+
"<tr><td>Single tunnel</td><td>AES-128-CBC + SHA1</td><td>92.1</td><td>1,031.5</td><td>93.1</td><td>381.3</td><td>92.3</td><td>47.3</td></tr>",
|
|
100
|
+
]);
|
|
101
|
+
const { testType } = parsePerformanceTable(table);
|
|
102
|
+
expect(testType).toBe("ipsec");
|
|
103
|
+
});
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
describe("thousands-separator handling", () => {
|
|
107
|
+
it("correctly parses values with comma thousands separators (regression: RB1100Dx4 512-byte row)", () => {
|
|
108
|
+
// Website: Bridging, none (fast path), 512B → kpps=1,736.4, Mbps=7,112.3
|
|
109
|
+
// Before fix: parseFloat("1,736.4") → 1, parseFloat("7,112.3") → 7
|
|
110
|
+
const table = makeEthernetTable([
|
|
111
|
+
"<tr><td>Bridging</td><td>none (fast path)</td><td>606.5</td><td>7,365.3</td><td>1,736.4</td><td>7,112.3</td><td>5,509.7</td><td>2,821.0</td></tr>",
|
|
112
|
+
]);
|
|
113
|
+
const { rows } = parsePerformanceTable(table);
|
|
114
|
+
|
|
115
|
+
const row512 = rows.find((r) => r.packet_size === 512);
|
|
116
|
+
expect(row512).toBeDefined();
|
|
117
|
+
expect(row512?.throughput_kpps).toBeCloseTo(1736.4);
|
|
118
|
+
expect(row512?.throughput_mbps).toBeCloseTo(7112.3);
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
it("correctly parses 1518-byte row with large comma-formatted values", () => {
|
|
122
|
+
const table = makeEthernetTable([
|
|
123
|
+
"<tr><td>Bridging</td><td>none (fast path)</td><td>606.5</td><td>7,365.3</td><td>1,736.4</td><td>7,112.3</td><td>5,509.7</td><td>2,821.0</td></tr>",
|
|
124
|
+
]);
|
|
125
|
+
const { rows } = parsePerformanceTable(table);
|
|
126
|
+
|
|
127
|
+
const row1518 = rows.find((r) => r.packet_size === 1518);
|
|
128
|
+
expect(row1518?.throughput_kpps).toBeCloseTo(606.5);
|
|
129
|
+
expect(row1518?.throughput_mbps).toBeCloseTo(7365.3);
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
it("correctly parses 64-byte row with large comma-formatted kpps", () => {
|
|
133
|
+
const table = makeEthernetTable([
|
|
134
|
+
"<tr><td>Bridging</td><td>none (fast path)</td><td>606.5</td><td>7,365.3</td><td>1,736.4</td><td>7,112.3</td><td>5,509.7</td><td>2,821.0</td></tr>",
|
|
135
|
+
]);
|
|
136
|
+
const { rows } = parsePerformanceTable(table);
|
|
137
|
+
|
|
138
|
+
const row64 = rows.find((r) => r.packet_size === 64);
|
|
139
|
+
expect(row64?.throughput_kpps).toBeCloseTo(5509.7);
|
|
140
|
+
expect(row64?.throughput_mbps).toBeCloseTo(2821.0);
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
it("still handles values without thousands separators", () => {
|
|
144
|
+
const table = makeEthernetTable([
|
|
145
|
+
"<tr><td>Routing</td><td>25 simple queues</td><td>606.5</td><td>7365.3</td><td>933.6</td><td>3824.0</td><td>960.3</td><td>491.7</td></tr>",
|
|
146
|
+
]);
|
|
147
|
+
const { rows } = parsePerformanceTable(table);
|
|
148
|
+
|
|
149
|
+
const row512 = rows.find((r) => r.packet_size === 512);
|
|
150
|
+
expect(row512?.throughput_kpps).toBeCloseTo(933.6);
|
|
151
|
+
expect(row512?.throughput_mbps).toBeCloseTo(3824.0);
|
|
152
|
+
});
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
describe("row structure", () => {
|
|
156
|
+
it("emits one row per packet size", () => {
|
|
157
|
+
const table = makeEthernetTable([
|
|
158
|
+
"<tr><td>Bridging</td><td>none (fast path)</td><td>606.5</td><td>7,365.3</td><td>1,736.4</td><td>7,112.3</td><td>5,509.7</td><td>2,821.0</td></tr>",
|
|
159
|
+
]);
|
|
160
|
+
const { rows } = parsePerformanceTable(table);
|
|
161
|
+
expect(rows).toHaveLength(3); // 1518, 512, 64
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
it("preserves mode and configuration strings", () => {
|
|
165
|
+
const table = makeEthernetTable([
|
|
166
|
+
"<tr><td>Routing</td><td>25 ip filter rules</td><td>543.7</td><td>6,602.7</td><td>561.8</td><td>2,301.1</td><td>564.6</td><td>289.1</td></tr>",
|
|
167
|
+
]);
|
|
168
|
+
const { rows } = parsePerformanceTable(table);
|
|
169
|
+
expect(rows[0].mode).toBe("Routing");
|
|
170
|
+
expect(rows[0].configuration).toBe("25 ip filter rules");
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
it("returns null for unparseable cell values", () => {
|
|
174
|
+
const table = makeEthernetTable([
|
|
175
|
+
"<tr><td>Bridging</td><td>none (fast path)</td><td>-</td><td>-</td><td>-</td><td>-</td><td>-</td><td>-</td></tr>",
|
|
176
|
+
]);
|
|
177
|
+
const { rows } = parsePerformanceTable(table);
|
|
178
|
+
expect(rows[0].throughput_kpps).toBeNull();
|
|
179
|
+
expect(rows[0].throughput_mbps).toBeNull();
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
it("parses ipsec rows with 1400/512/64 packet sizes", () => {
|
|
183
|
+
const table = makeIpsecTable([
|
|
184
|
+
"<tr><td>Single tunnel</td><td>AES-128-CBC + SHA1</td><td>92.1</td><td>1,031.5</td><td>93.1</td><td>381.3</td><td>92.3</td><td>47.3</td></tr>",
|
|
185
|
+
]);
|
|
186
|
+
const { rows } = parsePerformanceTable(table);
|
|
187
|
+
const sizes = rows.map((r) => r.packet_size);
|
|
188
|
+
expect(sizes).toEqual([1400, 512, 64]);
|
|
189
|
+
const row1400 = rows.find((r) => r.packet_size === 1400);
|
|
190
|
+
expect(row1400?.throughput_mbps).toBeCloseTo(1031.5);
|
|
191
|
+
});
|
|
192
|
+
});
|
|
193
|
+
});
|
|
@@ -19,7 +19,6 @@
|
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
21
|
import { parseHTML } from "linkedom";
|
|
22
|
-
import { db, initDb } from "./db.ts";
|
|
23
22
|
|
|
24
23
|
// ── CLI flags ──
|
|
25
24
|
|
|
@@ -58,7 +57,7 @@ const SLUG_OVERRIDES: Record<string, string> = {
|
|
|
58
57
|
|
|
59
58
|
// ── Types ──
|
|
60
59
|
|
|
61
|
-
interface TestResultRow {
|
|
60
|
+
export interface TestResultRow {
|
|
62
61
|
mode: string;
|
|
63
62
|
configuration: string;
|
|
64
63
|
packet_size: number;
|
|
@@ -76,7 +75,7 @@ interface ProductPageData {
|
|
|
76
75
|
// ── HTML Parsing ──
|
|
77
76
|
|
|
78
77
|
/** Parse a performance-table element into test result rows. */
|
|
79
|
-
function parsePerformanceTable(table: Element): { testType: string; rows: TestResultRow[] } {
|
|
78
|
+
export function parsePerformanceTable(table: Element): { testType: string; rows: TestResultRow[] } {
|
|
80
79
|
const rows: TestResultRow[] = [];
|
|
81
80
|
|
|
82
81
|
// Header row: first <tr> in <thead> has [product_code, test_description]
|
|
@@ -124,14 +123,16 @@ function parsePerformanceTable(table: Element): { testType: string; rows: TestRe
|
|
|
124
123
|
const config = (cells[1].textContent || "").trim();
|
|
125
124
|
|
|
126
125
|
// Each packet size has 2 columns: kpps, Mbps
|
|
126
|
+
// Strip thousands-separator commas before parsing (e.g. "7,112.3" → 7112.3)
|
|
127
|
+
const parseNum = (s: string) => Number.parseFloat(s.replace(/,/g, "").trim());
|
|
127
128
|
for (let i = 0; i < packetSizes.length; i++) {
|
|
128
129
|
const kppsIdx = 2 + i * 2;
|
|
129
130
|
const mbpsIdx = 3 + i * 2;
|
|
130
131
|
if (kppsIdx >= cells.length) break;
|
|
131
132
|
|
|
132
|
-
const kpps =
|
|
133
|
+
const kpps = parseNum((cells[kppsIdx].textContent || "").trim());
|
|
133
134
|
const mbps = mbpsIdx < cells.length
|
|
134
|
-
?
|
|
135
|
+
? parseNum((cells[mbpsIdx].textContent || "").trim())
|
|
135
136
|
: null;
|
|
136
137
|
|
|
137
138
|
rows.push({
|
|
@@ -309,124 +310,131 @@ function sleep(ms: number): Promise<void> {
|
|
|
309
310
|
}
|
|
310
311
|
|
|
311
312
|
// ── Main ──
|
|
313
|
+
async function main(): Promise<void> {
|
|
314
|
+
const { db, initDb } = await import("./db.ts");
|
|
312
315
|
|
|
313
|
-
initDb();
|
|
316
|
+
initDb();
|
|
314
317
|
|
|
315
|
-
// Get all devices from DB
|
|
316
|
-
const devices = db.prepare("SELECT id, product_name, product_code FROM devices ORDER BY product_name").all() as Array<{
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
}>;
|
|
318
|
+
// Get all devices from DB
|
|
319
|
+
const devices = db.prepare("SELECT id, product_name, product_code FROM devices ORDER BY product_name").all() as Array<{
|
|
320
|
+
id: number;
|
|
321
|
+
product_name: string;
|
|
322
|
+
product_code: string | null;
|
|
323
|
+
}>;
|
|
321
324
|
|
|
322
|
-
if (devices.length === 0) {
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
}
|
|
326
|
-
|
|
327
|
-
console.log(`Found ${devices.length} devices in database`);
|
|
328
|
-
|
|
329
|
-
// Fetch sitemap once to validate slugs and prioritize correct candidates
|
|
330
|
-
console.log("Loading product sitemap...");
|
|
331
|
-
const sitemapSlugs = await fetchSitemap();
|
|
332
|
-
|
|
333
|
-
// Build device → candidate slugs mapping
|
|
334
|
-
const deviceSlugs: Array<{ id: number; name: string; slugs: string[] }> = [];
|
|
335
|
-
for (const dev of devices) {
|
|
336
|
-
const slugs = buildSlugCandidates(dev.product_name, dev.product_code, sitemapSlugs);
|
|
337
|
-
deviceSlugs.push({ id: dev.id, name: dev.product_name, slugs });
|
|
338
|
-
}
|
|
339
|
-
|
|
340
|
-
// Idempotent: clear existing test results
|
|
341
|
-
db.run("DELETE FROM device_test_results");
|
|
342
|
-
|
|
343
|
-
// Prepare statements
|
|
344
|
-
const insertTest = db.prepare(`INSERT OR IGNORE INTO device_test_results (
|
|
345
|
-
device_id, test_type, mode, configuration, packet_size,
|
|
346
|
-
throughput_kpps, throughput_mbps
|
|
347
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?)`);
|
|
348
|
-
|
|
349
|
-
const updateDevice = db.prepare(`UPDATE devices
|
|
350
|
-
SET product_url = ?, block_diagram_url = ?
|
|
351
|
-
WHERE id = ?`);
|
|
352
|
-
|
|
353
|
-
console.log(`Fetching ${deviceSlugs.length} product pages (concurrency=${CONCURRENCY}, delay=${DELAY_MS}ms)...`);
|
|
354
|
-
|
|
355
|
-
let totalTests = 0;
|
|
356
|
-
let devicesWithTests = 0;
|
|
357
|
-
let devicesWithDiagrams = 0;
|
|
358
|
-
let fetchErrors = 0;
|
|
359
|
-
|
|
360
|
-
const insertAll = db.transaction(
|
|
361
|
-
(results: Array<{ deviceId: number; data: ProductPageData | null }>) => {
|
|
362
|
-
for (const { deviceId, data } of results) {
|
|
363
|
-
if (!data) {
|
|
364
|
-
fetchErrors++;
|
|
365
|
-
continue;
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
// Update device with URL and block diagram
|
|
369
|
-
updateDevice.run(
|
|
370
|
-
`https://mikrotik.com/product/${data.slug}`,
|
|
371
|
-
data.block_diagram_url,
|
|
372
|
-
deviceId,
|
|
373
|
-
);
|
|
325
|
+
if (devices.length === 0) {
|
|
326
|
+
console.error("No devices in database. Run extract-devices.ts first.");
|
|
327
|
+
process.exit(1);
|
|
328
|
+
}
|
|
374
329
|
|
|
375
|
-
|
|
330
|
+
console.log(`Found ${devices.length} devices in database`);
|
|
376
331
|
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
...data.ipsec_results.map((r) => ({ ...r, test_type: "ipsec" as const })),
|
|
381
|
-
];
|
|
332
|
+
// Fetch sitemap once to validate slugs and prioritize correct candidates
|
|
333
|
+
console.log("Loading product sitemap...");
|
|
334
|
+
const sitemapSlugs = await fetchSitemap();
|
|
382
335
|
|
|
383
|
-
|
|
336
|
+
// Build device -> candidate slugs mapping
|
|
337
|
+
const deviceSlugs: Array<{ id: number; name: string; slugs: string[] }> = [];
|
|
338
|
+
for (const dev of devices) {
|
|
339
|
+
const slugs = buildSlugCandidates(dev.product_name, dev.product_code, sitemapSlugs);
|
|
340
|
+
deviceSlugs.push({ id: dev.id, name: dev.product_name, slugs });
|
|
341
|
+
}
|
|
384
342
|
|
|
385
|
-
|
|
386
|
-
|
|
343
|
+
// Idempotent: clear existing test results
|
|
344
|
+
db.run("DELETE FROM device_test_results");
|
|
345
|
+
|
|
346
|
+
// Prepare statements
|
|
347
|
+
const insertTest = db.prepare(`INSERT OR IGNORE INTO device_test_results (
|
|
348
|
+
device_id, test_type, mode, configuration, packet_size,
|
|
349
|
+
throughput_kpps, throughput_mbps
|
|
350
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?)`);
|
|
351
|
+
|
|
352
|
+
const updateDevice = db.prepare(`UPDATE devices
|
|
353
|
+
SET product_url = ?, block_diagram_url = ?
|
|
354
|
+
WHERE id = ?`);
|
|
355
|
+
|
|
356
|
+
console.log(`Fetching ${deviceSlugs.length} product pages (concurrency=${CONCURRENCY}, delay=${DELAY_MS}ms)...`);
|
|
357
|
+
|
|
358
|
+
let totalTests = 0;
|
|
359
|
+
let devicesWithTests = 0;
|
|
360
|
+
let devicesWithDiagrams = 0;
|
|
361
|
+
let fetchErrors = 0;
|
|
362
|
+
|
|
363
|
+
const insertAll = db.transaction(
|
|
364
|
+
(results: Array<{ deviceId: number; data: ProductPageData | null }>) => {
|
|
365
|
+
for (const { deviceId, data } of results) {
|
|
366
|
+
if (!data) {
|
|
367
|
+
fetchErrors++;
|
|
368
|
+
continue;
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
// Update device with URL and block diagram
|
|
372
|
+
updateDevice.run(
|
|
373
|
+
`https://mikrotik.com/product/${data.slug}`,
|
|
374
|
+
data.block_diagram_url,
|
|
387
375
|
deviceId,
|
|
388
|
-
r.test_type,
|
|
389
|
-
r.mode,
|
|
390
|
-
r.configuration,
|
|
391
|
-
r.packet_size,
|
|
392
|
-
r.throughput_kpps,
|
|
393
|
-
r.throughput_mbps,
|
|
394
376
|
);
|
|
395
|
-
|
|
377
|
+
|
|
378
|
+
if (data.block_diagram_url) devicesWithDiagrams++;
|
|
379
|
+
|
|
380
|
+
// Insert test results
|
|
381
|
+
const allResults = [
|
|
382
|
+
...data.ethernet_results.map((r) => ({ ...r, test_type: "ethernet" as const })),
|
|
383
|
+
...data.ipsec_results.map((r) => ({ ...r, test_type: "ipsec" as const })),
|
|
384
|
+
];
|
|
385
|
+
|
|
386
|
+
if (allResults.length > 0) devicesWithTests++;
|
|
387
|
+
|
|
388
|
+
for (const r of allResults) {
|
|
389
|
+
insertTest.run(
|
|
390
|
+
deviceId,
|
|
391
|
+
r.test_type,
|
|
392
|
+
r.mode,
|
|
393
|
+
r.configuration,
|
|
394
|
+
r.packet_size,
|
|
395
|
+
r.throughput_kpps,
|
|
396
|
+
r.throughput_mbps,
|
|
397
|
+
);
|
|
398
|
+
totalTests++;
|
|
399
|
+
}
|
|
396
400
|
}
|
|
397
|
-
}
|
|
398
|
-
},
|
|
399
|
-
);
|
|
400
|
-
|
|
401
|
-
// Fetch all products with rate limiting
|
|
402
|
-
const allResults: Array<{ deviceId: number; data: ProductPageData | null }> = [];
|
|
403
|
-
let processed = 0;
|
|
404
|
-
|
|
405
|
-
for (let i = 0; i < deviceSlugs.length; i += CONCURRENCY) {
|
|
406
|
-
const batch = deviceSlugs.slice(i, i + CONCURRENCY);
|
|
407
|
-
const batchResults = await Promise.all(
|
|
408
|
-
batch.map(async (dev) => {
|
|
409
|
-
const data = await fetchProductPage(dev.slugs);
|
|
410
|
-
return { deviceId: dev.id, data };
|
|
411
|
-
}),
|
|
401
|
+
},
|
|
412
402
|
);
|
|
413
|
-
allResults.push(...batchResults);
|
|
414
|
-
processed += batch.length;
|
|
415
403
|
|
|
416
|
-
|
|
417
|
-
|
|
404
|
+
// Fetch all products with rate limiting
|
|
405
|
+
const allResults: Array<{ deviceId: number; data: ProductPageData | null }> = [];
|
|
406
|
+
let processed = 0;
|
|
407
|
+
|
|
408
|
+
for (let i = 0; i < deviceSlugs.length; i += CONCURRENCY) {
|
|
409
|
+
const batch = deviceSlugs.slice(i, i + CONCURRENCY);
|
|
410
|
+
const batchResults = await Promise.all(
|
|
411
|
+
batch.map(async (dev) => {
|
|
412
|
+
const data = await fetchProductPage(dev.slugs);
|
|
413
|
+
return { deviceId: dev.id, data };
|
|
414
|
+
}),
|
|
415
|
+
);
|
|
416
|
+
allResults.push(...batchResults);
|
|
417
|
+
processed += batch.length;
|
|
418
|
+
|
|
419
|
+
const pct = Math.round((processed / deviceSlugs.length) * 100);
|
|
420
|
+
process.stdout.write(`\r ${processed}/${deviceSlugs.length} (${pct}%)`);
|
|
421
|
+
|
|
422
|
+
if (i + CONCURRENCY < deviceSlugs.length) {
|
|
423
|
+
await sleep(DELAY_MS);
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
console.log(""); // newline after progress
|
|
418
427
|
|
|
419
|
-
|
|
420
|
-
|
|
428
|
+
// Insert all results in one transaction
|
|
429
|
+
insertAll(allResults);
|
|
430
|
+
|
|
431
|
+
console.log(`Test results: ${totalTests} rows for ${devicesWithTests} devices`);
|
|
432
|
+
console.log(`Block diagrams: ${devicesWithDiagrams} devices`);
|
|
433
|
+
if (fetchErrors > 0) {
|
|
434
|
+
console.warn(`Fetch errors: ${fetchErrors} products`);
|
|
421
435
|
}
|
|
422
436
|
}
|
|
423
|
-
console.log(""); // newline after progress
|
|
424
|
-
|
|
425
|
-
// Insert all results in one transaction
|
|
426
|
-
insertAll(allResults);
|
|
427
437
|
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
if (fetchErrors > 0) {
|
|
431
|
-
console.warn(`Fetch errors: ${fetchErrors} products`);
|
|
438
|
+
if (import.meta.main) {
|
|
439
|
+
await main();
|
|
432
440
|
}
|