@tikoci/rosetta 0.8.5 → 0.8.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/browse.ts +5 -3
- package/src/extract-test-results.test.ts +193 -0
- package/src/extract-test-results.ts +117 -109
- package/src/query.test.ts +43 -0
- package/src/query.ts +39 -2
package/package.json
CHANGED
package/src/browse.ts
CHANGED
|
@@ -51,6 +51,7 @@ import {
|
|
|
51
51
|
searchDude,
|
|
52
52
|
searchProperties,
|
|
53
53
|
searchVideos,
|
|
54
|
+
truncateDeviceTestResultsPrefer512,
|
|
54
55
|
} from "./query.ts";
|
|
55
56
|
|
|
56
57
|
// ── ANSI utilities (zero deps) ──
|
|
@@ -785,15 +786,16 @@ function renderDeviceCard(d: DeviceResult): string {
|
|
|
785
786
|
|
|
786
787
|
// Test results (attached for exact matches)
|
|
787
788
|
if (d.test_results && d.test_results.length > 0) {
|
|
789
|
+
const truncated = truncateDeviceTestResultsPrefer512(d.test_results, 12);
|
|
788
790
|
out.push("");
|
|
789
791
|
out.push(` ${bold("Benchmarks:")} ${dim(`(${d.test_results.length} tests)`)}`);
|
|
790
|
-
for (const t of
|
|
792
|
+
for (const t of truncated.rows) {
|
|
791
793
|
const mbps = t.throughput_mbps ? `${fmt(t.throughput_mbps)} Mbps` : "";
|
|
792
794
|
const kpps = t.throughput_kpps ? `${fmt(t.throughput_kpps)} Kpps` : "";
|
|
793
795
|
out.push(` ${dim(pad(t.test_type, 9))} ${pad(t.mode, 16)} ${dim(pad(t.configuration, 28))} ${pad(`${t.packet_size}B`, 6)} ${bold(mbps)} ${dim(kpps)}`);
|
|
794
796
|
}
|
|
795
|
-
if (
|
|
796
|
-
out.push(` ${dim(`... and ${
|
|
797
|
+
if (truncated.omitted > 0) {
|
|
798
|
+
out.push(` ${dim(`... and ${truncated.omitted} more (use`)} ${cyan("tests")} ${dim("for full listing)")}`);
|
|
797
799
|
}
|
|
798
800
|
}
|
|
799
801
|
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
// Set BEFORE importing extract-test-results.ts (which transitively imports
|
|
2
|
+
// db.ts). Prevents this test file from pinning the DB singleton to a real
|
|
3
|
+
// ros-help.db path before query.test.ts can enforce its in-memory guard.
|
|
4
|
+
process.env.DB_PATH = ":memory:";
|
|
5
|
+
|
|
6
|
+
import { describe, expect, it } from "bun:test";
|
|
7
|
+
import { parseHTML } from "linkedom";
|
|
8
|
+
|
|
9
|
+
// Dynamic import so the DB_PATH assignment above is visible before db.ts loads.
|
|
10
|
+
const { parsePerformanceTable } = await import("./extract-test-results.ts");
|
|
11
|
+
|
|
12
|
+
// ── Fixture helpers ──────────────────────────────────────────────────────────
|
|
13
|
+
|
|
14
|
+
/** Build a minimal performance-table HTML matching the real MikroTik page structure. */
|
|
15
|
+
function makeEthernetTable(rows: string[]): Element {
|
|
16
|
+
const html = `
|
|
17
|
+
<table class="performance-table">
|
|
18
|
+
<thead>
|
|
19
|
+
<tr>
|
|
20
|
+
<td>RB1100Dx4</td>
|
|
21
|
+
<td>AL21400 1G all port test</td>
|
|
22
|
+
</tr>
|
|
23
|
+
<tr>
|
|
24
|
+
<td>Mode</td>
|
|
25
|
+
<td>Configuration</td>
|
|
26
|
+
<td>1518 byte</td>
|
|
27
|
+
<td></td>
|
|
28
|
+
<td>512 byte</td>
|
|
29
|
+
<td></td>
|
|
30
|
+
<td>64 byte</td>
|
|
31
|
+
<td></td>
|
|
32
|
+
</tr>
|
|
33
|
+
<tr>
|
|
34
|
+
<td></td><td></td>
|
|
35
|
+
<td>kpps</td><td>Mbps</td>
|
|
36
|
+
<td>kpps</td><td>Mbps</td>
|
|
37
|
+
<td>kpps</td><td>Mbps</td>
|
|
38
|
+
</tr>
|
|
39
|
+
</thead>
|
|
40
|
+
<tbody>
|
|
41
|
+
${rows.join("\n ")}
|
|
42
|
+
</tbody>
|
|
43
|
+
</table>`;
|
|
44
|
+
const { document } = parseHTML(`<html><body>${html}</body></html>`);
|
|
45
|
+
const table = document.querySelector("table");
|
|
46
|
+
if (!table) throw new Error("fixture build failed");
|
|
47
|
+
return table;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function makeIpsecTable(rows: string[]): Element {
|
|
51
|
+
const html = `
|
|
52
|
+
<table class="performance-table">
|
|
53
|
+
<thead>
|
|
54
|
+
<tr>
|
|
55
|
+
<td>RB1100Dx4</td>
|
|
56
|
+
<td>IPsec AES hardware acceleration</td>
|
|
57
|
+
</tr>
|
|
58
|
+
<tr>
|
|
59
|
+
<td>Mode</td>
|
|
60
|
+
<td>Configuration</td>
|
|
61
|
+
<td>1400 byte</td>
|
|
62
|
+
<td></td>
|
|
63
|
+
<td>512 byte</td>
|
|
64
|
+
<td></td>
|
|
65
|
+
<td>64 byte</td>
|
|
66
|
+
<td></td>
|
|
67
|
+
</tr>
|
|
68
|
+
<tr>
|
|
69
|
+
<td></td><td></td>
|
|
70
|
+
<td>kpps</td><td>Mbps</td>
|
|
71
|
+
<td>kpps</td><td>Mbps</td>
|
|
72
|
+
<td>kpps</td><td>Mbps</td>
|
|
73
|
+
</tr>
|
|
74
|
+
</thead>
|
|
75
|
+
<tbody>
|
|
76
|
+
${rows.join("\n ")}
|
|
77
|
+
</tbody>
|
|
78
|
+
</table>`;
|
|
79
|
+
const { document } = parseHTML(`<html><body>${html}</body></html>`);
|
|
80
|
+
const table = document.querySelector("table");
|
|
81
|
+
if (!table) throw new Error("fixture build failed");
|
|
82
|
+
return table;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// ── Tests ────────────────────────────────────────────────────────────────────
|
|
86
|
+
|
|
87
|
+
describe("parsePerformanceTable", () => {
|
|
88
|
+
describe("type detection", () => {
|
|
89
|
+
it("detects ethernet type from header", () => {
|
|
90
|
+
const table = makeEthernetTable([
|
|
91
|
+
"<tr><td>Bridging</td><td>none (fast path)</td><td>606.5</td><td>7,365.3</td><td>1,736.4</td><td>7,112.3</td><td>5,509.7</td><td>2,821.0</td></tr>",
|
|
92
|
+
]);
|
|
93
|
+
const { testType } = parsePerformanceTable(table);
|
|
94
|
+
expect(testType).toBe("ethernet");
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
it("detects ipsec type from header", () => {
|
|
98
|
+
const table = makeIpsecTable([
|
|
99
|
+
"<tr><td>Single tunnel</td><td>AES-128-CBC + SHA1</td><td>92.1</td><td>1,031.5</td><td>93.1</td><td>381.3</td><td>92.3</td><td>47.3</td></tr>",
|
|
100
|
+
]);
|
|
101
|
+
const { testType } = parsePerformanceTable(table);
|
|
102
|
+
expect(testType).toBe("ipsec");
|
|
103
|
+
});
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
describe("thousands-separator handling", () => {
|
|
107
|
+
it("correctly parses values with comma thousands separators (regression: RB1100Dx4 512-byte row)", () => {
|
|
108
|
+
// Website: Bridging, none (fast path), 512B → kpps=1,736.4, Mbps=7,112.3
|
|
109
|
+
// Before fix: parseFloat("1,736.4") → 1, parseFloat("7,112.3") → 7
|
|
110
|
+
const table = makeEthernetTable([
|
|
111
|
+
"<tr><td>Bridging</td><td>none (fast path)</td><td>606.5</td><td>7,365.3</td><td>1,736.4</td><td>7,112.3</td><td>5,509.7</td><td>2,821.0</td></tr>",
|
|
112
|
+
]);
|
|
113
|
+
const { rows } = parsePerformanceTable(table);
|
|
114
|
+
|
|
115
|
+
const row512 = rows.find((r) => r.packet_size === 512);
|
|
116
|
+
expect(row512).toBeDefined();
|
|
117
|
+
expect(row512?.throughput_kpps).toBeCloseTo(1736.4);
|
|
118
|
+
expect(row512?.throughput_mbps).toBeCloseTo(7112.3);
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
it("correctly parses 1518-byte row with large comma-formatted values", () => {
|
|
122
|
+
const table = makeEthernetTable([
|
|
123
|
+
"<tr><td>Bridging</td><td>none (fast path)</td><td>606.5</td><td>7,365.3</td><td>1,736.4</td><td>7,112.3</td><td>5,509.7</td><td>2,821.0</td></tr>",
|
|
124
|
+
]);
|
|
125
|
+
const { rows } = parsePerformanceTable(table);
|
|
126
|
+
|
|
127
|
+
const row1518 = rows.find((r) => r.packet_size === 1518);
|
|
128
|
+
expect(row1518?.throughput_kpps).toBeCloseTo(606.5);
|
|
129
|
+
expect(row1518?.throughput_mbps).toBeCloseTo(7365.3);
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
it("correctly parses 64-byte row with large comma-formatted kpps", () => {
|
|
133
|
+
const table = makeEthernetTable([
|
|
134
|
+
"<tr><td>Bridging</td><td>none (fast path)</td><td>606.5</td><td>7,365.3</td><td>1,736.4</td><td>7,112.3</td><td>5,509.7</td><td>2,821.0</td></tr>",
|
|
135
|
+
]);
|
|
136
|
+
const { rows } = parsePerformanceTable(table);
|
|
137
|
+
|
|
138
|
+
const row64 = rows.find((r) => r.packet_size === 64);
|
|
139
|
+
expect(row64?.throughput_kpps).toBeCloseTo(5509.7);
|
|
140
|
+
expect(row64?.throughput_mbps).toBeCloseTo(2821.0);
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
it("still handles values without thousands separators", () => {
|
|
144
|
+
const table = makeEthernetTable([
|
|
145
|
+
"<tr><td>Routing</td><td>25 simple queues</td><td>606.5</td><td>7365.3</td><td>933.6</td><td>3824.0</td><td>960.3</td><td>491.7</td></tr>",
|
|
146
|
+
]);
|
|
147
|
+
const { rows } = parsePerformanceTable(table);
|
|
148
|
+
|
|
149
|
+
const row512 = rows.find((r) => r.packet_size === 512);
|
|
150
|
+
expect(row512?.throughput_kpps).toBeCloseTo(933.6);
|
|
151
|
+
expect(row512?.throughput_mbps).toBeCloseTo(3824.0);
|
|
152
|
+
});
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
describe("row structure", () => {
|
|
156
|
+
it("emits one row per packet size", () => {
|
|
157
|
+
const table = makeEthernetTable([
|
|
158
|
+
"<tr><td>Bridging</td><td>none (fast path)</td><td>606.5</td><td>7,365.3</td><td>1,736.4</td><td>7,112.3</td><td>5,509.7</td><td>2,821.0</td></tr>",
|
|
159
|
+
]);
|
|
160
|
+
const { rows } = parsePerformanceTable(table);
|
|
161
|
+
expect(rows).toHaveLength(3); // 1518, 512, 64
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
it("preserves mode and configuration strings", () => {
|
|
165
|
+
const table = makeEthernetTable([
|
|
166
|
+
"<tr><td>Routing</td><td>25 ip filter rules</td><td>543.7</td><td>6,602.7</td><td>561.8</td><td>2,301.1</td><td>564.6</td><td>289.1</td></tr>",
|
|
167
|
+
]);
|
|
168
|
+
const { rows } = parsePerformanceTable(table);
|
|
169
|
+
expect(rows[0].mode).toBe("Routing");
|
|
170
|
+
expect(rows[0].configuration).toBe("25 ip filter rules");
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
it("returns null for unparseable cell values", () => {
|
|
174
|
+
const table = makeEthernetTable([
|
|
175
|
+
"<tr><td>Bridging</td><td>none (fast path)</td><td>-</td><td>-</td><td>-</td><td>-</td><td>-</td><td>-</td></tr>",
|
|
176
|
+
]);
|
|
177
|
+
const { rows } = parsePerformanceTable(table);
|
|
178
|
+
expect(rows[0].throughput_kpps).toBeNull();
|
|
179
|
+
expect(rows[0].throughput_mbps).toBeNull();
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
it("parses ipsec rows with 1400/512/64 packet sizes", () => {
|
|
183
|
+
const table = makeIpsecTable([
|
|
184
|
+
"<tr><td>Single tunnel</td><td>AES-128-CBC + SHA1</td><td>92.1</td><td>1,031.5</td><td>93.1</td><td>381.3</td><td>92.3</td><td>47.3</td></tr>",
|
|
185
|
+
]);
|
|
186
|
+
const { rows } = parsePerformanceTable(table);
|
|
187
|
+
const sizes = rows.map((r) => r.packet_size);
|
|
188
|
+
expect(sizes).toEqual([1400, 512, 64]);
|
|
189
|
+
const row1400 = rows.find((r) => r.packet_size === 1400);
|
|
190
|
+
expect(row1400?.throughput_mbps).toBeCloseTo(1031.5);
|
|
191
|
+
});
|
|
192
|
+
});
|
|
193
|
+
});
|
|
@@ -19,7 +19,6 @@
|
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
21
|
import { parseHTML } from "linkedom";
|
|
22
|
-
import { db, initDb } from "./db.ts";
|
|
23
22
|
|
|
24
23
|
// ── CLI flags ──
|
|
25
24
|
|
|
@@ -58,7 +57,7 @@ const SLUG_OVERRIDES: Record<string, string> = {
|
|
|
58
57
|
|
|
59
58
|
// ── Types ──
|
|
60
59
|
|
|
61
|
-
interface TestResultRow {
|
|
60
|
+
export interface TestResultRow {
|
|
62
61
|
mode: string;
|
|
63
62
|
configuration: string;
|
|
64
63
|
packet_size: number;
|
|
@@ -76,7 +75,7 @@ interface ProductPageData {
|
|
|
76
75
|
// ── HTML Parsing ──
|
|
77
76
|
|
|
78
77
|
/** Parse a performance-table element into test result rows. */
|
|
79
|
-
function parsePerformanceTable(table: Element): { testType: string; rows: TestResultRow[] } {
|
|
78
|
+
export function parsePerformanceTable(table: Element): { testType: string; rows: TestResultRow[] } {
|
|
80
79
|
const rows: TestResultRow[] = [];
|
|
81
80
|
|
|
82
81
|
// Header row: first <tr> in <thead> has [product_code, test_description]
|
|
@@ -124,14 +123,16 @@ function parsePerformanceTable(table: Element): { testType: string; rows: TestRe
|
|
|
124
123
|
const config = (cells[1].textContent || "").trim();
|
|
125
124
|
|
|
126
125
|
// Each packet size has 2 columns: kpps, Mbps
|
|
126
|
+
// Strip thousands-separator commas before parsing (e.g. "7,112.3" → 7112.3)
|
|
127
|
+
const parseNum = (s: string) => Number.parseFloat(s.replace(/,/g, "").trim());
|
|
127
128
|
for (let i = 0; i < packetSizes.length; i++) {
|
|
128
129
|
const kppsIdx = 2 + i * 2;
|
|
129
130
|
const mbpsIdx = 3 + i * 2;
|
|
130
131
|
if (kppsIdx >= cells.length) break;
|
|
131
132
|
|
|
132
|
-
const kpps =
|
|
133
|
+
const kpps = parseNum((cells[kppsIdx].textContent || "").trim());
|
|
133
134
|
const mbps = mbpsIdx < cells.length
|
|
134
|
-
?
|
|
135
|
+
? parseNum((cells[mbpsIdx].textContent || "").trim())
|
|
135
136
|
: null;
|
|
136
137
|
|
|
137
138
|
rows.push({
|
|
@@ -309,124 +310,131 @@ function sleep(ms: number): Promise<void> {
|
|
|
309
310
|
}
|
|
310
311
|
|
|
311
312
|
// ── Main ──
|
|
313
|
+
async function main(): Promise<void> {
|
|
314
|
+
const { db, initDb } = await import("./db.ts");
|
|
312
315
|
|
|
313
|
-
initDb();
|
|
316
|
+
initDb();
|
|
314
317
|
|
|
315
|
-
// Get all devices from DB
|
|
316
|
-
const devices = db.prepare("SELECT id, product_name, product_code FROM devices ORDER BY product_name").all() as Array<{
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
}>;
|
|
318
|
+
// Get all devices from DB
|
|
319
|
+
const devices = db.prepare("SELECT id, product_name, product_code FROM devices ORDER BY product_name").all() as Array<{
|
|
320
|
+
id: number;
|
|
321
|
+
product_name: string;
|
|
322
|
+
product_code: string | null;
|
|
323
|
+
}>;
|
|
321
324
|
|
|
322
|
-
if (devices.length === 0) {
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
}
|
|
326
|
-
|
|
327
|
-
console.log(`Found ${devices.length} devices in database`);
|
|
328
|
-
|
|
329
|
-
// Fetch sitemap once to validate slugs and prioritize correct candidates
|
|
330
|
-
console.log("Loading product sitemap...");
|
|
331
|
-
const sitemapSlugs = await fetchSitemap();
|
|
332
|
-
|
|
333
|
-
// Build device → candidate slugs mapping
|
|
334
|
-
const deviceSlugs: Array<{ id: number; name: string; slugs: string[] }> = [];
|
|
335
|
-
for (const dev of devices) {
|
|
336
|
-
const slugs = buildSlugCandidates(dev.product_name, dev.product_code, sitemapSlugs);
|
|
337
|
-
deviceSlugs.push({ id: dev.id, name: dev.product_name, slugs });
|
|
338
|
-
}
|
|
339
|
-
|
|
340
|
-
// Idempotent: clear existing test results
|
|
341
|
-
db.run("DELETE FROM device_test_results");
|
|
342
|
-
|
|
343
|
-
// Prepare statements
|
|
344
|
-
const insertTest = db.prepare(`INSERT OR IGNORE INTO device_test_results (
|
|
345
|
-
device_id, test_type, mode, configuration, packet_size,
|
|
346
|
-
throughput_kpps, throughput_mbps
|
|
347
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?)`);
|
|
348
|
-
|
|
349
|
-
const updateDevice = db.prepare(`UPDATE devices
|
|
350
|
-
SET product_url = ?, block_diagram_url = ?
|
|
351
|
-
WHERE id = ?`);
|
|
352
|
-
|
|
353
|
-
console.log(`Fetching ${deviceSlugs.length} product pages (concurrency=${CONCURRENCY}, delay=${DELAY_MS}ms)...`);
|
|
354
|
-
|
|
355
|
-
let totalTests = 0;
|
|
356
|
-
let devicesWithTests = 0;
|
|
357
|
-
let devicesWithDiagrams = 0;
|
|
358
|
-
let fetchErrors = 0;
|
|
359
|
-
|
|
360
|
-
const insertAll = db.transaction(
|
|
361
|
-
(results: Array<{ deviceId: number; data: ProductPageData | null }>) => {
|
|
362
|
-
for (const { deviceId, data } of results) {
|
|
363
|
-
if (!data) {
|
|
364
|
-
fetchErrors++;
|
|
365
|
-
continue;
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
// Update device with URL and block diagram
|
|
369
|
-
updateDevice.run(
|
|
370
|
-
`https://mikrotik.com/product/${data.slug}`,
|
|
371
|
-
data.block_diagram_url,
|
|
372
|
-
deviceId,
|
|
373
|
-
);
|
|
325
|
+
if (devices.length === 0) {
|
|
326
|
+
console.error("No devices in database. Run extract-devices.ts first.");
|
|
327
|
+
process.exit(1);
|
|
328
|
+
}
|
|
374
329
|
|
|
375
|
-
|
|
330
|
+
console.log(`Found ${devices.length} devices in database`);
|
|
376
331
|
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
...data.ipsec_results.map((r) => ({ ...r, test_type: "ipsec" as const })),
|
|
381
|
-
];
|
|
332
|
+
// Fetch sitemap once to validate slugs and prioritize correct candidates
|
|
333
|
+
console.log("Loading product sitemap...");
|
|
334
|
+
const sitemapSlugs = await fetchSitemap();
|
|
382
335
|
|
|
383
|
-
|
|
336
|
+
// Build device -> candidate slugs mapping
|
|
337
|
+
const deviceSlugs: Array<{ id: number; name: string; slugs: string[] }> = [];
|
|
338
|
+
for (const dev of devices) {
|
|
339
|
+
const slugs = buildSlugCandidates(dev.product_name, dev.product_code, sitemapSlugs);
|
|
340
|
+
deviceSlugs.push({ id: dev.id, name: dev.product_name, slugs });
|
|
341
|
+
}
|
|
384
342
|
|
|
385
|
-
|
|
386
|
-
|
|
343
|
+
// Idempotent: clear existing test results
|
|
344
|
+
db.run("DELETE FROM device_test_results");
|
|
345
|
+
|
|
346
|
+
// Prepare statements
|
|
347
|
+
const insertTest = db.prepare(`INSERT OR IGNORE INTO device_test_results (
|
|
348
|
+
device_id, test_type, mode, configuration, packet_size,
|
|
349
|
+
throughput_kpps, throughput_mbps
|
|
350
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?)`);
|
|
351
|
+
|
|
352
|
+
const updateDevice = db.prepare(`UPDATE devices
|
|
353
|
+
SET product_url = ?, block_diagram_url = ?
|
|
354
|
+
WHERE id = ?`);
|
|
355
|
+
|
|
356
|
+
console.log(`Fetching ${deviceSlugs.length} product pages (concurrency=${CONCURRENCY}, delay=${DELAY_MS}ms)...`);
|
|
357
|
+
|
|
358
|
+
let totalTests = 0;
|
|
359
|
+
let devicesWithTests = 0;
|
|
360
|
+
let devicesWithDiagrams = 0;
|
|
361
|
+
let fetchErrors = 0;
|
|
362
|
+
|
|
363
|
+
const insertAll = db.transaction(
|
|
364
|
+
(results: Array<{ deviceId: number; data: ProductPageData | null }>) => {
|
|
365
|
+
for (const { deviceId, data } of results) {
|
|
366
|
+
if (!data) {
|
|
367
|
+
fetchErrors++;
|
|
368
|
+
continue;
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
// Update device with URL and block diagram
|
|
372
|
+
updateDevice.run(
|
|
373
|
+
`https://mikrotik.com/product/${data.slug}`,
|
|
374
|
+
data.block_diagram_url,
|
|
387
375
|
deviceId,
|
|
388
|
-
r.test_type,
|
|
389
|
-
r.mode,
|
|
390
|
-
r.configuration,
|
|
391
|
-
r.packet_size,
|
|
392
|
-
r.throughput_kpps,
|
|
393
|
-
r.throughput_mbps,
|
|
394
376
|
);
|
|
395
|
-
|
|
377
|
+
|
|
378
|
+
if (data.block_diagram_url) devicesWithDiagrams++;
|
|
379
|
+
|
|
380
|
+
// Insert test results
|
|
381
|
+
const allResults = [
|
|
382
|
+
...data.ethernet_results.map((r) => ({ ...r, test_type: "ethernet" as const })),
|
|
383
|
+
...data.ipsec_results.map((r) => ({ ...r, test_type: "ipsec" as const })),
|
|
384
|
+
];
|
|
385
|
+
|
|
386
|
+
if (allResults.length > 0) devicesWithTests++;
|
|
387
|
+
|
|
388
|
+
for (const r of allResults) {
|
|
389
|
+
insertTest.run(
|
|
390
|
+
deviceId,
|
|
391
|
+
r.test_type,
|
|
392
|
+
r.mode,
|
|
393
|
+
r.configuration,
|
|
394
|
+
r.packet_size,
|
|
395
|
+
r.throughput_kpps,
|
|
396
|
+
r.throughput_mbps,
|
|
397
|
+
);
|
|
398
|
+
totalTests++;
|
|
399
|
+
}
|
|
396
400
|
}
|
|
397
|
-
}
|
|
398
|
-
},
|
|
399
|
-
);
|
|
400
|
-
|
|
401
|
-
// Fetch all products with rate limiting
|
|
402
|
-
const allResults: Array<{ deviceId: number; data: ProductPageData | null }> = [];
|
|
403
|
-
let processed = 0;
|
|
404
|
-
|
|
405
|
-
for (let i = 0; i < deviceSlugs.length; i += CONCURRENCY) {
|
|
406
|
-
const batch = deviceSlugs.slice(i, i + CONCURRENCY);
|
|
407
|
-
const batchResults = await Promise.all(
|
|
408
|
-
batch.map(async (dev) => {
|
|
409
|
-
const data = await fetchProductPage(dev.slugs);
|
|
410
|
-
return { deviceId: dev.id, data };
|
|
411
|
-
}),
|
|
401
|
+
},
|
|
412
402
|
);
|
|
413
|
-
allResults.push(...batchResults);
|
|
414
|
-
processed += batch.length;
|
|
415
403
|
|
|
416
|
-
|
|
417
|
-
|
|
404
|
+
// Fetch all products with rate limiting
|
|
405
|
+
const allResults: Array<{ deviceId: number; data: ProductPageData | null }> = [];
|
|
406
|
+
let processed = 0;
|
|
407
|
+
|
|
408
|
+
for (let i = 0; i < deviceSlugs.length; i += CONCURRENCY) {
|
|
409
|
+
const batch = deviceSlugs.slice(i, i + CONCURRENCY);
|
|
410
|
+
const batchResults = await Promise.all(
|
|
411
|
+
batch.map(async (dev) => {
|
|
412
|
+
const data = await fetchProductPage(dev.slugs);
|
|
413
|
+
return { deviceId: dev.id, data };
|
|
414
|
+
}),
|
|
415
|
+
);
|
|
416
|
+
allResults.push(...batchResults);
|
|
417
|
+
processed += batch.length;
|
|
418
|
+
|
|
419
|
+
const pct = Math.round((processed / deviceSlugs.length) * 100);
|
|
420
|
+
process.stdout.write(`\r ${processed}/${deviceSlugs.length} (${pct}%)`);
|
|
421
|
+
|
|
422
|
+
if (i + CONCURRENCY < deviceSlugs.length) {
|
|
423
|
+
await sleep(DELAY_MS);
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
console.log(""); // newline after progress
|
|
418
427
|
|
|
419
|
-
|
|
420
|
-
|
|
428
|
+
// Insert all results in one transaction
|
|
429
|
+
insertAll(allResults);
|
|
430
|
+
|
|
431
|
+
console.log(`Test results: ${totalTests} rows for ${devicesWithTests} devices`);
|
|
432
|
+
console.log(`Block diagrams: ${devicesWithDiagrams} devices`);
|
|
433
|
+
if (fetchErrors > 0) {
|
|
434
|
+
console.warn(`Fetch errors: ${fetchErrors} products`);
|
|
421
435
|
}
|
|
422
436
|
}
|
|
423
|
-
console.log(""); // newline after progress
|
|
424
|
-
|
|
425
|
-
// Insert all results in one transaction
|
|
426
|
-
insertAll(allResults);
|
|
427
437
|
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
if (fetchErrors > 0) {
|
|
431
|
-
console.warn(`Fetch errors: ${fetchErrors} products`);
|
|
438
|
+
if (import.meta.main) {
|
|
439
|
+
await main();
|
|
432
440
|
}
|
package/src/query.test.ts
CHANGED
|
@@ -43,6 +43,7 @@ const {
|
|
|
43
43
|
searchDeviceTests,
|
|
44
44
|
getTestResultMeta,
|
|
45
45
|
normalizeDeviceQuery,
|
|
46
|
+
truncateDeviceTestResultsPrefer512,
|
|
46
47
|
searchVideos,
|
|
47
48
|
searchDude,
|
|
48
49
|
getDudePage,
|
|
@@ -1426,6 +1427,48 @@ describe("dataset CSV exports", () => {
|
|
|
1426
1427
|
});
|
|
1427
1428
|
});
|
|
1428
1429
|
|
|
1430
|
+
describe("truncateDeviceTestResultsPrefer512", () => {
|
|
1431
|
+
test("keeps all 512B rows when truncating", () => {
|
|
1432
|
+
const rows = [
|
|
1433
|
+
{ packet_size: 1518, id: "a" },
|
|
1434
|
+
{ packet_size: 512, id: "b" },
|
|
1435
|
+
{ packet_size: 64, id: "c" },
|
|
1436
|
+
{ packet_size: 512, id: "d" },
|
|
1437
|
+
];
|
|
1438
|
+
const res = truncateDeviceTestResultsPrefer512(rows, 2);
|
|
1439
|
+
|
|
1440
|
+
expect(res.rows.map((r) => r.id)).toEqual(["b", "d"]);
|
|
1441
|
+
expect(res.omitted).toBe(2);
|
|
1442
|
+
});
|
|
1443
|
+
|
|
1444
|
+
test("fills remaining slots with non-512 rows", () => {
|
|
1445
|
+
const rows = [
|
|
1446
|
+
{ packet_size: 1518, id: "a" },
|
|
1447
|
+
{ packet_size: 512, id: "b" },
|
|
1448
|
+
{ packet_size: 64, id: "c" },
|
|
1449
|
+
{ packet_size: 512, id: "d" },
|
|
1450
|
+
];
|
|
1451
|
+
const res = truncateDeviceTestResultsPrefer512(rows, 3);
|
|
1452
|
+
|
|
1453
|
+
expect(res.rows.map((r) => r.id)).toEqual(["b", "d", "a"]);
|
|
1454
|
+
expect(res.omitted).toBe(1);
|
|
1455
|
+
});
|
|
1456
|
+
|
|
1457
|
+
test("returns all 512B rows even if they exceed maxRows", () => {
|
|
1458
|
+
const rows = [
|
|
1459
|
+
{ packet_size: 512, id: "a" },
|
|
1460
|
+
{ packet_size: 512, id: "b" },
|
|
1461
|
+
{ packet_size: 1518, id: "c" },
|
|
1462
|
+
{ packet_size: 512, id: "d" },
|
|
1463
|
+
{ packet_size: 64, id: "e" },
|
|
1464
|
+
];
|
|
1465
|
+
const res = truncateDeviceTestResultsPrefer512(rows, 2);
|
|
1466
|
+
|
|
1467
|
+
expect(res.rows.map((r) => r.id)).toEqual(["a", "b", "d"]);
|
|
1468
|
+
expect(res.omitted).toBe(2);
|
|
1469
|
+
});
|
|
1470
|
+
});
|
|
1471
|
+
|
|
1429
1472
|
describe("getTestResultMeta", () => {
|
|
1430
1473
|
test("returns distinct values", () => {
|
|
1431
1474
|
const meta = getTestResultMeta();
|
package/src/query.ts
CHANGED
|
@@ -298,8 +298,8 @@ export function searchAll(query: string, limit = DEFAULT_LIMIT): SearchAllRespon
|
|
|
298
298
|
path: node.path,
|
|
299
299
|
type: node.type,
|
|
300
300
|
description: node.description,
|
|
301
|
-
...(node.page_id
|
|
302
|
-
? { linked_page: { id: node.page_id, title: node.page_title
|
|
301
|
+
...(node.page_id && node.page_title && node.page_url
|
|
302
|
+
? { linked_page: { id: node.page_id, title: node.page_title, url: node.page_url } }
|
|
303
303
|
: {}),
|
|
304
304
|
};
|
|
305
305
|
}
|
|
@@ -1330,6 +1330,43 @@ export type DeviceTestResult = {
|
|
|
1330
1330
|
throughput_mbps: number | null;
|
|
1331
1331
|
};
|
|
1332
1332
|
|
|
1333
|
+
/**
|
|
1334
|
+
* Truncate benchmark rows for display while always keeping all 512-byte rows.
|
|
1335
|
+
*
|
|
1336
|
+
* Used by both TUI and MCP-facing adapters when they need a compact view.
|
|
1337
|
+
* If the 512B bucket itself exceeds maxRows, return all 512B rows anyway.
|
|
1338
|
+
*/
|
|
1339
|
+
export function truncateDeviceTestResultsPrefer512<T extends { packet_size: number }>(
|
|
1340
|
+
rows: T[],
|
|
1341
|
+
maxRows: number,
|
|
1342
|
+
): { rows: T[]; omitted: number } {
|
|
1343
|
+
if (rows.length === 0) return { rows: [], omitted: 0 };
|
|
1344
|
+
|
|
1345
|
+
const safeMax = Math.max(0, Math.floor(maxRows));
|
|
1346
|
+
const rows512 = rows.filter((r) => r.packet_size === 512);
|
|
1347
|
+
const non512 = rows.filter((r) => r.packet_size !== 512);
|
|
1348
|
+
|
|
1349
|
+
if (safeMax === 0) {
|
|
1350
|
+
return { rows: rows512, omitted: rows.length - rows512.length };
|
|
1351
|
+
}
|
|
1352
|
+
|
|
1353
|
+
if (rows.length <= safeMax) {
|
|
1354
|
+
return { rows, omitted: 0 };
|
|
1355
|
+
}
|
|
1356
|
+
|
|
1357
|
+
if (rows512.length === 0) {
|
|
1358
|
+
return { rows: rows.slice(0, safeMax), omitted: rows.length - safeMax };
|
|
1359
|
+
}
|
|
1360
|
+
|
|
1361
|
+
if (rows512.length >= safeMax) {
|
|
1362
|
+
return { rows: rows512, omitted: rows.length - rows512.length };
|
|
1363
|
+
}
|
|
1364
|
+
|
|
1365
|
+
const takeNon512 = safeMax - rows512.length;
|
|
1366
|
+
const selected = [...rows512, ...non512.slice(0, takeNon512)];
|
|
1367
|
+
return { rows: selected, omitted: rows.length - selected.length };
|
|
1368
|
+
}
|
|
1369
|
+
|
|
1333
1370
|
/** Map Unicode superscript digits to ASCII equivalents for product name matching.
|
|
1334
1371
|
* MikroTik uses ² and ³ in product names (hAP ax³, hAP ac²), but users type ASCII. */
|
|
1335
1372
|
const SUPERSCRIPT_TO_ASCII: [string, string][] = [
|