@tikoci/rosetta 0.6.3 → 0.6.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/browse.ts +4 -0
- package/src/canonicalize.test.ts +520 -0
- package/src/canonicalize.ts +629 -0
- package/src/extract-changelogs.test.ts +24 -0
- package/src/extract-changelogs.ts +17 -4
- package/src/extract-html.test.ts +30 -6
- package/src/extract-html.ts +78 -2
- package/src/mcp.ts +2 -0
- package/src/query.test.ts +18 -0
- package/src/query.ts +40 -0
|
@@ -0,0 +1,629 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RouterOS CLI path canonicalizer.
|
|
3
|
+
*
|
|
4
|
+
* Maps any CLI-ish input form to canonical `{ path, verb, args }` tuples that
|
|
5
|
+
* rosetta can `WHERE path = ?` on. Handles:
|
|
6
|
+
*
|
|
7
|
+
* - Absolute paths: `/ip/address/set`, `/ip address set`
|
|
8
|
+
* - Relative paths with cwd: `set` with cwd `/ip/address`
|
|
9
|
+
* - Mixed slash/space: `/ip firewall/filter/add`
|
|
10
|
+
* - `..` navigation: `../route print` from `/ip/address`
|
|
11
|
+
* - `[...]` subshells (nested commands inheriting outer path context)
|
|
12
|
+
* - `{ }` blocks (path persists from the prefix before the block)
|
|
13
|
+
* - `;` and newline command separators
|
|
14
|
+
* - Missing leading slash tolerance: `ip address print`
|
|
15
|
+
* - Slash-only shortcut: `/` = root
|
|
16
|
+
*
|
|
17
|
+
* This module is intentionally pure — no DB, no I/O. It knows nothing about
|
|
18
|
+
* REST API mapping. Consumers use the canonical paths for DB lookups.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
// ---------------------------------------------------------------------------
|
|
22
|
+
// Public types
|
|
23
|
+
// ---------------------------------------------------------------------------
|
|
24
|
+
|
|
25
|
+
export interface CanonicalCommand {
|
|
26
|
+
/** Absolute dir path, e.g. '/ip/address'. Always starts with '/'. */
|
|
27
|
+
path: string;
|
|
28
|
+
/** The command verb, e.g. 'set', 'print', 'add'. Empty string for bare navigation. */
|
|
29
|
+
verb: string;
|
|
30
|
+
/** Raw argument tokens after the verb (unparsed key=value pairs, unnamed params, etc.) */
|
|
31
|
+
args: string[];
|
|
32
|
+
/** For subshell commands extracted from [...], this is true */
|
|
33
|
+
subshell?: boolean;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export interface ParseResult {
|
|
37
|
+
/** All commands extracted from the input, in order */
|
|
38
|
+
commands: CanonicalCommand[];
|
|
39
|
+
/** The cwd after executing all navigation (useful for interactive sessions) */
|
|
40
|
+
finalPath: string;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// ---------------------------------------------------------------------------
|
|
44
|
+
// Known RouterOS general commands (verbs).
|
|
45
|
+
// These appear at every menu level. Used to distinguish "is this token a
|
|
46
|
+
// path segment or a command verb?"
|
|
47
|
+
// ---------------------------------------------------------------------------
|
|
48
|
+
|
|
49
|
+
const GENERAL_COMMANDS = new Set([
|
|
50
|
+
'add', 'comment', 'disable', 'edit', 'enable', 'export', 'find',
|
|
51
|
+
'get', 'move', 'print', 'remove', 'reset', 'set',
|
|
52
|
+
]);
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Commands that are clearly verbs even though they only appear at certain
|
|
56
|
+
* menu levels. We keep this small — the heuristic is "if in doubt, treat
|
|
57
|
+
* as path segment" because the DB lookup will resolve ambiguity.
|
|
58
|
+
*/
|
|
59
|
+
const EXTRA_VERBS = new Set([
|
|
60
|
+
'monitor', 'monitor-traffic', 'scan', 'run', 'start', 'stop',
|
|
61
|
+
'flush', 'release', 'renew', 'upgrade', 'downgrade', 'reboot',
|
|
62
|
+
'shutdown', 'check-for-updates',
|
|
63
|
+
]);
|
|
64
|
+
|
|
65
|
+
function isKnownVerb(token: string): boolean {
|
|
66
|
+
return GENERAL_COMMANDS.has(token) || EXTRA_VERBS.has(token);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// ---------------------------------------------------------------------------
|
|
70
|
+
// Tokenizer — splits a RouterOS command line into meaningful tokens
|
|
71
|
+
// ---------------------------------------------------------------------------
|
|
72
|
+
|
|
73
|
+
/** Token types emitted by the tokenizer */
|
|
74
|
+
enum Tok {
|
|
75
|
+
Word, // any identifier / path-segment / value
|
|
76
|
+
Slash, // /
|
|
77
|
+
Semicolon, // ;
|
|
78
|
+
LBracket, // [
|
|
79
|
+
RBracket, // ]
|
|
80
|
+
LBrace, // {
|
|
81
|
+
RBrace, // }
|
|
82
|
+
Equals, // = (attached to preceding word)
|
|
83
|
+
Newline, // \n
|
|
84
|
+
Colon, // : (ICE command prefix)
|
|
85
|
+
DotDot, // ..
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
interface Token {
|
|
89
|
+
type: Tok;
|
|
90
|
+
value: string;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Tokenize a RouterOS CLI input string.
|
|
95
|
+
* This is intentionally loose — we don't need a full parser, just enough
|
|
96
|
+
* structure to extract paths and verbs.
|
|
97
|
+
*/
|
|
98
|
+
function tokenize(input: string): Token[] {
|
|
99
|
+
const tokens: Token[] = [];
|
|
100
|
+
let i = 0;
|
|
101
|
+
const len = input.length;
|
|
102
|
+
|
|
103
|
+
while (i < len) {
|
|
104
|
+
const ch = input[i];
|
|
105
|
+
|
|
106
|
+
// Skip whitespace (except newlines)
|
|
107
|
+
if (ch === ' ' || ch === '\t' || ch === '\r') { i++; continue; }
|
|
108
|
+
|
|
109
|
+
// Newline
|
|
110
|
+
if (ch === '\n') { tokens.push({ type: Tok.Newline, value: '\n' }); i++; continue; }
|
|
111
|
+
|
|
112
|
+
// Line continuation
|
|
113
|
+
if (ch === '\\' && i + 1 < len && input[i + 1] === '\n') { i += 2; continue; }
|
|
114
|
+
|
|
115
|
+
// Comments
|
|
116
|
+
if (ch === '#') {
|
|
117
|
+
while (i < len && input[i] !== '\n') i++;
|
|
118
|
+
continue;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// Structural tokens
|
|
122
|
+
if (ch === '[') { tokens.push({ type: Tok.LBracket, value: '[' }); i++; continue; }
|
|
123
|
+
if (ch === ']') { tokens.push({ type: Tok.RBracket, value: ']' }); i++; continue; }
|
|
124
|
+
if (ch === '{') { tokens.push({ type: Tok.LBrace, value: '{' }); i++; continue; }
|
|
125
|
+
if (ch === '}') { tokens.push({ type: Tok.RBrace, value: '}' }); i++; continue; }
|
|
126
|
+
if (ch === ';') { tokens.push({ type: Tok.Semicolon, value: ';' }); i++; continue; }
|
|
127
|
+
|
|
128
|
+
// Slash (path separator or root)
|
|
129
|
+
if (ch === '/') { tokens.push({ type: Tok.Slash, value: '/' }); i++; continue; }
|
|
130
|
+
|
|
131
|
+
// .. (parent navigation)
|
|
132
|
+
if (ch === '.' && i + 1 < len && input[i + 1] === '.') {
|
|
133
|
+
tokens.push({ type: Tok.DotDot, value: '..' });
|
|
134
|
+
i += 2;
|
|
135
|
+
continue;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
// : prefix for ICE commands
|
|
139
|
+
if (ch === ':') { tokens.push({ type: Tok.Colon, value: ':' }); i++; continue; }
|
|
140
|
+
|
|
141
|
+
// Quoted string — consume as a single word token
|
|
142
|
+
if (ch === '"') {
|
|
143
|
+
let str = '';
|
|
144
|
+
i++; // skip opening quote
|
|
145
|
+
while (i < len && input[i] !== '"') {
|
|
146
|
+
if (input[i] === '\\' && i + 1 < len) {
|
|
147
|
+
str += input[i] + input[i + 1];
|
|
148
|
+
i += 2;
|
|
149
|
+
} else {
|
|
150
|
+
str += input[i];
|
|
151
|
+
i++;
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
if (i < len) i++; // skip closing quote
|
|
155
|
+
tokens.push({ type: Tok.Word, value: `"${str}"` });
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// Word — identifier, number, IP, key=value, etc.
|
|
160
|
+
// A word can contain letters, digits, hyphens, underscores, dots, colons (for IPs),
|
|
161
|
+
// asterisks (for IDs like *1), plus, @, etc.
|
|
162
|
+
// It ends at whitespace, structural chars, or a slash.
|
|
163
|
+
let word = '';
|
|
164
|
+
while (i < len) {
|
|
165
|
+
const c = input[i];
|
|
166
|
+
if (c === ' ' || c === '\t' || c === '\r' || c === '\n') break;
|
|
167
|
+
if (c === '[' || c === ']' || c === '{' || c === '}' || c === ';') break;
|
|
168
|
+
if (c === '#') break;
|
|
169
|
+
// Slash breaks a word UNLESS it's inside a CIDR or we're at a slash
|
|
170
|
+
// that starts a new path segment mid-word (like firewall/filter)
|
|
171
|
+
if (c === '/') {
|
|
172
|
+
// If the previous char was a digit and next is a digit, it's a CIDR mask
|
|
173
|
+
if (word.length > 0 && /\d$/.test(word) && i + 1 < len && /\d/.test(input[i + 1])) {
|
|
174
|
+
word += c;
|
|
175
|
+
i++;
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
break;
|
|
179
|
+
}
|
|
180
|
+
// Equals sign — if it's part of key=value, include the key part
|
|
181
|
+
if (c === '=') {
|
|
182
|
+
word += c;
|
|
183
|
+
i++;
|
|
184
|
+
// Consume the value after = as part of this word.
|
|
185
|
+
// If the value starts with [ it's a subshell — emit the key= as a word,
|
|
186
|
+
// then let the structural token handler deal with [ and ].
|
|
187
|
+
if (i < len && input[i] === '[') {
|
|
188
|
+
tokens.push({ type: Tok.Word, value: word });
|
|
189
|
+
word = '';
|
|
190
|
+
break;
|
|
191
|
+
}
|
|
192
|
+
// Otherwise consume the plain value up to the next separator
|
|
193
|
+
while (i < len) {
|
|
194
|
+
const vc = input[i];
|
|
195
|
+
if (vc === '"') {
|
|
196
|
+
// consume quoted string
|
|
197
|
+
word += vc;
|
|
198
|
+
i++;
|
|
199
|
+
while (i < len && input[i] !== '"') {
|
|
200
|
+
if (input[i] === '\\' && i + 1 < len) { word += input[i] + input[i + 1]; i += 2; }
|
|
201
|
+
else { word += input[i]; i++; }
|
|
202
|
+
}
|
|
203
|
+
if (i < len) { word += input[i]; i++; }
|
|
204
|
+
continue;
|
|
205
|
+
}
|
|
206
|
+
if (vc === ' ' || vc === '\t' || vc === '\r' || vc === '\n' || vc === ';' || vc === '}' || vc === '{' || vc === '[' || vc === ']') break;
|
|
207
|
+
word += vc;
|
|
208
|
+
i++;
|
|
209
|
+
}
|
|
210
|
+
tokens.push({ type: Tok.Word, value: word });
|
|
211
|
+
word = '';
|
|
212
|
+
break;
|
|
213
|
+
}
|
|
214
|
+
word += c;
|
|
215
|
+
i++;
|
|
216
|
+
}
|
|
217
|
+
if (word.length > 0) {
|
|
218
|
+
tokens.push({ type: Tok.Word, value: word });
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
return tokens;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
// ---------------------------------------------------------------------------
|
|
226
|
+
// Path normalization helpers
|
|
227
|
+
// ---------------------------------------------------------------------------
|
|
228
|
+
|
|
229
|
+
/** Normalize a path: ensure leading slash, no trailing slash, collapse double slashes */
|
|
230
|
+
function normalizePath(p: string): string {
|
|
231
|
+
if (!p || p === '/') return '/';
|
|
232
|
+
let result = p.startsWith('/') ? p : `/${p}`;
|
|
233
|
+
// Collapse double slashes
|
|
234
|
+
result = result.replace(/\/+/g, '/');
|
|
235
|
+
// Remove trailing slash
|
|
236
|
+
if (result.length > 1 && result.endsWith('/')) {
|
|
237
|
+
result = result.slice(0, -1);
|
|
238
|
+
}
|
|
239
|
+
return result;
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/** Resolve `..` segments in a path */
|
|
243
|
+
function resolveParent(base: string): string {
|
|
244
|
+
const parts = base.split('/').filter(Boolean);
|
|
245
|
+
parts.pop();
|
|
246
|
+
return parts.length === 0 ? '/' : `/${parts.join('/')}`;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/** Join two path segments */
|
|
250
|
+
function joinPath(base: string, segment: string): string {
|
|
251
|
+
if (base === '/') return `/${segment}`;
|
|
252
|
+
return `${base}/${segment}`;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
// ---------------------------------------------------------------------------
|
|
256
|
+
// Core parser — extracts commands from token stream
|
|
257
|
+
// ---------------------------------------------------------------------------
|
|
258
|
+
|
|
259
|
+
interface ParseContext {
|
|
260
|
+
tokens: Token[];
|
|
261
|
+
pos: number;
|
|
262
|
+
cwd: string;
|
|
263
|
+
commands: CanonicalCommand[];
|
|
264
|
+
subshell: boolean;
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/**
|
|
268
|
+
* Parse a sequence of tokens into commands, starting from a given cwd.
|
|
269
|
+
* Returns the final cwd after all commands are processed.
|
|
270
|
+
*
|
|
271
|
+
* This handles the RouterOS scoping rules:
|
|
272
|
+
* - `/` at the start of a command resets to root
|
|
273
|
+
* - Path segments before a verb navigate the cwd for that command
|
|
274
|
+
* - `[...]` creates a subshell that inherits the current outer path
|
|
275
|
+
* - `{ }` blocks inherit the path from the prefix before the block
|
|
276
|
+
* - `;` and newline separate commands but DON'T reset path in scripts
|
|
277
|
+
*/
|
|
278
|
+
function parseCommands(ctx: ParseContext, stopAt?: Tok): string {
|
|
279
|
+
let currentPath = ctx.cwd;
|
|
280
|
+
let commandStartPath = currentPath;
|
|
281
|
+
|
|
282
|
+
// Accumulates path segments for the current command line
|
|
283
|
+
let pathSegments: string[] = [];
|
|
284
|
+
let isAbsolute = false;
|
|
285
|
+
let verb = '';
|
|
286
|
+
let args: string[] = [];
|
|
287
|
+
let inCommand = false; // true once we've seen at least one token in this command
|
|
288
|
+
let inIce = false; // true if we're in an ICE command (: prefix)
|
|
289
|
+
|
|
290
|
+
function flushCommand() {
|
|
291
|
+
// Build the resolved path from segments
|
|
292
|
+
let resolvedPath = commandStartPath;
|
|
293
|
+
if (isAbsolute) {
|
|
294
|
+
resolvedPath = '/';
|
|
295
|
+
}
|
|
296
|
+
for (const seg of pathSegments) {
|
|
297
|
+
if (seg === '..') {
|
|
298
|
+
resolvedPath = resolveParent(resolvedPath);
|
|
299
|
+
} else {
|
|
300
|
+
resolvedPath = joinPath(resolvedPath, seg);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
resolvedPath = normalizePath(resolvedPath);
|
|
304
|
+
|
|
305
|
+
if (verb || args.length > 0 || pathSegments.length > 0) {
|
|
306
|
+
// If no explicit verb but we have path segments, the last segment might be a verb
|
|
307
|
+
if (!verb && pathSegments.length > 0) {
|
|
308
|
+
const lastSeg = pathSegments[pathSegments.length - 1];
|
|
309
|
+
if (lastSeg !== '..' && isKnownVerb(lastSeg)) {
|
|
310
|
+
verb = lastSeg;
|
|
311
|
+
// Remove the verb from the resolved path
|
|
312
|
+
resolvedPath = resolveParent(resolvedPath);
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
if (verb || args.length > 0) {
|
|
317
|
+
ctx.commands.push({
|
|
318
|
+
path: resolvedPath,
|
|
319
|
+
verb,
|
|
320
|
+
args: [...args],
|
|
321
|
+
...(ctx.subshell ? { subshell: true } : {}),
|
|
322
|
+
});
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
// After a command with a verb, path does NOT change for next command in same scope
|
|
326
|
+
// Only bare navigation (path without verb) changes the cwd
|
|
327
|
+
if (!verb && pathSegments.length > 0) {
|
|
328
|
+
// Pure navigation — update cwd
|
|
329
|
+
currentPath = resolvedPath;
|
|
330
|
+
}
|
|
331
|
+
// If we had a verb + path prefix (like `/ip/address print`), the path prefix
|
|
332
|
+
// also updates cwd for the NEXT command (RouterOS behavior: path changes on cmd line)
|
|
333
|
+
if (verb && (isAbsolute || pathSegments.length > 1)) {
|
|
334
|
+
// The path (without the verb) becomes the new cwd
|
|
335
|
+
currentPath = resolvedPath;
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
// Reset for next command
|
|
340
|
+
pathSegments = [];
|
|
341
|
+
isAbsolute = false;
|
|
342
|
+
verb = '';
|
|
343
|
+
args = [];
|
|
344
|
+
inCommand = false;
|
|
345
|
+
inIce = false;
|
|
346
|
+
commandStartPath = currentPath;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
while (ctx.pos < ctx.tokens.length) {
|
|
350
|
+
const tok = ctx.tokens[ctx.pos];
|
|
351
|
+
|
|
352
|
+
// Stop at enclosing bracket/brace
|
|
353
|
+
if (stopAt !== undefined && tok.type === stopAt) {
|
|
354
|
+
break;
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
switch (tok.type) {
|
|
358
|
+
case Tok.Semicolon:
|
|
359
|
+
case Tok.Newline:
|
|
360
|
+
flushCommand();
|
|
361
|
+
ctx.pos++;
|
|
362
|
+
break;
|
|
363
|
+
|
|
364
|
+
case Tok.Slash:
|
|
365
|
+
ctx.pos++;
|
|
366
|
+
if (!inCommand) {
|
|
367
|
+
// Leading / — this is an absolute path
|
|
368
|
+
isAbsolute = true;
|
|
369
|
+
inCommand = true;
|
|
370
|
+
} else if (pathSegments.length > 0 || isAbsolute) {
|
|
371
|
+
// Slash in the middle of a path sequence — next word is a path segment
|
|
372
|
+
// (handled naturally by the Word case)
|
|
373
|
+
}
|
|
374
|
+
// Check if bare `/` (no further path tokens) — this is root navigation
|
|
375
|
+
if (isAbsolute && pathSegments.length === 0) {
|
|
376
|
+
const nextPos = ctx.pos;
|
|
377
|
+
// Peek ahead: if next token is a command separator, end-of-input, or stopAt,
|
|
378
|
+
// this is bare root navigation
|
|
379
|
+
if (nextPos >= ctx.tokens.length ||
|
|
380
|
+
ctx.tokens[nextPos].type === Tok.Semicolon ||
|
|
381
|
+
ctx.tokens[nextPos].type === Tok.Newline ||
|
|
382
|
+
(stopAt !== undefined && ctx.tokens[nextPos].type === stopAt)) {
|
|
383
|
+
currentPath = '/';
|
|
384
|
+
commandStartPath = '/';
|
|
385
|
+
isAbsolute = false;
|
|
386
|
+
inCommand = false;
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
break;
|
|
390
|
+
|
|
391
|
+
case Tok.DotDot:
|
|
392
|
+
ctx.pos++;
|
|
393
|
+
pathSegments.push('..');
|
|
394
|
+
inCommand = true;
|
|
395
|
+
break;
|
|
396
|
+
|
|
397
|
+
case Tok.Colon:
|
|
398
|
+
ctx.pos++;
|
|
399
|
+
// ICE command prefix — skip the global/ICE command name
|
|
400
|
+
inIce = true;
|
|
401
|
+
inCommand = true;
|
|
402
|
+
// The next word is the ICE command name (put, local, global, etc.)
|
|
403
|
+
// We don't extract ICE commands as path commands — skip them
|
|
404
|
+
// But they might contain subshells [...]
|
|
405
|
+
if (ctx.pos < ctx.tokens.length && ctx.tokens[ctx.pos].type === Tok.Word) {
|
|
406
|
+
ctx.pos++; // skip ICE command name
|
|
407
|
+
}
|
|
408
|
+
break;
|
|
409
|
+
|
|
410
|
+
case Tok.LBracket: {
|
|
411
|
+
ctx.pos++; // consume [
|
|
412
|
+
// Subshell — inherits the in-progress resolved path, not the stale cwd.
|
|
413
|
+
// Resolve what the current path would be from accumulated segments.
|
|
414
|
+
let subshellCwd = commandStartPath;
|
|
415
|
+
if (isAbsolute) subshellCwd = '/';
|
|
416
|
+
for (const seg of pathSegments) {
|
|
417
|
+
if (seg === '..') { subshellCwd = resolveParent(subshellCwd); }
|
|
418
|
+
else { subshellCwd = joinPath(subshellCwd, seg); }
|
|
419
|
+
}
|
|
420
|
+
// If a verb has been identified, the subshell cwd is the dir path (not verb)
|
|
421
|
+
subshellCwd = normalizePath(subshellCwd);
|
|
422
|
+
if (verb) {
|
|
423
|
+
// cwd for the subshell is the path (dir), not including the verb
|
|
424
|
+
// (already computed without verb)
|
|
425
|
+
}
|
|
426
|
+
const subCtx: ParseContext = {
|
|
427
|
+
tokens: ctx.tokens,
|
|
428
|
+
pos: ctx.pos,
|
|
429
|
+
cwd: subshellCwd,
|
|
430
|
+
commands: ctx.commands,
|
|
431
|
+
subshell: true,
|
|
432
|
+
};
|
|
433
|
+
parseCommands(subCtx, Tok.RBracket);
|
|
434
|
+
ctx.pos = subCtx.pos;
|
|
435
|
+
if (ctx.pos < ctx.tokens.length && ctx.tokens[ctx.pos].type === Tok.RBracket) {
|
|
436
|
+
ctx.pos++; // consume ]
|
|
437
|
+
}
|
|
438
|
+
// After ], we're back in the outer command — subshell result is a value
|
|
439
|
+
break;
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
case Tok.RBracket:
|
|
443
|
+
// Should be handled by stopAt, but if we hit it unexpectedly, just stop
|
|
444
|
+
flushCommand();
|
|
445
|
+
return currentPath;
|
|
446
|
+
|
|
447
|
+
case Tok.LBrace: {
|
|
448
|
+
ctx.pos++; // consume {
|
|
449
|
+
// Block — inherit path from current command prefix
|
|
450
|
+
let blockPath = commandStartPath;
|
|
451
|
+
if (isAbsolute || pathSegments.length > 0) {
|
|
452
|
+
// Resolve the path prefix before the block
|
|
453
|
+
blockPath = isAbsolute ? '/' : commandStartPath;
|
|
454
|
+
for (const seg of pathSegments) {
|
|
455
|
+
if (seg === '..') {
|
|
456
|
+
blockPath = resolveParent(blockPath);
|
|
457
|
+
} else {
|
|
458
|
+
blockPath = joinPath(blockPath, seg);
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
blockPath = normalizePath(blockPath);
|
|
462
|
+
}
|
|
463
|
+
const blockCtx: ParseContext = {
|
|
464
|
+
tokens: ctx.tokens,
|
|
465
|
+
pos: ctx.pos,
|
|
466
|
+
cwd: blockPath,
|
|
467
|
+
commands: ctx.commands,
|
|
468
|
+
subshell: ctx.subshell,
|
|
469
|
+
};
|
|
470
|
+
parseCommands(blockCtx, Tok.RBrace);
|
|
471
|
+
ctx.pos = blockCtx.pos;
|
|
472
|
+
if (ctx.pos < ctx.tokens.length && ctx.tokens[ctx.pos].type === Tok.RBrace) {
|
|
473
|
+
ctx.pos++; // consume }
|
|
474
|
+
}
|
|
475
|
+
// After { }, the path set by the prefix persists
|
|
476
|
+
currentPath = blockPath;
|
|
477
|
+
commandStartPath = currentPath;
|
|
478
|
+
// Reset the current command line since the block consumed it
|
|
479
|
+
pathSegments = [];
|
|
480
|
+
isAbsolute = false;
|
|
481
|
+
verb = '';
|
|
482
|
+
args = [];
|
|
483
|
+
inCommand = false;
|
|
484
|
+
break;
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
case Tok.RBrace:
|
|
488
|
+
flushCommand();
|
|
489
|
+
return currentPath;
|
|
490
|
+
|
|
491
|
+
case Tok.Word: {
|
|
492
|
+
const w = tok.value;
|
|
493
|
+
ctx.pos++;
|
|
494
|
+
inCommand = true;
|
|
495
|
+
|
|
496
|
+
if (inIce) {
|
|
497
|
+
// Inside an ICE command — everything is an argument, skip
|
|
498
|
+
// But still handle subshells within arguments
|
|
499
|
+
break;
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
// Is this an argument (contains = but isn't a path)?
|
|
503
|
+
if (w.includes('=') && !w.startsWith('/')) {
|
|
504
|
+
args.push(w);
|
|
505
|
+
break;
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
// Is this a known verb?
|
|
509
|
+
// Recognized when we have a path prefix (explicit segments or absolute /),
|
|
510
|
+
// OR when cwd is non-root (e.g., inside a subshell or script context)
|
|
511
|
+
if (!verb && isKnownVerb(w) && (pathSegments.length > 0 || isAbsolute || commandStartPath !== '/')) {
|
|
512
|
+
verb = w;
|
|
513
|
+
break;
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
// Is this an unnamed param (starts with * for item ID, or is a value after a verb)?
|
|
517
|
+
if (verb) {
|
|
518
|
+
args.push(w);
|
|
519
|
+
break;
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
// If we already have path segments and this doesn't look like a path segment,
|
|
523
|
+
// it might be a verb we don't know about, or an unnamed param
|
|
524
|
+
if ((pathSegments.length > 0 || isAbsolute) && !w.includes('-') && !w.includes('_') &&
|
|
525
|
+
w === w.toLowerCase() && w.length <= 3 && !/^[a-z]/.test(w)) {
|
|
526
|
+
args.push(w);
|
|
527
|
+
break;
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
// Must be a path segment (could also be a verb — we'll resolve at flush time)
|
|
531
|
+
// Handle compound segments like "firewall/filter" (RouterOS allows spaces OR slashes)
|
|
532
|
+
if (w.includes('/')) {
|
|
533
|
+
const subParts = w.split('/').filter(Boolean);
|
|
534
|
+
pathSegments.push(...subParts);
|
|
535
|
+
} else {
|
|
536
|
+
pathSegments.push(w);
|
|
537
|
+
}
|
|
538
|
+
break;
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
case Tok.Equals:
|
|
542
|
+
ctx.pos++;
|
|
543
|
+
break;
|
|
544
|
+
|
|
545
|
+
default:
|
|
546
|
+
ctx.pos++;
|
|
547
|
+
break;
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
// Flush any remaining command
|
|
552
|
+
flushCommand();
|
|
553
|
+
return currentPath;
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
// ---------------------------------------------------------------------------
|
|
557
|
+
// Public API
|
|
558
|
+
// ---------------------------------------------------------------------------
|
|
559
|
+
|
|
560
|
+
/**
|
|
561
|
+
* Parse a RouterOS CLI input string and extract canonical command tuples.
|
|
562
|
+
*
|
|
563
|
+
* @param input - One or more RouterOS command lines (can contain `;`, `\n`,
|
|
564
|
+
* `[...]` subshells, `{ }` blocks)
|
|
565
|
+
* @param cwd - Current working directory in the menu hierarchy.
|
|
566
|
+
* Defaults to '/' (root).
|
|
567
|
+
* @returns ParseResult with all extracted commands and the final path
|
|
568
|
+
*
|
|
569
|
+
* @example
|
|
570
|
+
* ```ts
|
|
571
|
+
* canonicalize('/ip/address/set .id=*1 disabled=yes')
|
|
572
|
+
* // => { commands: [{ path: '/ip/address', verb: 'set', args: ['.id=*1', 'disabled=yes'] }], finalPath: '/ip/address' }
|
|
573
|
+
*
|
|
574
|
+
* canonicalize('set disabled=yes', '/ip/address')
|
|
575
|
+
* // => { commands: [{ path: '/ip/address', verb: 'set', args: ['disabled=yes'] }], finalPath: '/ip/address' }
|
|
576
|
+
*
|
|
577
|
+
* canonicalize('/ip/address set [find interface=ether1] disabled=yes')
|
|
578
|
+
* // => { commands: [
|
|
579
|
+
* // { path: '/ip/address', verb: 'find', args: ['interface=ether1'], subshell: true },
|
|
580
|
+
* // { path: '/ip/address', verb: 'set', args: ['disabled=yes'] },
|
|
581
|
+
* // ], finalPath: '/ip/address' }
|
|
582
|
+
* ```
|
|
583
|
+
*/
|
|
584
|
+
export function canonicalize(input: string, cwd = '/'): ParseResult {
|
|
585
|
+
const tokens = tokenize(input);
|
|
586
|
+
const ctx: ParseContext = {
|
|
587
|
+
tokens,
|
|
588
|
+
pos: 0,
|
|
589
|
+
cwd: normalizePath(cwd),
|
|
590
|
+
commands: [],
|
|
591
|
+
subshell: false,
|
|
592
|
+
};
|
|
593
|
+
const finalPath = parseCommands(ctx);
|
|
594
|
+
return { commands: ctx.commands, finalPath: normalizePath(finalPath) };
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
/**
|
|
598
|
+
* Convenience: extract just the canonical paths from an input.
|
|
599
|
+
* Returns unique paths in order of first appearance.
|
|
600
|
+
* Useful for "what commands does this script reference?" queries.
|
|
601
|
+
*/
|
|
602
|
+
export function extractPaths(input: string, cwd = '/'): string[] {
|
|
603
|
+
const { commands } = canonicalize(input, cwd);
|
|
604
|
+
const seen = new Set<string>();
|
|
605
|
+
const paths: string[] = [];
|
|
606
|
+
for (const cmd of commands) {
|
|
607
|
+
const full = cmd.verb ? `${cmd.path}/${cmd.verb}` : cmd.path;
|
|
608
|
+
const normalized = normalizePath(full);
|
|
609
|
+
if (!seen.has(normalized)) {
|
|
610
|
+
seen.add(normalized);
|
|
611
|
+
paths.push(normalized);
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
return paths;
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
/**
|
|
618
|
+
* Convenience: extract the primary command path (the first non-subshell command).
|
|
619
|
+
* Useful for quick lookup: "what's the main path this user is asking about?"
|
|
620
|
+
*/
|
|
621
|
+
export function primaryPath(input: string, cwd = '/'): string | null {
|
|
622
|
+
const { commands } = canonicalize(input, cwd);
|
|
623
|
+
const primary = commands.find(c => !c.subshell) ?? commands[0];
|
|
624
|
+
if (!primary) return null;
|
|
625
|
+
return normalizePath(primary.path);
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
// Export for testing
|
|
629
|
+
export { normalizePath as _normalizePath, tokenize as _tokenize };
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// Set BEFORE imports that can transitively load db.ts
|
|
2
|
+
process.env.DB_PATH = ":memory:";
|
|
3
|
+
|
|
4
|
+
import { describe, expect, test } from "bun:test";
|
|
5
|
+
|
|
6
|
+
const { buildChangelogVersionSet, LEGACY_FORMATTED_V7_BASE_VERSIONS } = await import("./extract-changelogs.ts");
|
|
7
|
+
|
|
8
|
+
describe("buildChangelogVersionSet", () => {
|
|
9
|
+
test("always includes legacy v7 baseline versions", () => {
|
|
10
|
+
const versions = buildChangelogVersionSet(["7.22.1", "7.23beta2"]);
|
|
11
|
+
|
|
12
|
+
for (const legacy of LEGACY_FORMATTED_V7_BASE_VERSIONS) {
|
|
13
|
+
expect(versions).toContain(legacy);
|
|
14
|
+
}
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
test("keeps existing known versions and de-duplicates overlaps", () => {
|
|
18
|
+
const versions = buildChangelogVersionSet(["7.22.1", "7.2", "7.8"]);
|
|
19
|
+
|
|
20
|
+
expect(versions).toContain("7.22.1");
|
|
21
|
+
expect(versions.filter((v) => v === "7.2")).toHaveLength(1);
|
|
22
|
+
expect(versions.filter((v) => v === "7.8")).toHaveLength(1);
|
|
23
|
+
});
|
|
24
|
+
});
|
|
@@ -15,6 +15,10 @@ import { db, initDb } from "./db.ts";
|
|
|
15
15
|
const CHANGELOG_BASE = "https://download.mikrotik.com/routeros";
|
|
16
16
|
const FETCH_DELAY_MS = 200; // polite delay between requests
|
|
17
17
|
|
|
18
|
+
// Oldest RouterOS v7 versions that use the current changelog header/entry format.
|
|
19
|
+
// Keep these in default extraction even though command tree retention starts later.
|
|
20
|
+
export const LEGACY_FORMATTED_V7_BASE_VERSIONS = ["7.1.1", "7.2", "7.3", "7.4", "7.5", "7.6", "7.7", "7.8"];
|
|
21
|
+
|
|
18
22
|
// ── Types ──
|
|
19
23
|
|
|
20
24
|
type ChangelogEntry = {
|
|
@@ -145,9 +149,18 @@ function getKnownVersions(): string[] {
|
|
|
145
149
|
return rows.map((r) => r.version);
|
|
146
150
|
}
|
|
147
151
|
|
|
152
|
+
export function buildChangelogVersionSet(knownVersions: string[]): string[] {
|
|
153
|
+
const all = new Set([...knownVersions, ...LEGACY_FORMATTED_V7_BASE_VERSIONS]);
|
|
154
|
+
return [...all];
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function getDefaultVersions(): string[] {
|
|
158
|
+
return buildChangelogVersionSet(getKnownVersions());
|
|
159
|
+
}
|
|
160
|
+
|
|
148
161
|
/** Probe patch versions: for each minor (7.X), try 7.X.1, 7.X.2, ... up to first 404. */
|
|
149
162
|
async function probePatchVersions(): Promise<string[]> {
|
|
150
|
-
const known = new Set(
|
|
163
|
+
const known = new Set(getDefaultVersions());
|
|
151
164
|
const patches: string[] = [];
|
|
152
165
|
|
|
153
166
|
// Find all minor versions: extract unique 7.X prefixes
|
|
@@ -198,15 +211,15 @@ if (versionsArg) {
|
|
|
198
211
|
console.log(`Changelog extraction: ${versions.length} explicit versions`);
|
|
199
212
|
} else if (probePatches) {
|
|
200
213
|
console.log("Changelog extraction: probing patch versions...");
|
|
201
|
-
const known =
|
|
214
|
+
const known = getDefaultVersions();
|
|
202
215
|
const patches = await probePatchVersions();
|
|
203
216
|
// Merge: known + discovered patches (deduplicated)
|
|
204
217
|
const all = new Set([...known, ...patches]);
|
|
205
218
|
versions = [...all];
|
|
206
219
|
console.log(` ${versions.length} versions (${patches.length} from patch probing)`);
|
|
207
220
|
} else {
|
|
208
|
-
versions =
|
|
209
|
-
console.log(`Changelog extraction: ${versions.length} versions from ros_versions`);
|
|
221
|
+
versions = getDefaultVersions();
|
|
222
|
+
console.log(`Changelog extraction: ${versions.length} versions from ros_versions + legacy v7 baseline`);
|
|
210
223
|
}
|
|
211
224
|
|
|
212
225
|
if (versions.length === 0) {
|