@tikoci/rosetta 0.8.9 → 0.8.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/package.json +1 -1
- package/src/browse.ts +12 -2
- package/src/canonicalize-resolver.ts +59 -0
- package/src/canonicalize.fuzz.test.ts +371 -0
- package/src/canonicalize.test.ts +68 -0
- package/src/canonicalize.ts +240 -20
- package/src/classify.test.ts +16 -3
- package/src/classify.ts +36 -8
- package/src/extract-videos.ts +46 -4
- package/src/gc-versions.test.ts +201 -0
- package/src/gc-versions.ts +230 -0
- package/src/mcp-contract.test.ts +7 -10
- package/src/mcp-http.test.ts +6 -5
- package/src/mcp.ts +91 -49
- package/src/query.test.ts +146 -3
- package/src/query.ts +252 -14
- package/src/release.test.ts +127 -1
- package/src/setup.test.ts +109 -2
- package/src/setup.ts +359 -120
package/src/canonicalize.ts
CHANGED
|
@@ -31,15 +31,56 @@ export interface CanonicalCommand {
|
|
|
31
31
|
args: string[];
|
|
32
32
|
/** For subshell commands extracted from [...], this is true */
|
|
33
33
|
subshell?: boolean;
|
|
34
|
+
/**
|
|
35
|
+
* How well-formed the extraction was. Lets consumers filter prose-extracted
|
|
36
|
+
* results when they need higher precision (e.g. LSP hover).
|
|
37
|
+
* - 'high' — absolute path with directly-identified verb (well-formed CLI)
|
|
38
|
+
* - 'medium' — relative-with-cwd, pure navigation, or block/subshell context
|
|
39
|
+
* - 'low' — verb was inferred from a trailing path segment at flush time,
|
|
40
|
+
* i.e. extracted from looser/prose-shaped input
|
|
41
|
+
*/
|
|
42
|
+
confidence: 'high' | 'medium' | 'low';
|
|
34
43
|
}
|
|
35
44
|
|
|
36
45
|
export interface ParseResult {
|
|
37
46
|
/** All commands extracted from the input, in order */
|
|
38
47
|
commands: CanonicalCommand[];
|
|
48
|
+
/**
|
|
49
|
+
* Every distinct path the input *referenced* — including bare navigation
|
|
50
|
+
* (e.g. `/ip/firewall/filter` with no verb). Superset of `commands[i].path`.
|
|
51
|
+
* Used by `extractMentions()` for "what does this text reference?" queries.
|
|
52
|
+
*/
|
|
53
|
+
mentions: string[];
|
|
39
54
|
/** The cwd after executing all navigation (useful for interactive sessions) */
|
|
40
55
|
finalPath: string;
|
|
41
56
|
}
|
|
42
57
|
|
|
58
|
+
/**
|
|
59
|
+
* Optional behaviour knobs. Keeping the module pure: the resolver is supplied
|
|
60
|
+
* by the caller, never imported. rosetta wires a DB-backed resolver against
|
|
61
|
+
* its `commands` table; lsp-routeros-ts wires a static `verbs.json`-backed
|
|
62
|
+
* one. The built-in universal-verb-set heuristic always remains active so
|
|
63
|
+
* resolvers only need to supply path-specific verbs, not every common command.
|
|
64
|
+
*/
|
|
65
|
+
export interface CanonicalizeOptions {
|
|
66
|
+
/**
|
|
67
|
+
* Path-aware verb classifier. Called when the parser is deciding whether a
|
|
68
|
+
* token at the end of an in-flight command is a verb or another path
|
|
69
|
+
* segment. Return `true` to treat `token` as the verb.
|
|
70
|
+
*
|
|
71
|
+
* `parentPath` is the absolute dir path the verb would attach to (the
|
|
72
|
+
* resolved path of all path segments seen so far, *without* this token).
|
|
73
|
+
* MUST be synchronous and side-effect-free.
|
|
74
|
+
*
|
|
75
|
+
* The small built-in universal verb set (GENERAL_COMMANDS + EXTRA_VERBS)
|
|
76
|
+
* is checked first; this resolver supplements it for path-context-specific
|
|
77
|
+
* verbs. That keeps callers from having to enumerate ubiquitous helpers
|
|
78
|
+
* like `find` while still allowing DB/live-router precision for ambiguous
|
|
79
|
+
* tokens (see issue #5, finding #4).
|
|
80
|
+
*/
|
|
81
|
+
isVerb?: (token: string, parentPath: string) => boolean;
|
|
82
|
+
}
|
|
83
|
+
|
|
43
84
|
// ---------------------------------------------------------------------------
|
|
44
85
|
// Known RouterOS general commands (verbs).
|
|
45
86
|
// These appear at every menu level. Used to distinguish "is this token a
|
|
@@ -47,10 +88,17 @@ export interface ParseResult {
|
|
|
47
88
|
// ---------------------------------------------------------------------------
|
|
48
89
|
|
|
49
90
|
const GENERAL_COMMANDS = new Set([
|
|
50
|
-
'add', 'comment', 'disable', 'edit', 'enable', 'export', 'find',
|
|
51
|
-
'get', 'move', 'print', 'remove', 'reset', '
|
|
91
|
+
'add', 'clear', 'comment', 'disable', 'edit', 'enable', 'export', 'find',
|
|
92
|
+
'get', 'move', 'print', 'remove', 'reset', 'reset-counters',
|
|
93
|
+
'reset-counters-all', 'set', 'unset',
|
|
52
94
|
]);
|
|
53
95
|
|
|
96
|
+
// Verbs that are NOT in this set despite being common at certain menus:
|
|
97
|
+
// 'info', 'warning', 'error', 'debug' — used by /log, but `info` is also a
|
|
98
|
+
// dir at /interface/wireless, and /error is itself a top-level cmd.
|
|
99
|
+
// Adding them here would mis-resolve those paths. Use a path-aware verb
|
|
100
|
+
// resolver (see CanonicalizeOptions.isVerb) for menu-specific verbs.
|
|
101
|
+
|
|
54
102
|
/**
|
|
55
103
|
* Commands that are clearly verbs even though they only appear at certain
|
|
56
104
|
* menu levels. We keep this small — the heuristic is "if in doubt, treat
|
|
@@ -97,14 +145,21 @@ interface Token {
|
|
|
97
145
|
*/
|
|
98
146
|
function tokenize(input: string): Token[] {
|
|
99
147
|
const tokens: Token[] = [];
|
|
100
|
-
|
|
148
|
+
// Strip leading BOM (U+FEFF) — common when input is read from a UTF-8 file
|
|
149
|
+
// with a BOM marker. Without this, the BOM ends up embedded in the first
|
|
150
|
+
// path segment.
|
|
151
|
+
let i = input.charCodeAt(0) === 0xfeff ? 1 : 0;
|
|
101
152
|
const len = input.length;
|
|
102
153
|
|
|
103
154
|
while (i < len) {
|
|
104
155
|
const ch = input[i];
|
|
105
156
|
|
|
106
|
-
// Skip whitespace (except newlines)
|
|
107
|
-
|
|
157
|
+
// Skip whitespace (except newlines).
|
|
158
|
+
// Backticks (`) are treated as whitespace so markdown-style snippets like
|
|
159
|
+
// `/ip/address/print` extract cleanly from prose.
|
|
160
|
+
// Zero-width space (U+200B) is also stripped — sometimes copy-pasted from
|
|
161
|
+
// documentation and otherwise pollutes path segments.
|
|
162
|
+
if (ch === ' ' || ch === '\t' || ch === '\r' || ch === '`' || ch === '') { i++; continue; }
|
|
108
163
|
|
|
109
164
|
// Newline
|
|
110
165
|
if (ch === '\n') { tokens.push({ type: Tok.Newline, value: '\n' }); i++; continue; }
|
|
@@ -164,6 +219,9 @@ function tokenize(input: string): Token[] {
|
|
|
164
219
|
while (i < len) {
|
|
165
220
|
const c = input[i];
|
|
166
221
|
if (c === ' ' || c === '\t' || c === '\r' || c === '\n') break;
|
|
222
|
+
// Treat the same as outer loop: backtick + ZWSP are whitespace inside words too,
|
|
223
|
+
// so trailing markdown markers don't get glued onto path segments.
|
|
224
|
+
if (c === '`' || c === '') break;
|
|
167
225
|
if (c === '[' || c === ']' || c === '{' || c === '}' || c === ';') break;
|
|
168
226
|
if (c === '#') break;
|
|
169
227
|
// Slash breaks a word UNLESS it's inside a CIDR or we're at a slash
|
|
@@ -262,6 +320,46 @@ interface ParseContext {
|
|
|
262
320
|
cwd: string;
|
|
263
321
|
commands: CanonicalCommand[];
|
|
264
322
|
subshell: boolean;
|
|
323
|
+
options: CanonicalizeOptions;
|
|
324
|
+
/**
|
|
325
|
+
* Accumulator for paths the input *navigated to* — including bare
|
|
326
|
+
* navigation that doesn't produce a CanonicalCommand. Surfaced via
|
|
327
|
+
* `ParseResult.mentions` and `extractMentions()`.
|
|
328
|
+
*/
|
|
329
|
+
mentions: string[];
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/**
|
|
333
|
+
* Resolve the absolute path for a token's `parentPath` argument when the
|
|
334
|
+
* parser is deciding whether the token is a verb. Mirrors the resolution
|
|
335
|
+
* `flushCommand` would do, *without* including the token itself.
|
|
336
|
+
*/
|
|
337
|
+
function resolveParentPath(
|
|
338
|
+
commandStartPath: string,
|
|
339
|
+
isAbsolute: boolean,
|
|
340
|
+
pathSegments: string[],
|
|
341
|
+
): string {
|
|
342
|
+
let p = isAbsolute ? '/' : commandStartPath;
|
|
343
|
+
for (const seg of pathSegments) {
|
|
344
|
+
if (seg === '..') p = resolveParent(p);
|
|
345
|
+
else p = joinPath(p, seg);
|
|
346
|
+
}
|
|
347
|
+
return normalizePath(p);
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
/**
|
|
351
|
+
* Path-aware verb check. The curated universal verb set remains active even
|
|
352
|
+
* when a resolver is supplied because RouterOS introspection data does not
|
|
353
|
+
* necessarily enumerate helper verbs like `find` at every parent path.
|
|
354
|
+
* Resolver hits add path-specific verbs (for example, `info` at menus where
|
|
355
|
+
* it is actually a command).
|
|
356
|
+
*/
|
|
357
|
+
function isVerbAt(
|
|
358
|
+
token: string,
|
|
359
|
+
parentPath: string,
|
|
360
|
+
options: CanonicalizeOptions,
|
|
361
|
+
): boolean {
|
|
362
|
+
return isKnownVerb(token) || options.isVerb?.(token, parentPath) === true;
|
|
265
363
|
}
|
|
266
364
|
|
|
267
365
|
/**
|
|
@@ -283,6 +381,10 @@ function parseCommands(ctx: ParseContext, stopAt?: Tok): string {
|
|
|
283
381
|
let pathSegments: string[] = [];
|
|
284
382
|
let isAbsolute = false;
|
|
285
383
|
let verb = '';
|
|
384
|
+
// True when the verb was directly identified during Word handling (well-formed
|
|
385
|
+
// CLI). False when the verb is inferred from the trailing path segment in
|
|
386
|
+
// flushCommand — that path is prose-shaped / lower confidence.
|
|
387
|
+
let verbExplicit = false;
|
|
286
388
|
let args: string[] = [];
|
|
287
389
|
let inCommand = false; // true once we've seen at least one token in this command
|
|
288
390
|
let inIce = false; // true if we're in an ICE command (: prefix)
|
|
@@ -303,25 +405,54 @@ function parseCommands(ctx: ParseContext, stopAt?: Tok): string {
|
|
|
303
405
|
resolvedPath = normalizePath(resolvedPath);
|
|
304
406
|
|
|
305
407
|
if (verb || args.length > 0 || pathSegments.length > 0) {
|
|
306
|
-
// If no explicit verb but we have path segments, the last segment might
|
|
408
|
+
// If no explicit verb but we have path segments, the last segment might
|
|
409
|
+
// be a verb. Use the resolver-aware check so path-specific verbs resolve
|
|
410
|
+
// when a resolver is wired, while staying conservative when not.
|
|
411
|
+
let verbInferred = false;
|
|
307
412
|
if (!verb && pathSegments.length > 0) {
|
|
308
413
|
const lastSeg = pathSegments[pathSegments.length - 1];
|
|
309
|
-
if (lastSeg !== '..'
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
414
|
+
if (lastSeg !== '..') {
|
|
415
|
+
const parentOfLast = resolveParent(resolvedPath);
|
|
416
|
+
if (isVerbAt(lastSeg, parentOfLast, ctx.options)) {
|
|
417
|
+
verb = lastSeg;
|
|
418
|
+
verbInferred = true;
|
|
419
|
+
// Remove the verb from the resolved path
|
|
420
|
+
resolvedPath = parentOfLast;
|
|
421
|
+
}
|
|
313
422
|
}
|
|
314
423
|
}
|
|
315
424
|
|
|
316
425
|
if (verb || args.length > 0) {
|
|
426
|
+
// Confidence:
|
|
427
|
+
// - 'low' when the verb was inferred from a trailing path segment
|
|
428
|
+
// at flush time (looser/prose-shaped input)
|
|
429
|
+
// - 'high' absolute path + directly-identified verb (well-formed)
|
|
430
|
+
// - 'medium' everything else (relative-with-cwd, navigation, blocks)
|
|
431
|
+
let confidence: 'high' | 'medium' | 'low';
|
|
432
|
+
if (verbInferred) {
|
|
433
|
+
confidence = 'low';
|
|
434
|
+
} else if (isAbsolute && verbExplicit) {
|
|
435
|
+
confidence = 'high';
|
|
436
|
+
} else {
|
|
437
|
+
confidence = 'medium';
|
|
438
|
+
}
|
|
439
|
+
|
|
317
440
|
ctx.commands.push({
|
|
318
441
|
path: resolvedPath,
|
|
319
442
|
verb,
|
|
320
443
|
args: [...args],
|
|
321
444
|
...(ctx.subshell ? { subshell: true } : {}),
|
|
445
|
+
confidence,
|
|
322
446
|
});
|
|
323
447
|
}
|
|
324
448
|
|
|
449
|
+
// Track every path the input mentions, even when no command is emitted
|
|
450
|
+
// (pure navigation like `/ip/firewall/filter`). Surfaced via
|
|
451
|
+
// ParseResult.mentions and extractMentions().
|
|
452
|
+
if (pathSegments.length > 0 || isAbsolute) {
|
|
453
|
+
ctx.mentions.push(resolvedPath);
|
|
454
|
+
}
|
|
455
|
+
|
|
325
456
|
// After a command with a verb, path does NOT change for next command in same scope
|
|
326
457
|
// Only bare navigation (path without verb) changes the cwd
|
|
327
458
|
if (!verb && pathSegments.length > 0) {
|
|
@@ -340,6 +471,7 @@ function parseCommands(ctx: ParseContext, stopAt?: Tok): string {
|
|
|
340
471
|
pathSegments = [];
|
|
341
472
|
isAbsolute = false;
|
|
342
473
|
verb = '';
|
|
474
|
+
verbExplicit = false;
|
|
343
475
|
args = [];
|
|
344
476
|
inCommand = false;
|
|
345
477
|
inIce = false;
|
|
@@ -429,6 +561,8 @@ function parseCommands(ctx: ParseContext, stopAt?: Tok): string {
|
|
|
429
561
|
cwd: subshellCwd,
|
|
430
562
|
commands: ctx.commands,
|
|
431
563
|
subshell: true,
|
|
564
|
+
options: ctx.options,
|
|
565
|
+
mentions: ctx.mentions,
|
|
432
566
|
};
|
|
433
567
|
parseCommands(subCtx, Tok.RBracket);
|
|
434
568
|
ctx.pos = subCtx.pos;
|
|
@@ -466,6 +600,8 @@ function parseCommands(ctx: ParseContext, stopAt?: Tok): string {
|
|
|
466
600
|
cwd: blockPath,
|
|
467
601
|
commands: ctx.commands,
|
|
468
602
|
subshell: ctx.subshell,
|
|
603
|
+
options: ctx.options,
|
|
604
|
+
mentions: ctx.mentions,
|
|
469
605
|
};
|
|
470
606
|
parseCommands(blockCtx, Tok.RBrace);
|
|
471
607
|
ctx.pos = blockCtx.pos;
|
|
@@ -507,10 +643,17 @@ function parseCommands(ctx: ParseContext, stopAt?: Tok): string {
|
|
|
507
643
|
|
|
508
644
|
// Is this a known verb?
|
|
509
645
|
// Recognized when we have a path prefix (explicit segments or absolute /),
|
|
510
|
-
// OR when cwd is non-root (e.g., inside a subshell or script context)
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
646
|
+
// OR when cwd is non-root (e.g., inside a subshell or script context).
|
|
647
|
+
// Path-aware: consults the optional resolver against the parent path of
|
|
648
|
+
// segments seen so far (without `w`), so menu-specific commands and
|
|
649
|
+
// same-named directories disambiguate correctly when a resolver is wired.
|
|
650
|
+
if (!verb && (pathSegments.length > 0 || isAbsolute || commandStartPath !== '/')) {
|
|
651
|
+
const parentPath = resolveParentPath(commandStartPath, isAbsolute, pathSegments);
|
|
652
|
+
if (isVerbAt(w, parentPath, ctx.options)) {
|
|
653
|
+
verb = w;
|
|
654
|
+
verbExplicit = true;
|
|
655
|
+
break;
|
|
656
|
+
}
|
|
514
657
|
}
|
|
515
658
|
|
|
516
659
|
// Is this an unnamed param (starts with * for item ID, or is a value after a verb)?
|
|
@@ -581,7 +724,11 @@ function parseCommands(ctx: ParseContext, stopAt?: Tok): string {
|
|
|
581
724
|
* // ], finalPath: '/ip/address' }
|
|
582
725
|
* ```
|
|
583
726
|
*/
|
|
584
|
-
export function canonicalize(
|
|
727
|
+
export function canonicalize(
|
|
728
|
+
input: string,
|
|
729
|
+
cwd = '/',
|
|
730
|
+
options: CanonicalizeOptions = {},
|
|
731
|
+
): ParseResult {
|
|
585
732
|
const tokens = tokenize(input);
|
|
586
733
|
const ctx: ParseContext = {
|
|
587
734
|
tokens,
|
|
@@ -589,18 +736,40 @@ export function canonicalize(input: string, cwd = '/'): ParseResult {
|
|
|
589
736
|
cwd: normalizePath(cwd),
|
|
590
737
|
commands: [],
|
|
591
738
|
subshell: false,
|
|
739
|
+
options,
|
|
740
|
+
mentions: [],
|
|
592
741
|
};
|
|
593
742
|
const finalPath = parseCommands(ctx);
|
|
594
|
-
|
|
743
|
+
// Dedupe mentions in order of first appearance
|
|
744
|
+
const seenMentions = new Set<string>();
|
|
745
|
+
const mentions: string[] = [];
|
|
746
|
+
for (const m of ctx.mentions) {
|
|
747
|
+
if (!seenMentions.has(m)) {
|
|
748
|
+
seenMentions.add(m);
|
|
749
|
+
mentions.push(m);
|
|
750
|
+
}
|
|
751
|
+
}
|
|
752
|
+
return {
|
|
753
|
+
commands: ctx.commands,
|
|
754
|
+
mentions,
|
|
755
|
+
finalPath: normalizePath(finalPath),
|
|
756
|
+
};
|
|
595
757
|
}
|
|
596
758
|
|
|
597
759
|
/**
|
|
598
760
|
* Convenience: extract just the canonical paths from an input.
|
|
599
761
|
* Returns unique paths in order of first appearance.
|
|
600
762
|
* Useful for "what commands does this script reference?" queries.
|
|
763
|
+
*
|
|
764
|
+
* Only includes paths attached to a verb. For navigation-only mentions
|
|
765
|
+
* (e.g. bare `/ip/firewall/filter`), use {@link extractMentions}.
|
|
601
766
|
*/
|
|
602
|
-
export function extractPaths(
|
|
603
|
-
|
|
767
|
+
export function extractPaths(
|
|
768
|
+
input: string,
|
|
769
|
+
cwd = '/',
|
|
770
|
+
options: CanonicalizeOptions = {},
|
|
771
|
+
): string[] {
|
|
772
|
+
const { commands } = canonicalize(input, cwd, options);
|
|
604
773
|
const seen = new Set<string>();
|
|
605
774
|
const paths: string[] = [];
|
|
606
775
|
for (const cmd of commands) {
|
|
@@ -614,12 +783,63 @@ export function extractPaths(input: string, cwd = '/'): string[] {
|
|
|
614
783
|
return paths;
|
|
615
784
|
}
|
|
616
785
|
|
|
786
|
+
/**
|
|
787
|
+
* "What does this text reference?" — every distinct RouterOS path the input
|
|
788
|
+
* mentions, including bare navigation with no verb (e.g. `/ip/firewall/filter`
|
|
789
|
+
* sitting alone in prose). Superset of {@link extractPaths}.
|
|
790
|
+
*
|
|
791
|
+
* Useful for the rosetta classifier, LSP document-link providers, and
|
|
792
|
+
* MCP context feeders that want to surface every path the user gestured at,
|
|
793
|
+
* not just well-formed CLI commands.
|
|
794
|
+
*
|
|
795
|
+
* @example
|
|
796
|
+
* ```ts
|
|
797
|
+
* extractMentions('/ip/firewall/filter ; /ip/firewall/nat')
|
|
798
|
+
* // => ['/ip/firewall/filter', '/ip/firewall/nat']
|
|
799
|
+
* ```
|
|
800
|
+
*/
|
|
801
|
+
export function extractMentions(
|
|
802
|
+
input: string,
|
|
803
|
+
cwd = '/',
|
|
804
|
+
options: CanonicalizeOptions = {},
|
|
805
|
+
): string[] {
|
|
806
|
+
const { commands, mentions } = canonicalize(input, cwd, options);
|
|
807
|
+
const seen = new Set<string>();
|
|
808
|
+
const out: string[] = [];
|
|
809
|
+
// Verbed paths first (commands carry richer signal), then bare mentions.
|
|
810
|
+
for (const cmd of commands) {
|
|
811
|
+
const full = cmd.verb ? `${cmd.path}/${cmd.verb}` : cmd.path;
|
|
812
|
+
const normalized = normalizePath(full);
|
|
813
|
+
if (!seen.has(normalized)) {
|
|
814
|
+
seen.add(normalized);
|
|
815
|
+
out.push(normalized);
|
|
816
|
+
}
|
|
817
|
+
// Also surface the dir path itself (not just path/verb) so
|
|
818
|
+
// /ip/firewall/filter/print also mentions /ip/firewall/filter.
|
|
819
|
+
if (cmd.verb && !seen.has(cmd.path)) {
|
|
820
|
+
seen.add(cmd.path);
|
|
821
|
+
out.push(cmd.path);
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
for (const m of mentions) {
|
|
825
|
+
if (!seen.has(m)) {
|
|
826
|
+
seen.add(m);
|
|
827
|
+
out.push(m);
|
|
828
|
+
}
|
|
829
|
+
}
|
|
830
|
+
return out;
|
|
831
|
+
}
|
|
832
|
+
|
|
617
833
|
/**
|
|
618
834
|
* Convenience: extract the primary command path (the first non-subshell command).
|
|
619
835
|
* Useful for quick lookup: "what's the main path this user is asking about?"
|
|
620
836
|
*/
|
|
621
|
-
export function primaryPath(
|
|
622
|
-
|
|
837
|
+
export function primaryPath(
|
|
838
|
+
input: string,
|
|
839
|
+
cwd = '/',
|
|
840
|
+
options: CanonicalizeOptions = {},
|
|
841
|
+
): string | null {
|
|
842
|
+
const { commands } = canonicalize(input, cwd, options);
|
|
623
843
|
const primary = commands.find(c => !c.subshell) ?? commands[0];
|
|
624
844
|
if (!primary) return null;
|
|
625
845
|
return normalizePath(primary.path);
|
package/src/classify.test.ts
CHANGED
|
@@ -13,6 +13,7 @@ type Expectation = {
|
|
|
13
13
|
topics?: string[]; // must be a subset of detected topics (order-insensitive)
|
|
14
14
|
topicsExact?: string[]; // exact match on detected topics (order-sensitive)
|
|
15
15
|
command_path?: string;
|
|
16
|
+
command_path_confidence?: "high" | "medium" | "low";
|
|
16
17
|
fragment_pairs?: Array<{ key: string; value: string }>;
|
|
17
18
|
fragment_verbs?: string[];
|
|
18
19
|
device?: string;
|
|
@@ -57,17 +58,27 @@ const CASES: Array<{ name: string; input: string; expect: Expectation }> = [
|
|
|
57
58
|
{
|
|
58
59
|
name: "absolute slash path",
|
|
59
60
|
input: "/ip/firewall/filter",
|
|
60
|
-
expect: { command_path: "/ip/firewall/filter", topics: ["ip", "firewall", "filter"] },
|
|
61
|
+
expect: { command_path: "/ip/firewall/filter", command_path_confidence: "medium", topics: ["ip", "firewall", "filter"] },
|
|
61
62
|
},
|
|
62
63
|
{
|
|
63
64
|
name: "space-separated absolute path",
|
|
64
65
|
input: "/ip firewall filter",
|
|
65
|
-
expect: { command_path: "/ip/firewall/filter" },
|
|
66
|
+
expect: { command_path: "/ip/firewall/filter", command_path_confidence: "medium" },
|
|
66
67
|
},
|
|
67
68
|
{
|
|
68
69
|
name: "path without leading slash",
|
|
69
70
|
input: "ip firewall filter",
|
|
70
|
-
expect: { command_path: "/ip/firewall/filter" },
|
|
71
|
+
expect: { command_path: "/ip/firewall/filter", command_path_confidence: "medium" },
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
name: "absolute path with explicit verb has high confidence",
|
|
75
|
+
input: "/ip firewall filter add chain=forward",
|
|
76
|
+
expect: {
|
|
77
|
+
command_path: "/ip/firewall/filter",
|
|
78
|
+
command_path_confidence: "high",
|
|
79
|
+
fragment_pairs: [{ key: "chain", value: "forward" }],
|
|
80
|
+
fragment_verbs: ["add"],
|
|
81
|
+
},
|
|
71
82
|
},
|
|
72
83
|
{
|
|
73
84
|
name: "system scheduler path",
|
|
@@ -142,6 +153,7 @@ describe("classifyQuery", () => {
|
|
|
142
153
|
if (want.none) {
|
|
143
154
|
expect(result.version).toBeUndefined();
|
|
144
155
|
expect(result.command_path).toBeUndefined();
|
|
156
|
+
expect(result.command_path_confidence).toBeUndefined();
|
|
145
157
|
expect(result.command_fragment).toBeUndefined();
|
|
146
158
|
expect(result.device).toBeUndefined();
|
|
147
159
|
expect(result.property).toBeUndefined();
|
|
@@ -151,6 +163,7 @@ describe("classifyQuery", () => {
|
|
|
151
163
|
|
|
152
164
|
if (want.version !== undefined) expect(result.version).toBe(want.version);
|
|
153
165
|
if (want.command_path !== undefined) expect(result.command_path).toBe(want.command_path);
|
|
166
|
+
if (want.command_path_confidence !== undefined) expect(result.command_path_confidence).toBe(want.command_path_confidence);
|
|
154
167
|
if (want.device !== undefined) expect(result.device).toBe(want.device);
|
|
155
168
|
if (want.property !== undefined) expect(result.property).toBe(want.property);
|
|
156
169
|
|
package/src/classify.ts
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* See DESIGN.md "North Star Architecture — Unified routeros_search".
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
|
-
import { canonicalize } from "./canonicalize.ts";
|
|
16
|
+
import { type CanonicalCommand, type CanonicalizeOptions, canonicalize } from "./canonicalize.ts";
|
|
17
17
|
|
|
18
18
|
/**
|
|
19
19
|
* Known RouterOS topics — extracted from changelog categories, top-level
|
|
@@ -65,6 +65,8 @@ export type CommandFragment = {
|
|
|
65
65
|
verbs: string[];
|
|
66
66
|
};
|
|
67
67
|
|
|
68
|
+
export type CommandPathConfidence = CanonicalCommand["confidence"];
|
|
69
|
+
|
|
68
70
|
export type QueryClassification = {
|
|
69
71
|
/** Raw input (unchanged). */
|
|
70
72
|
input: string;
|
|
@@ -74,6 +76,8 @@ export type QueryClassification = {
|
|
|
74
76
|
topics: string[];
|
|
75
77
|
/** Canonical command path (e.g. `/ip/firewall/filter`) if the input looks path-ish. */
|
|
76
78
|
command_path?: string;
|
|
79
|
+
/** Confidence reported by canonicalize for `command_path`; pure navigation is `medium`. */
|
|
80
|
+
command_path_confidence?: CommandPathConfidence;
|
|
77
81
|
/** `key=value` pairs / verbs parsed from fragment-style input (`add chain=forward`). */
|
|
78
82
|
command_fragment?: CommandFragment;
|
|
79
83
|
/** Device model candidate (e.g. `RB1100AHx4`, `hAP`, `CCR2216`). DB resolution happens in searchAll. */
|
|
@@ -137,8 +141,13 @@ function detectVersion(input: string): string | undefined {
|
|
|
137
141
|
return m ? m[0] : undefined;
|
|
138
142
|
}
|
|
139
143
|
|
|
140
|
-
|
|
141
|
-
|
|
144
|
+
type CommandPathDetection = {
|
|
145
|
+
path: string;
|
|
146
|
+
confidence: CommandPathConfidence;
|
|
147
|
+
};
|
|
148
|
+
|
|
149
|
+
/** Detect command path. Uses canonicalize for robust handling of `/ip address`, `ip/address`, etc. */
|
|
150
|
+
function detectCommandPath(input: string, options: CanonicalizeOptions): CommandPathDetection | undefined {
|
|
142
151
|
// Heuristic: only run the canonicalizer if the input contains a forward slash
|
|
143
152
|
// OR starts with a known top-level command word. Otherwise every "routing"
|
|
144
153
|
// query would be classified as a command path.
|
|
@@ -151,11 +160,14 @@ function detectCommandPath(input: string): string | undefined {
|
|
|
151
160
|
try {
|
|
152
161
|
// Prefer the parsed command's dir path if a verb was recognized; otherwise
|
|
153
162
|
// fall back to finalPath (pure navigation like `/ip/firewall/filter`).
|
|
154
|
-
const { commands, finalPath } = canonicalize(trimmed, "/");
|
|
163
|
+
const { commands, finalPath } = canonicalize(trimmed, "/", options);
|
|
155
164
|
const primary = commands.find((c) => !c.subshell) ?? commands[0];
|
|
156
165
|
const path = primary?.path ?? finalPath;
|
|
157
166
|
if (!path || path === "/") return undefined;
|
|
158
|
-
return
|
|
167
|
+
return {
|
|
168
|
+
path,
|
|
169
|
+
confidence: primary?.confidence ?? "medium",
|
|
170
|
+
};
|
|
159
171
|
} catch {
|
|
160
172
|
return undefined;
|
|
161
173
|
}
|
|
@@ -219,11 +231,23 @@ function detectProperty(
|
|
|
219
231
|
return trimmed.toLowerCase();
|
|
220
232
|
}
|
|
221
233
|
|
|
234
|
+
/**
|
|
235
|
+
* Options for {@link classifyQuery}. Currently a thin pass-through to
|
|
236
|
+
* {@link CanonicalizeOptions}; rosetta's `searchAll()` wires a DB-backed
|
|
237
|
+
* `isVerb` resolver so `/log/info` and `/system/script/run` classify as
|
|
238
|
+
* commands instead of falling back to bare navigation. Kept generic so
|
|
239
|
+
* non-DB callers (TUI in offline mode, tests) work without a resolver.
|
|
240
|
+
*/
|
|
241
|
+
export interface ClassifyOptions {
|
|
242
|
+
/** See {@link CanonicalizeOptions.isVerb}. */
|
|
243
|
+
isVerb?: CanonicalizeOptions["isVerb"];
|
|
244
|
+
}
|
|
245
|
+
|
|
222
246
|
/**
|
|
223
247
|
* Classify a user query. Detectors run independently; return object lists every
|
|
224
248
|
* signal found so `searchAll()` can route parallel side queries.
|
|
225
249
|
*/
|
|
226
|
-
export function classifyQuery(input: string): QueryClassification {
|
|
250
|
+
export function classifyQuery(input: string, options: ClassifyOptions = {}): QueryClassification {
|
|
227
251
|
const normalized = input ?? "";
|
|
228
252
|
const result: QueryClassification = {
|
|
229
253
|
input: normalized,
|
|
@@ -235,8 +259,12 @@ export function classifyQuery(input: string): QueryClassification {
|
|
|
235
259
|
const version = detectVersion(normalized);
|
|
236
260
|
if (version) result.version = version;
|
|
237
261
|
|
|
238
|
-
const
|
|
239
|
-
|
|
262
|
+
const canonOptions: CanonicalizeOptions = options.isVerb ? { isVerb: options.isVerb } : {};
|
|
263
|
+
const commandPath = detectCommandPath(normalized, canonOptions);
|
|
264
|
+
if (commandPath) {
|
|
265
|
+
result.command_path = commandPath.path;
|
|
266
|
+
result.command_path_confidence = commandPath.confidence;
|
|
267
|
+
}
|
|
240
268
|
|
|
241
269
|
const fragment = detectCommandFragment(normalized);
|
|
242
270
|
if (fragment) result.command_fragment = fragment;
|
package/src/extract-videos.ts
CHANGED
|
@@ -148,9 +148,51 @@ function checkYtDlp(ytdlp = YTDLP_DEFAULT): boolean {
|
|
|
148
148
|
// ── VTT parsing ──
|
|
149
149
|
|
|
150
150
|
const VTT_TIMESTAMP_RE = /^(\d{2}):(\d{2}):(\d{2})\.(\d{3}) --> /;
|
|
151
|
-
const HTML_TAG_RE = /<[^>]+>/g;
|
|
152
151
|
// Inline timestamp tags like <00:00:01.234>
|
|
153
152
|
const INLINE_TS_RE = /<\d{2}:\d{2}:\d{2}\.\d{3}>/g;
|
|
153
|
+
const VTT_CUE_TAG_RE =
|
|
154
|
+
/^<\/?(?:b|i|u|ruby|rt|c(?:\.[A-Za-z0-9_-]+)*|v(?:\s+[^<>]+)?|lang(?:\s+[A-Za-z0-9_-]+)?)>/;
|
|
155
|
+
|
|
156
|
+
function stripHtmlLikeMarkup(input: string): string {
|
|
157
|
+
let output = "";
|
|
158
|
+
let i = 0;
|
|
159
|
+
|
|
160
|
+
while (i < input.length) {
|
|
161
|
+
const char = input[i];
|
|
162
|
+
|
|
163
|
+
if (char === "<") {
|
|
164
|
+
const validCueTag = input.slice(i).match(VTT_CUE_TAG_RE);
|
|
165
|
+
if (validCueTag) {
|
|
166
|
+
i += validCueTag[0].length;
|
|
167
|
+
continue;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
let j = i + 1;
|
|
171
|
+
let lastClose = -1;
|
|
172
|
+
while (j < input.length) {
|
|
173
|
+
if (input[j] === ">") lastClose = j;
|
|
174
|
+
if (lastClose !== -1 && /\s/.test(input[j])) break;
|
|
175
|
+
j++;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
if (lastClose > i && lastClose + 1 < j) {
|
|
179
|
+
output += input.slice(lastClose + 1, j).replace(/[<>]/g, "");
|
|
180
|
+
}
|
|
181
|
+
i = j;
|
|
182
|
+
continue;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
if (char === ">") {
|
|
186
|
+
i++;
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
output += char;
|
|
191
|
+
i++;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
return output;
|
|
195
|
+
}
|
|
154
196
|
|
|
155
197
|
function vttTimestampToSeconds(text: string): number {
|
|
156
198
|
const m = text.match(VTT_TIMESTAMP_RE);
|
|
@@ -175,8 +217,8 @@ export function parseVtt(vttText: string): VttCue[] {
|
|
|
175
217
|
if (currentLines.length === 0) return;
|
|
176
218
|
const raw = currentLines.join(" ").trim();
|
|
177
219
|
if (!raw) return;
|
|
178
|
-
// Strip
|
|
179
|
-
const clean = raw.replace(INLINE_TS_RE, "")
|
|
220
|
+
// Strip inline timestamp tags and HTML-shaped cue markup.
|
|
221
|
+
const clean = stripHtmlLikeMarkup(raw.replace(INLINE_TS_RE, "")).replace(/\s+/g, " ").trim();
|
|
180
222
|
if (!clean) return;
|
|
181
223
|
// Deduplicate: skip if clean text is a suffix of what we've already emitted
|
|
182
224
|
if (prevAccumulated.endsWith(clean)) return;
|
|
@@ -186,7 +228,7 @@ export function parseVtt(vttText: string): VttCue[] {
|
|
|
186
228
|
}
|
|
187
229
|
|
|
188
230
|
for (const rawLine of lines) {
|
|
189
|
-
const line = rawLine.replace(/\r
|
|
231
|
+
const line = rawLine.replace(/\r/g, "");
|
|
190
232
|
|
|
191
233
|
if (line.startsWith("WEBVTT") || line.startsWith("Kind:") || line.startsWith("Language:")) {
|
|
192
234
|
inCueText = false;
|