@vincemakes/kiso-tools-node 0.26.1 → 0.26.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +6 -0
- package/dist/index.js +141 -269
- package/dist/search-worker.d.ts +49 -0
- package/dist/search-worker.js +136 -0
- package/package.json +2 -2
package/dist/index.d.ts
CHANGED
|
@@ -140,6 +140,12 @@ export declare function readFileTool(opts: WorkspaceToolsOptions): Tool<{
|
|
|
140
140
|
export declare function listDirTool(opts: WorkspaceToolsOptions): Tool<{
|
|
141
141
|
path?: string;
|
|
142
142
|
}>;
|
|
143
|
+
/** The instrument behind gate (d): live workers and queue depth. */
|
|
144
|
+
export declare function searchWorkerStats(): {
|
|
145
|
+
alive: number;
|
|
146
|
+
inFlight: number;
|
|
147
|
+
queued: number;
|
|
148
|
+
};
|
|
143
149
|
export declare function searchTextTool(opts: WorkspaceToolsOptions): Tool<{
|
|
144
150
|
pattern: string;
|
|
145
151
|
path?: string;
|
package/dist/index.js
CHANGED
|
@@ -19,8 +19,9 @@
|
|
|
19
19
|
*/
|
|
20
20
|
import { execFile, execFileSync, spawn } from "node:child_process";
|
|
21
21
|
import { promisify } from "node:util";
|
|
22
|
-
import { chmodSync, existsSync, linkSync,
|
|
23
|
-
import {
|
|
22
|
+
import { appendFileSync, chmodSync, existsSync, linkSync, mkdirSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, statSync, unlinkSync, writeFileSync } from "node:fs";
|
|
23
|
+
import { Worker } from "node:worker_threads";
|
|
24
|
+
import { fileURLToPath } from "node:url";
|
|
24
25
|
import { createHash } from "node:crypto";
|
|
25
26
|
import { tmpdir } from "node:os";
|
|
26
27
|
import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path";
|
|
@@ -492,6 +493,107 @@ export function listDirTool(opts) {
|
|
|
492
493
|
},
|
|
493
494
|
});
|
|
494
495
|
}
|
|
496
|
+
// ───────────────────────────── CX-1 F4: the search worker host ─────────────────────────────
|
|
497
|
+
//
|
|
498
|
+
// One Worker per call, at most SEARCH_WORKERS in flight; further calls
|
|
499
|
+
// QUEUE, and queue time counts against the call's budget. Each call
|
|
500
|
+
// carries a token, so a message from a terminated or superseded worker
|
|
501
|
+
// is ignored. Workers are unreferenced (an idle one never holds the
|
|
502
|
+
// process open) and always terminated when the call settles. Pooling is
|
|
503
|
+
// a later decision, on the startup cost gate (e) measures.
|
|
504
|
+
const SEARCH_WORKERS = 2;
|
|
505
|
+
/** The host kills the worker this long after the deadline — the room a
|
|
506
|
+
* cooperative stop (between files, counters reported) needs to land
|
|
507
|
+
* before the backstop for an uncooperative regex fires. */
|
|
508
|
+
const SEARCH_KILL_GRACE_MS = 150;
|
|
509
|
+
let searchInFlight = 0;
|
|
510
|
+
let searchAlive = 0;
|
|
511
|
+
let searchToken = 0;
|
|
512
|
+
const searchQueue = [];
|
|
513
|
+
/** The instrument behind gate (d): live workers and queue depth. */
|
|
514
|
+
export function searchWorkerStats() {
|
|
515
|
+
return { alive: searchAlive, inFlight: searchInFlight, queued: searchQueue.length };
|
|
516
|
+
}
|
|
517
|
+
function searchWorkerUrl() {
|
|
518
|
+
// Built layout: dist/index.js beside dist/search-worker.js. Under a
|
|
519
|
+
// source-mode test runner the sibling is a .ts file the Worker cannot
|
|
520
|
+
// load, so the built copy is used — tools-node tests run against a
|
|
521
|
+
// built dist, like the tui's.
|
|
522
|
+
const beside = new URL("./search-worker.js", import.meta.url);
|
|
523
|
+
if (existsSync(fileURLToPath(beside)))
|
|
524
|
+
return beside;
|
|
525
|
+
return new URL("../dist/search-worker.js", import.meta.url);
|
|
526
|
+
}
|
|
527
|
+
async function runSearchWorker(req, deadline, signal) {
|
|
528
|
+
if (signal.aborted)
|
|
529
|
+
return { kind: "aborted" };
|
|
530
|
+
// the slot — queue time is inside the budget
|
|
531
|
+
if (searchInFlight >= SEARCH_WORKERS) {
|
|
532
|
+
const waited = await new Promise((resolve) => {
|
|
533
|
+
const timer = setTimeout(() => {
|
|
534
|
+
const i = searchQueue.indexOf(wake);
|
|
535
|
+
if (i >= 0)
|
|
536
|
+
searchQueue.splice(i, 1);
|
|
537
|
+
resolve("timeout");
|
|
538
|
+
}, Math.max(0, deadline - Date.now()));
|
|
539
|
+
const onAbort = () => {
|
|
540
|
+
clearTimeout(timer);
|
|
541
|
+
const i = searchQueue.indexOf(wake);
|
|
542
|
+
if (i >= 0)
|
|
543
|
+
searchQueue.splice(i, 1);
|
|
544
|
+
resolve("aborted");
|
|
545
|
+
};
|
|
546
|
+
const wake = () => {
|
|
547
|
+
clearTimeout(timer);
|
|
548
|
+
signal.removeEventListener?.("abort", onAbort);
|
|
549
|
+
resolve("slot");
|
|
550
|
+
};
|
|
551
|
+
signal.addEventListener?.("abort", onAbort, { once: true });
|
|
552
|
+
searchQueue.push(wake);
|
|
553
|
+
});
|
|
554
|
+
if (waited !== "slot")
|
|
555
|
+
return waited === "timeout" ? { kind: "timeout" } : { kind: "aborted" };
|
|
556
|
+
}
|
|
557
|
+
searchInFlight += 1;
|
|
558
|
+
searchAlive += 1;
|
|
559
|
+
const token = ++searchToken;
|
|
560
|
+
const worker = new Worker(searchWorkerUrl());
|
|
561
|
+
worker.unref();
|
|
562
|
+
const settle = () => {
|
|
563
|
+
searchInFlight -= 1;
|
|
564
|
+
searchQueue.shift()?.();
|
|
565
|
+
};
|
|
566
|
+
return new Promise((resolve) => {
|
|
567
|
+
let done = false;
|
|
568
|
+
const finish = (outcome) => {
|
|
569
|
+
if (done)
|
|
570
|
+
return;
|
|
571
|
+
done = true;
|
|
572
|
+
clearTimeout(timer);
|
|
573
|
+
signal.removeEventListener?.("abort", onAbort);
|
|
574
|
+
void worker.terminate().then(() => {
|
|
575
|
+
searchAlive -= 1;
|
|
576
|
+
}, () => {
|
|
577
|
+
searchAlive -= 1;
|
|
578
|
+
});
|
|
579
|
+
settle();
|
|
580
|
+
resolve(outcome);
|
|
581
|
+
};
|
|
582
|
+
const onAbort = () => finish({ kind: "aborted" });
|
|
583
|
+
const timer = setTimeout(() => finish({ kind: "timeout" }), Math.max(0, deadline + SEARCH_KILL_GRACE_MS - Date.now()));
|
|
584
|
+
signal.addEventListener?.("abort", onAbort, { once: true });
|
|
585
|
+
worker.on("message", (m) => {
|
|
586
|
+
if (m.token === token)
|
|
587
|
+
finish({ kind: "done", reply: m });
|
|
588
|
+
});
|
|
589
|
+
worker.on("error", (err) => finish({ kind: "error", message: err.message }));
|
|
590
|
+
worker.on("exit", (code) => {
|
|
591
|
+
if (!done)
|
|
592
|
+
finish({ kind: "error", message: `search worker exited with code ${code}` });
|
|
593
|
+
});
|
|
594
|
+
worker.postMessage({ ...req, token });
|
|
595
|
+
});
|
|
596
|
+
}
|
|
495
597
|
export function searchTextTool(opts) {
|
|
496
598
|
return defineTool({
|
|
497
599
|
name: "search_text",
|
|
@@ -514,7 +616,7 @@ export function searchTextTool(opts) {
|
|
|
514
616
|
effects: { precommitSafe: true, concurrency: "shared" },
|
|
515
617
|
promptSnippet: "search_text — regex search over workspace files",
|
|
516
618
|
promptGuidelines: ["narrow the pattern when the result caps — never re-run a broad search"],
|
|
517
|
-
execute: async ({ pattern, path, caseSensitive }) => {
|
|
619
|
+
execute: async ({ pattern, path, caseSensitive }, ctx) => {
|
|
518
620
|
let root;
|
|
519
621
|
try {
|
|
520
622
|
root = resolveWithinRoot(opts.workspaceRoot, path ?? ".");
|
|
@@ -524,24 +626,15 @@ export function searchTextTool(opts) {
|
|
|
524
626
|
return escapeResult(err.message);
|
|
525
627
|
throw err;
|
|
526
628
|
}
|
|
527
|
-
// DC-23
|
|
528
|
-
//
|
|
529
|
-
|
|
530
|
-
// model could act on. And the "i" flag was hardcoded, so a
|
|
531
|
-
// case-sensitive search was not expressible at all.
|
|
532
|
-
let regex;
|
|
629
|
+
// DC-23: an INVALID pattern is a result the model can act on, never
|
|
630
|
+
// a crash — validated here; the worker compiles its own copy.
|
|
631
|
+
const flags = caseSensitive === true ? "" : "i";
|
|
533
632
|
try {
|
|
534
|
-
|
|
633
|
+
new RegExp(pattern, flags);
|
|
535
634
|
}
|
|
536
635
|
catch (err) {
|
|
537
636
|
return { content: `search_text failed: invalid pattern — ${err.message}`, isError: true, errorKind: "invalid_input" };
|
|
538
637
|
}
|
|
539
|
-
// DC-23: a FILE is a place text lives. The tool took only a
|
|
540
|
-
// directory and answered a file path with libuv's own words
|
|
541
|
-
// ("ENOTDIR: not a directory, scandir <path>"), which is the
|
|
542
|
-
// obvious thing to ask for — the file is already known and the
|
|
543
|
-
// question is where in it something is. The real dogfood model
|
|
544
|
-
// asked twice and learned nothing either time. One stat.
|
|
545
638
|
let single = null;
|
|
546
639
|
try {
|
|
547
640
|
if (statSync(root).isFile())
|
|
@@ -550,29 +643,6 @@ export function searchTextTool(opts) {
|
|
|
550
643
|
catch (err) {
|
|
551
644
|
return { content: `search_text failed: ${path ?? "."} — ${err.message}`, isError: true, errorKind: "invalid_input" };
|
|
552
645
|
}
|
|
553
|
-
// The walk NEVER early-aborts on the cap: the overflow note's count
|
|
554
|
-
// must be the file-true total, not a bound (the red line). The
|
|
555
|
-
// depth cap and the node_modules/dotfile skip stay.
|
|
556
|
-
const matches = [];
|
|
557
|
-
let totalMatches = 0;
|
|
558
|
-
// DC-52 — what the search did NOT look at, so the note can say so.
|
|
559
|
-
let multiLink = 0;
|
|
560
|
-
let unreadableDirs = 0;
|
|
561
|
-
// DC-54 — the same discipline for the two new refusals, and for
|
|
562
|
-
// the call budget: a silent skip is a result the model cannot
|
|
563
|
-
// tell is incomplete.
|
|
564
|
-
let skippedFiles = 0;
|
|
565
|
-
let excludedDirs = 0;
|
|
566
|
-
let filesSeen = 0;
|
|
567
|
-
// DC-49 — realpath on BOTH sides, so a symlinked HOME still
|
|
568
|
-
// matches (the reviewer's constraint (d): `/tmp` is a symlink to
|
|
569
|
-
// `/private/tmp` on darwin, and a raw string compare there is a
|
|
570
|
-
// gate that passes on one machine and not another).
|
|
571
|
-
//
|
|
572
|
-
// A root that CONTAINS the search root is not an exclusion: the
|
|
573
|
-
// caller pointed at it, and refusing would make an explicit path
|
|
574
|
-
// unservable. That is the reviewer's constraint (c), expressed
|
|
575
|
-
// where it belongs — in the predicate, not in four call sites.
|
|
576
646
|
const realOrSelf = (p) => {
|
|
577
647
|
try {
|
|
578
648
|
return realpathSync(p);
|
|
@@ -584,244 +654,46 @@ export function searchTextTool(opts) {
|
|
|
584
654
|
const searchRootReal = realOrSelf(root);
|
|
585
655
|
const excluded = (opts.excludeRoots ?? [])
|
|
586
656
|
.map(realOrSelf)
|
|
587
|
-
.filter((ex) => !(searchRootReal === ex || searchRootReal.startsWith(`${ex}/`)))
|
|
588
|
-
|
|
589
|
-
if (excluded.length === 0)
|
|
590
|
-
return false;
|
|
591
|
-
const r = realOrSelf(dir);
|
|
592
|
-
return excluded.some((ex) => r === ex || r.startsWith(`${ex}/`));
|
|
593
|
-
};
|
|
594
|
-
// An explicit flag, NOT `stoppedAt > 0`: a wall-clock budget can
|
|
595
|
-
// expire before the first file is scanned, and a zero-valued
|
|
596
|
-
// sentinel would then read as "never stopped" — the walk would
|
|
597
|
-
// still end, but silently, which is the one thing every note in
|
|
598
|
-
// this function exists to prevent.
|
|
599
|
-
let stopped = false;
|
|
600
|
-
let stoppedAt = 0;
|
|
657
|
+
.filter((ex) => !(searchRootReal === ex || searchRootReal.startsWith(`${ex}/`)))
|
|
658
|
+
.map((ex) => relative(searchRootReal, ex));
|
|
601
659
|
const maxFileBytes = opts.limits?.searchMaxFileBytes ?? SEARCH_MAX_FILE_BYTES;
|
|
602
660
|
const maxFiles = opts.limits?.searchMaxFiles ?? SEARCH_MAX_FILES;
|
|
603
661
|
const deadline = Date.now() + (opts.limits?.searchMaxMs ?? SEARCH_MAX_MS);
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
return false;
|
|
618
|
-
};
|
|
619
|
-
// R3 — the walk YIELDS. It was `readdirSync` + `readFileSync` all
|
|
620
|
-
// the way down inside an `async` body, which is the shape that
|
|
621
|
-
// blocks Node's event loop for the whole traversal: measured at
|
|
622
|
-
// 18.4 seconds over a home directory, during which no timer
|
|
623
|
-
// fires, no frame paints and the `working` mark freezes solid.
|
|
624
|
-
// A user reasonably reads a frozen liveness mark as a crash.
|
|
625
|
-
//
|
|
626
|
-
// The fix is not "make it faster" — a big tree is legitimately
|
|
627
|
-
// slow. It is to stop OWNING the loop: `fs.promises.readdir`,
|
|
628
|
-
// and a yield every YIELD_EVERY files.
|
|
629
|
-
//
|
|
630
|
-
// DC-54 corrects what this comment used to claim next — that
|
|
631
|
-
// the yield also stopped "a long read stretch" monopolising the
|
|
632
|
-
// loop. It never did and could not: `breathe()` runs BETWEEN
|
|
633
|
-
// files, and nothing preempts a single synchronous read once
|
|
634
|
-
// entered. R3 bounded the traversal; the FILE stayed unbounded
|
|
635
|
-
// until DC-54 ①–③ above, and one 994 GB image was enough to
|
|
636
|
-
// stop the process dead with this yield fully in place.
|
|
637
|
-
let sinceYield = 0;
|
|
638
|
-
const breathe = async () => {
|
|
639
|
-
sinceYield += 1;
|
|
640
|
-
if (sinceYield < YIELD_EVERY)
|
|
641
|
-
return;
|
|
642
|
-
sinceYield = 0;
|
|
643
|
-
await new Promise((r) => setImmediate(r));
|
|
644
|
-
};
|
|
645
|
-
// DC-23: the per-file scan is its own function now, because the
|
|
646
|
-
// single-file path and the walk must scan a file the SAME way —
|
|
647
|
-
// same inode boundary, same cap accounting, same excerpt shape.
|
|
648
|
-
// Two copies would be two answers to "what does searching this
|
|
649
|
-
// file mean".
|
|
650
|
-
const scanFile = async (full) => {
|
|
651
|
-
if (outOfBudget())
|
|
652
|
-
return;
|
|
653
|
-
filesSeen += 1;
|
|
654
|
-
await breathe();
|
|
655
|
-
try {
|
|
656
|
-
// DC-52 — SEARCH DOES NOT RUN THE INODE GUARD.
|
|
657
|
-
//
|
|
658
|
-
// Round 8 gave search the same inode boundary read_file
|
|
659
|
-
// has, and the boundary is right; what was wrong is the
|
|
660
|
-
// price. The guard's verification is a `find` over the
|
|
661
|
-
// whole workspace root, once per multi-link file — with
|
|
662
|
-
// the root at `~` that is tens of seconds each, and it
|
|
663
|
-
// ran on the owner's machine for eight minutes without
|
|
664
|
-
// finishing. A search returns a 160-character excerpt of
|
|
665
|
-
// a line. No excerpt is worth a disk traversal.
|
|
666
|
-
//
|
|
667
|
-
// So a multi-link file is SKIPPED, and counted, and the
|
|
668
|
-
// count is said. That is fail-closed at zero cost: the
|
|
669
|
-
// external-link case the guard exists for is refused
|
|
670
|
-
// exactly as before, and the legal case is refused too,
|
|
671
|
-
// which is a loss of coverage rather than of safety.
|
|
672
|
-
// read_file keeps the guard (bounded, above), and that
|
|
673
|
-
// is where the file's contents can actually be had.
|
|
674
|
-
const st = statSync(full);
|
|
675
|
-
if (st.nlink > 1) {
|
|
676
|
-
multiLink += 1;
|
|
677
|
-
return;
|
|
678
|
-
}
|
|
679
|
-
// DC-54 ① — SIZE, decided before anything is opened.
|
|
680
|
-
//
|
|
681
|
-
// This is the line that was missing when a workspace
|
|
682
|
-
// rooted at `~` met `Docker.raw`: 994 GB went into
|
|
683
|
-
// `readFileSync(full, "utf8")` and the process never came
|
|
684
|
-
// back. Measured on the owner's machine, without it: a
|
|
685
|
-
// 467 MB `.mov` read SUCCESSFULLY, 3,817 ms of dead loop
|
|
686
|
-
// and 2.06 GB of RSS, split into 1,823,112 "lines".
|
|
687
|
-
if (st.size > maxFileBytes) {
|
|
688
|
-
skippedFiles += 1;
|
|
689
|
-
return;
|
|
690
|
-
}
|
|
691
|
-
// DC-54 ② — BINARY, decided from the head alone.
|
|
692
|
-
//
|
|
693
|
-
// One open serves both the sniff and the read. The sniff
|
|
694
|
-
// passes an explicit position, which by contract leaves
|
|
695
|
-
// the handle's own position at 0, so `readFile()` below
|
|
696
|
-
// still sees the whole file.
|
|
697
|
-
const fh = await open(full, "r");
|
|
698
|
-
let text;
|
|
699
|
-
try {
|
|
700
|
-
const headLen = Math.min(BINARY_SNIFF_BYTES, st.size);
|
|
701
|
-
const head = Buffer.alloc(headLen);
|
|
702
|
-
if (headLen > 0) {
|
|
703
|
-
// An explicit `position` leaves the handle's own
|
|
704
|
-
// position at 0, so the whole-file read below still
|
|
705
|
-
// starts at byte 0. The gate proves it with a
|
|
706
|
-
// needle placed PAST the sniff window: were the
|
|
707
|
-
// position to advance, every line number in this
|
|
708
|
-
// file's matches would be silently wrong.
|
|
709
|
-
await fh.read(head, 0, headLen, 0);
|
|
710
|
-
if (head.includes(0)) {
|
|
711
|
-
skippedFiles += 1;
|
|
712
|
-
return;
|
|
713
|
-
}
|
|
714
|
-
}
|
|
715
|
-
// DC-54 ③ — the read is ASYNCHRONOUS. R3 made the walk
|
|
716
|
-
// yield and its comment claimed that stopped a long
|
|
717
|
-
// read owning the loop; it never could. `breathe()`
|
|
718
|
-
// runs BETWEEN files, and nothing preempts a
|
|
719
|
-
// `readFileSync` once entered. Bounded above by ①, so
|
|
720
|
-
// this is at most a 1 MiB read that shares the loop.
|
|
721
|
-
//
|
|
722
|
-
// A file that fits inside the sniff window is already
|
|
723
|
-
// entirely in `head`: reading it a second time would
|
|
724
|
-
// double the syscalls for the commonest small file.
|
|
725
|
-
text = (st.size <= headLen ? head : await fh.readFile()).toString("utf8");
|
|
726
|
-
}
|
|
727
|
-
finally {
|
|
728
|
-
await fh.close();
|
|
729
|
-
}
|
|
730
|
-
for (const [i, line] of text.split("\n").entries()) {
|
|
731
|
-
if (regex.test(line)) {
|
|
732
|
-
totalMatches += 1;
|
|
733
|
-
if (matches.length < MAX_SEARCH_MATCHES) {
|
|
734
|
-
matches.push(`${full}:${i + 1}: ${line.trim().slice(0, 160)}`);
|
|
735
|
-
}
|
|
736
|
-
}
|
|
737
|
-
}
|
|
738
|
-
}
|
|
739
|
-
catch {
|
|
740
|
-
// unreadable file — skip
|
|
741
|
-
}
|
|
742
|
-
};
|
|
743
|
-
const walk = async (dir, depth) => {
|
|
744
|
-
if (depth > 8 || outOfBudget())
|
|
745
|
-
return;
|
|
746
|
-
// DC-52 — a directory the OS REFUSES is skipped, not fatal.
|
|
747
|
-
//
|
|
748
|
-
// An unreadable FILE has always been skipped (the catch in
|
|
749
|
-
// scanFile); an unreadable DIRECTORY threw out of the walk
|
|
750
|
-
// and failed the whole tool. On macOS that is not an edge
|
|
751
|
-
// case — `~/Library/Accounts` and its neighbours are TCC
|
|
752
|
-
// protected, so a search anywhere under `~` died on one of
|
|
753
|
-
// them. The asymmetry was the defect: same fact, same
|
|
754
|
-
// remedy.
|
|
755
|
-
let entries;
|
|
756
|
-
try {
|
|
757
|
-
entries = await readdir(dir, { withFileTypes: true });
|
|
758
|
-
}
|
|
759
|
-
catch (err) {
|
|
760
|
-
const code = err.code;
|
|
761
|
-
if (code === "EACCES" || code === "EPERM") {
|
|
762
|
-
unreadableDirs += 1;
|
|
763
|
-
return;
|
|
764
|
-
}
|
|
765
|
-
throw err;
|
|
766
|
-
}
|
|
767
|
-
for (const entry of entries) {
|
|
768
|
-
if (outOfBudget())
|
|
769
|
-
return;
|
|
770
|
-
if (entry.name.startsWith(".") || entry.name === "node_modules")
|
|
771
|
-
continue;
|
|
772
|
-
const full = join(dir, entry.name);
|
|
773
|
-
if (entry.isDirectory()) {
|
|
774
|
-
// DC-49 — a walk does not DESCEND into an excluded root.
|
|
775
|
-
// Reaching it from above is what is refused; being
|
|
776
|
-
// pointed at it is not (see `excluded` below, which is
|
|
777
|
-
// seeded from the SEARCH ROOT and therefore empty when
|
|
778
|
-
// the root is itself excluded).
|
|
779
|
-
if (isExcluded(full)) {
|
|
780
|
-
excludedDirs += 1;
|
|
781
|
-
continue;
|
|
782
|
-
}
|
|
783
|
-
await walk(full, depth + 1);
|
|
784
|
-
}
|
|
785
|
-
else if (entry.isFile())
|
|
786
|
-
await scanFile(full);
|
|
787
|
-
}
|
|
788
|
-
};
|
|
789
|
-
try {
|
|
790
|
-
if (single !== null)
|
|
791
|
-
await scanFile(single);
|
|
792
|
-
else
|
|
793
|
-
await walk(root, 0);
|
|
794
|
-
}
|
|
795
|
-
catch (err) {
|
|
796
|
-
return { content: `search_text failed: ${err.message}`, isError: true, errorKind: "fatal" };
|
|
797
|
-
}
|
|
662
|
+
// CX-1 F4 (audit F4): the walk-and-match runs on its OWN thread, which
|
|
663
|
+
// the deadline and the abort both TERMINATE. A catastrophic regex used
|
|
664
|
+
// to block this loop — no budget check, timer or abort could run.
|
|
665
|
+
const outcome = await runSearchWorker({ token: 0, root: searchRootReal, single, pattern, flags, excluded, maxFileBytes, maxFiles, deadline, maxMatches: MAX_SEARCH_MATCHES, sniffBytes: BINARY_SNIFF_BYTES }, deadline, ctx.signal);
|
|
666
|
+
if (outcome.kind === "aborted")
|
|
667
|
+
return { content: "search_text aborted", isError: true, errorKind: "fatal" };
|
|
668
|
+
if (outcome.kind === "error")
|
|
669
|
+
return { content: `search_text failed: ${outcome.message}`, isError: true, errorKind: "fatal" };
|
|
670
|
+
const r = outcome.reply;
|
|
671
|
+
if (r?.error !== undefined)
|
|
672
|
+
return { content: `search_text failed: ${r.error}`, isError: true, errorKind: "fatal" };
|
|
673
|
+
const matches = r?.matches ?? [];
|
|
674
|
+
const stopped = outcome.kind === "timeout" || (r?.stopped ?? false);
|
|
798
675
|
let content = matches.length ? cap(matches.join("\n")) : "(no matches)";
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
// that always fires says nothing.
|
|
807
|
-
// DC-49 — what a WALK did not enter, in the same discipline the
|
|
808
|
-
// file skips already follow: a scoped result that reads as total
|
|
809
|
-
// is one the model cannot tell is scoped.
|
|
810
|
-
if (excludedDirs > 0) {
|
|
811
|
-
content += `\n… ${excludedDirs} ${excludedDirs === 1 ? "directory" : "directories"} excluded`;
|
|
676
|
+
if (r !== undefined) {
|
|
677
|
+
if (r.multiLink > 0)
|
|
678
|
+
content += `\n… ${r.multiLink} multi-link ${r.multiLink === 1 ? "file" : "files"} skipped (read_file verifies them individually)`;
|
|
679
|
+
if (r.unreadableDirs > 0)
|
|
680
|
+
content += `\n… ${r.unreadableDirs} unreadable ${r.unreadableDirs === 1 ? "directory" : "directories"} skipped`;
|
|
681
|
+
if (r.excludedDirs > 0)
|
|
682
|
+
content += `\n… ${r.excludedDirs} ${r.excludedDirs === 1 ? "directory" : "directories"} excluded`;
|
|
812
683
|
}
|
|
813
|
-
if (skippedFiles > 0 || stopped) {
|
|
684
|
+
if ((r?.skippedFiles ?? 0) > 0 || stopped) {
|
|
814
685
|
const parts = [];
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
686
|
+
const skipped = r?.skippedFiles ?? 0;
|
|
687
|
+
if (skipped > 0)
|
|
688
|
+
parts.push(`${skipped} ${skipped === 1 ? "file" : "files"} skipped (large or binary)`);
|
|
689
|
+
if (outcome.kind === "timeout")
|
|
690
|
+
parts.push("stopped — the search budget elapsed before it finished (narrow the path or the pattern)");
|
|
691
|
+
else if (stopped)
|
|
692
|
+
parts.push(`stopped after ${r?.stoppedAt ?? 0} ${(r?.stoppedAt ?? 0) === 1 ? "file" : "files"} — narrow the path`);
|
|
819
693
|
content += `\n… ${parts.join(" · ")}`;
|
|
820
694
|
}
|
|
821
|
-
if (totalMatches > matches.length) {
|
|
822
|
-
|
|
823
|
-
// continuation (narrow the pattern for more).
|
|
824
|
-
content += `\n… ${matches.length} of ${totalMatches} matches shown (narrow the pattern for more)`;
|
|
695
|
+
if (r !== undefined && r.totalMatches > matches.length) {
|
|
696
|
+
content += `\n… ${matches.length} of ${r.totalMatches} matches shown (narrow the pattern for more)`;
|
|
825
697
|
}
|
|
826
698
|
return { content, isError: false };
|
|
827
699
|
},
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CX-1 F4 — the search walk-and-match, on its own thread.
|
|
3
|
+
*
|
|
4
|
+
* `search_text` compiles a model-supplied regex and runs it per line;
|
|
5
|
+
* a catastrophic pattern (`(a+)+$` on 33 characters) blocks the event
|
|
6
|
+
* loop, and no budget check, timer or abort can run (audit F4). The
|
|
7
|
+
* walk lives here, in a `worker_threads` Worker the main thread can
|
|
8
|
+
* TERMINATE: the deadline and the abort signal both kill it. The root
|
|
9
|
+
* is resolved and confined on the main side (`resolveWithinRoot`); this
|
|
10
|
+
* side only walks under it, with the same skip rules as before (dot
|
|
11
|
+
* paths, node_modules, depth 8, excluded dirs, binary sniff, per-file
|
|
12
|
+
* bytes, per-call files).
|
|
13
|
+
*
|
|
14
|
+
* Messages: the parent posts one `SearchRequest`; the worker answers
|
|
15
|
+
* one `SearchReply` and exits. A call token rides both ways so a late
|
|
16
|
+
* message from a superseded worker is ignored.
|
|
17
|
+
*/
|
|
18
|
+
export interface SearchRequest {
|
|
19
|
+
readonly token: number;
|
|
20
|
+
readonly root: string;
|
|
21
|
+
/** a single file to scan instead of walking `root` */
|
|
22
|
+
readonly single: string | null;
|
|
23
|
+
readonly pattern: string;
|
|
24
|
+
readonly flags: string;
|
|
25
|
+
readonly excluded: readonly string[];
|
|
26
|
+
readonly maxFileBytes: number;
|
|
27
|
+
readonly maxFiles: number;
|
|
28
|
+
/** the call's wall-clock deadline (epoch ms): the walk stops COOPERATIVELY
|
|
29
|
+
* between files and reports its counters (the DC-54 note); the host's
|
|
30
|
+
* terminate is the backstop for the one thing that cannot cooperate —
|
|
31
|
+
* a regex that never returns */
|
|
32
|
+
readonly deadline: number;
|
|
33
|
+
readonly maxMatches: number;
|
|
34
|
+
readonly sniffBytes: number;
|
|
35
|
+
}
|
|
36
|
+
export interface SearchReply {
|
|
37
|
+
readonly token: number;
|
|
38
|
+
readonly matches: string[];
|
|
39
|
+
readonly totalMatches: number;
|
|
40
|
+
readonly filesSeen: number;
|
|
41
|
+
readonly skippedFiles: number;
|
|
42
|
+
readonly multiLink: number;
|
|
43
|
+
readonly unreadableDirs: number;
|
|
44
|
+
readonly excludedDirs: number;
|
|
45
|
+
readonly stopped: boolean;
|
|
46
|
+
readonly stoppedAt: number;
|
|
47
|
+
readonly error?: string;
|
|
48
|
+
}
|
|
49
|
+
export declare function runSearch(req: SearchRequest): Promise<SearchReply>;
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CX-1 F4 — the search walk-and-match, on its own thread.
|
|
3
|
+
*
|
|
4
|
+
* `search_text` compiles a model-supplied regex and runs it per line;
|
|
5
|
+
* a catastrophic pattern (`(a+)+$` on 33 characters) blocks the event
|
|
6
|
+
* loop, and no budget check, timer or abort can run (audit F4). The
|
|
7
|
+
* walk lives here, in a `worker_threads` Worker the main thread can
|
|
8
|
+
* TERMINATE: the deadline and the abort signal both kill it. The root
|
|
9
|
+
* is resolved and confined on the main side (`resolveWithinRoot`); this
|
|
10
|
+
* side only walks under it, with the same skip rules as before (dot
|
|
11
|
+
* paths, node_modules, depth 8, excluded dirs, binary sniff, per-file
|
|
12
|
+
* bytes, per-call files).
|
|
13
|
+
*
|
|
14
|
+
* Messages: the parent posts one `SearchRequest`; the worker answers
|
|
15
|
+
* one `SearchReply` and exits. A call token rides both ways so a late
|
|
16
|
+
* message from a superseded worker is ignored.
|
|
17
|
+
*/
|
|
18
|
+
import { open, readdir } from "node:fs/promises";
|
|
19
|
+
import { join, relative } from "node:path";
|
|
20
|
+
import { isMainThread, parentPort } from "node:worker_threads";
|
|
21
|
+
export async function runSearch(req) {
|
|
22
|
+
const regex = new RegExp(req.pattern, req.flags);
|
|
23
|
+
const matches = [];
|
|
24
|
+
let totalMatches = 0;
|
|
25
|
+
let filesSeen = 0;
|
|
26
|
+
let skippedFiles = 0;
|
|
27
|
+
let multiLink = 0;
|
|
28
|
+
let unreadableDirs = 0;
|
|
29
|
+
let excludedDirs = 0;
|
|
30
|
+
let stopped = false;
|
|
31
|
+
let stoppedAt = 0;
|
|
32
|
+
const isExcluded = (full) => {
|
|
33
|
+
const r = relative(req.root, full);
|
|
34
|
+
return req.excluded.some((ex) => r === ex || r.startsWith(`${ex}/`));
|
|
35
|
+
};
|
|
36
|
+
const outOfBudget = () => {
|
|
37
|
+
if (stopped)
|
|
38
|
+
return true;
|
|
39
|
+
if (filesSeen >= req.maxFiles || Date.now() > req.deadline) {
|
|
40
|
+
stopped = true;
|
|
41
|
+
stoppedAt = filesSeen;
|
|
42
|
+
return true;
|
|
43
|
+
}
|
|
44
|
+
return false;
|
|
45
|
+
};
|
|
46
|
+
const scanFile = async (full) => {
|
|
47
|
+
if (outOfBudget())
|
|
48
|
+
return;
|
|
49
|
+
filesSeen += 1;
|
|
50
|
+
try {
|
|
51
|
+
const fh = await open(full, "r");
|
|
52
|
+
let text;
|
|
53
|
+
try {
|
|
54
|
+
const st = await fh.stat();
|
|
55
|
+
if (st.nlink > 1) {
|
|
56
|
+
multiLink += 1;
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
if (st.size > req.maxFileBytes) {
|
|
60
|
+
skippedFiles += 1;
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
const headLen = Math.min(req.sniffBytes, st.size);
|
|
64
|
+
const head = Buffer.alloc(headLen);
|
|
65
|
+
if (headLen > 0) {
|
|
66
|
+
await fh.read(head, 0, headLen, 0);
|
|
67
|
+
if (head.includes(0)) {
|
|
68
|
+
skippedFiles += 1;
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
text = (st.size <= headLen ? head : await fh.readFile()).toString("utf8");
|
|
73
|
+
}
|
|
74
|
+
finally {
|
|
75
|
+
await fh.close();
|
|
76
|
+
}
|
|
77
|
+
for (const [i, line] of text.split("\n").entries()) {
|
|
78
|
+
if (regex.test(line)) {
|
|
79
|
+
totalMatches += 1;
|
|
80
|
+
if (matches.length < req.maxMatches)
|
|
81
|
+
matches.push(`${full}:${i + 1}: ${line.trim().slice(0, 160)}`);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
// unreadable file: skipped, like before
|
|
87
|
+
}
|
|
88
|
+
};
|
|
89
|
+
const walk = async (dir, depth) => {
|
|
90
|
+
if (depth > 8 || outOfBudget())
|
|
91
|
+
return;
|
|
92
|
+
let entries;
|
|
93
|
+
try {
|
|
94
|
+
entries = await readdir(dir, { withFileTypes: true });
|
|
95
|
+
}
|
|
96
|
+
catch (err) {
|
|
97
|
+
const code = err.code;
|
|
98
|
+
if (code === "EACCES" || code === "EPERM") {
|
|
99
|
+
unreadableDirs += 1;
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
throw err;
|
|
103
|
+
}
|
|
104
|
+
for (const entry of entries) {
|
|
105
|
+
if (outOfBudget())
|
|
106
|
+
return;
|
|
107
|
+
if (entry.name.startsWith(".") || entry.name === "node_modules")
|
|
108
|
+
continue;
|
|
109
|
+
const full = join(dir, entry.name);
|
|
110
|
+
if (entry.isDirectory()) {
|
|
111
|
+
if (isExcluded(full)) {
|
|
112
|
+
excludedDirs += 1;
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
await walk(full, depth + 1);
|
|
116
|
+
}
|
|
117
|
+
else if (entry.isFile())
|
|
118
|
+
await scanFile(full);
|
|
119
|
+
}
|
|
120
|
+
};
|
|
121
|
+
try {
|
|
122
|
+
if (req.single !== null)
|
|
123
|
+
await scanFile(req.single);
|
|
124
|
+
else
|
|
125
|
+
await walk(req.root, 0);
|
|
126
|
+
}
|
|
127
|
+
catch (err) {
|
|
128
|
+
return { token: req.token, matches, totalMatches, filesSeen, skippedFiles, multiLink, unreadableDirs, excludedDirs, stopped, stoppedAt, error: err.message };
|
|
129
|
+
}
|
|
130
|
+
return { token: req.token, matches, totalMatches, filesSeen, skippedFiles, multiLink, unreadableDirs, excludedDirs, stopped, stoppedAt };
|
|
131
|
+
}
|
|
132
|
+
if (!isMainThread && parentPort !== null) {
|
|
133
|
+
parentPort.once("message", (req) => {
|
|
134
|
+
void runSearch(req).then((reply) => parentPort.postMessage(reply), (err) => parentPort.postMessage({ token: req.token, matches: [], totalMatches: 0, filesSeen: 0, skippedFiles: 0, multiLink: 0, unreadableDirs: 0, excludedDirs: 0, stopped: false, stoppedAt: 0, error: err.message }));
|
|
135
|
+
});
|
|
136
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vincemakes/kiso-tools-node",
|
|
3
|
-
"version": "0.26.
|
|
3
|
+
"version": "0.26.2",
|
|
4
4
|
"description": "kiso coding tools for Node hosts — read file, list directory, search text, write/edit file, shell command.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"test": "vitest run"
|
|
22
22
|
},
|
|
23
23
|
"dependencies": {
|
|
24
|
-
"@vincemakes/kiso-core": "0.26.
|
|
24
|
+
"@vincemakes/kiso-core": "0.26.2"
|
|
25
25
|
},
|
|
26
26
|
"devDependencies": {
|
|
27
27
|
"@types/node": "^26.1.2",
|