@nightloom/isy 1.0.5 → 1.0.6-next.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -2
- package/dist/agents/claude.js +3 -2
- package/dist/agents/claude.js.map +1 -1
- package/dist/agents/codex.js +14 -5
- package/dist/agents/codex.js.map +1 -1
- package/dist/agents/index.d.ts +14 -10
- package/dist/agents/index.js +65 -15
- package/dist/agents/index.js.map +1 -1
- package/dist/agents/kimi.js +26 -8
- package/dist/agents/kimi.js.map +1 -1
- package/dist/agents/types.d.ts +12 -3
- package/dist/commands/commit.d.ts +6 -0
- package/dist/commands/commit.js +326 -16
- package/dist/commands/commit.js.map +1 -1
- package/dist/commands/health.js +1 -1
- package/dist/commands/health.js.map +1 -1
- package/dist/commands/init.js +30 -5
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/status.d.ts +2 -0
- package/dist/commands/status.js +7 -4
- package/dist/commands/status.js.map +1 -1
- package/dist/commands/upload.d.ts +8 -1
- package/dist/commands/upload.js +46 -39
- package/dist/commands/upload.js.map +1 -1
- package/dist/detectors.d.ts +32 -0
- package/dist/detectors.js +279 -72
- package/dist/detectors.js.map +1 -1
- package/dist/git.d.ts +52 -0
- package/dist/git.js +166 -9
- package/dist/git.js.map +1 -1
- package/dist/githook.d.ts +18 -7
- package/dist/githook.js +147 -17
- package/dist/githook.js.map +1 -1
- package/dist/hook.d.ts +22 -2
- package/dist/hook.js +28 -3
- package/dist/hook.js.map +1 -1
- package/dist/index.js +14 -5
- package/dist/index.js.map +1 -1
- package/dist/mask-paths.d.ts +4 -3
- package/dist/mask-paths.js +4 -3
- package/dist/mask-paths.js.map +1 -1
- package/dist/parser.d.ts +58 -9
- package/dist/parser.js +337 -58
- package/dist/parser.js.map +1 -1
- package/dist/paths.d.ts +19 -1
- package/dist/paths.js +28 -3
- package/dist/paths.js.map +1 -1
- package/dist/redact.d.ts +2 -0
- package/dist/redact.js +956 -97
- package/dist/redact.js.map +1 -1
- package/dist/stage0.js +27 -8
- package/dist/stage0.js.map +1 -1
- package/dist/state.d.ts +15 -0
- package/dist/state.js +8 -0
- package/dist/state.js.map +1 -1
- package/dist/test/check.test.js +1 -1
- package/dist/test/check.test.js.map +1 -1
- package/dist/test/codex.test.js +27 -4
- package/dist/test/codex.test.js.map +1 -1
- package/dist/test/commit.test.js +1466 -6
- package/dist/test/commit.test.js.map +1 -1
- package/dist/test/e2e-harness.test.d.ts +1 -0
- package/dist/test/e2e-harness.test.js +53 -0
- package/dist/test/e2e-harness.test.js.map +1 -0
- package/dist/test/git.test.js +72 -2
- package/dist/test/git.test.js.map +1 -1
- package/dist/test/hook.test.js +138 -3
- package/dist/test/hook.test.js.map +1 -1
- package/dist/test/init.test.js +60 -1
- package/dist/test/init.test.js.map +1 -1
- package/dist/test/kimi-config.test.js +27 -5
- package/dist/test/kimi-config.test.js.map +1 -1
- package/dist/test/kimi-hook.test.js +1 -1
- package/dist/test/kimi-hook.test.js.map +1 -1
- package/dist/test/kimi-listing.test.js +42 -2
- package/dist/test/kimi-listing.test.js.map +1 -1
- package/dist/test/kimi.test.js +26 -2
- package/dist/test/kimi.test.js.map +1 -1
- package/dist/test/parser.test.js +527 -1
- package/dist/test/parser.test.js.map +1 -1
- package/dist/test/redact.test.js +2442 -14
- package/dist/test/redact.test.js.map +1 -1
- package/dist/test/stage0.test.js +617 -1
- package/dist/test/stage0.test.js.map +1 -1
- package/dist/test/sweep.test.js +99 -3
- package/dist/test/sweep.test.js.map +1 -1
- package/package.json +1 -1
- package/src/agents/claude.ts +3 -2
- package/src/agents/codex.ts +14 -5
- package/src/agents/index.ts +71 -16
- package/src/agents/kimi.ts +28 -8
- package/src/agents/types.ts +12 -3
- package/src/commands/commit.ts +354 -18
- package/src/commands/health.ts +1 -1
- package/src/commands/init.ts +37 -7
- package/src/commands/status.ts +9 -4
- package/src/commands/upload.ts +54 -54
- package/src/detectors.ts +284 -75
- package/src/git.ts +175 -9
- package/src/githook.ts +151 -19
- package/src/hook.ts +30 -3
- package/src/index.ts +15 -5
- package/src/mask-paths.ts +5 -4
- package/src/parser.ts +355 -51
- package/src/paths.ts +30 -3
- package/src/redact.ts +959 -100
- package/src/stage0.ts +28 -8
- package/src/state.ts +18 -0
package/dist/parser.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { posix } from "node:path";
|
|
2
2
|
import { fileLines } from "./lines.js";
|
|
3
|
+
import { REDACTION_MARKER } from "./mask-paths.js";
|
|
3
4
|
export const EDIT_TOOLS = new Set(["Edit", "Write", "MultiEdit", "NotebookEdit"]);
|
|
4
5
|
export const LOOKUP_TOOLS = new Set(["Read", "Grep", "Glob"]);
|
|
5
6
|
/**
|
|
@@ -80,6 +81,8 @@ function createState() {
|
|
|
80
81
|
delegatedEdits: false,
|
|
81
82
|
delegated: new Set(),
|
|
82
83
|
counted: new Set(),
|
|
84
|
+
shell: { left: SHELL_CHARS },
|
|
85
|
+
structures: { left: STRUCTURES },
|
|
83
86
|
};
|
|
84
87
|
}
|
|
85
88
|
function blockText(block) {
|
|
@@ -156,8 +159,10 @@ function pushFileEdits(state, use) {
|
|
|
156
159
|
});
|
|
157
160
|
}
|
|
158
161
|
const existing = state.fileEdits.get(filePath);
|
|
162
|
+
// One at a time, as spreading a long `edits` overflows the call's arguments.
|
|
159
163
|
if (existing)
|
|
160
|
-
|
|
164
|
+
for (const entry of entries)
|
|
165
|
+
existing.push(entry);
|
|
161
166
|
else
|
|
162
167
|
state.fileEdits.set(filePath, entries);
|
|
163
168
|
}
|
|
@@ -167,10 +172,11 @@ function pushLookup(state, use) {
|
|
|
167
172
|
state.filesRead.add(target);
|
|
168
173
|
}
|
|
169
174
|
/**
|
|
170
|
-
* A heredoc opener: `<<EOF`, `<<-EOF`, `<<'PY'`. The delimiter is what
|
|
171
|
-
* body, and quoting it changes nothing about where the body stops.
|
|
175
|
+
* A heredoc opener: `<<EOF`, `<<-EOF`, `<<'PY'`, `<<\EOF`. The delimiter is what
|
|
176
|
+
* ends the body, and quoting it changes nothing about where the body stops. `<<<`
|
|
177
|
+
* is a here-string, whose word is the whole of its input, and it has no body.
|
|
172
178
|
*/
|
|
173
|
-
export const HEREDOC =
|
|
179
|
+
export const HEREDOC = /(?<!<)<<-?\s*\\?(['"]?)([A-Za-z_][A-Za-z0-9_]*)\1/;
|
|
174
180
|
/**
|
|
175
181
|
* The command with every heredoc body cut out.
|
|
176
182
|
*
|
|
@@ -204,9 +210,8 @@ export function withoutHeredocs(command) {
|
|
|
204
210
|
* inside a search pattern is the commonest way a `|` shows up in a command that
|
|
205
211
|
* runs nothing at all.
|
|
206
212
|
*
|
|
207
|
-
* ponytail: quotes nest as a single state
|
|
208
|
-
*
|
|
209
|
-
* dependency, and this is here to stop three regexes from lying.
|
|
213
|
+
* ponytail: quotes nest as a single state. Real parsing is a dependency, and this
|
|
214
|
+
* is here to stop three regexes from lying.
|
|
210
215
|
*/
|
|
211
216
|
export function commandSegments(command) {
|
|
212
217
|
const segments = [];
|
|
@@ -222,9 +227,16 @@ export function commandSegments(command) {
|
|
|
222
227
|
});
|
|
223
228
|
for (let i = 0; i < flat.length; i += 1) {
|
|
224
229
|
const char = flat[i];
|
|
230
|
+
// What a backslash escapes is no quote and no operator, except in single quotes;
|
|
231
|
+
// at the end of a line it only joins that line to the next.
|
|
232
|
+
if (char === "\\" && quote !== "'" && flat[i + 1] !== "\n") {
|
|
233
|
+
current += char + (flat[i + 1] ?? "");
|
|
234
|
+
i += 1;
|
|
235
|
+
continue;
|
|
236
|
+
}
|
|
225
237
|
if (quote !== undefined) {
|
|
226
238
|
current += char;
|
|
227
|
-
if (char === quote
|
|
239
|
+
if (char === quote)
|
|
228
240
|
quote = undefined;
|
|
229
241
|
continue;
|
|
230
242
|
}
|
|
@@ -351,13 +363,68 @@ export function mayWrite(name, input) {
|
|
|
351
363
|
return false;
|
|
352
364
|
return keys.length === 0 || told;
|
|
353
365
|
}
|
|
366
|
+
/** What an MCP tool's name opens with when all it does is read. */
|
|
367
|
+
const MCP_READ_VERBS = new Set([
|
|
368
|
+
"read", "get", "list", "search", "find", "fetch", "query", "show", "view", "describe", "resolve", "validate", "wait",
|
|
369
|
+
]);
|
|
370
|
+
/**
|
|
371
|
+
* Whether an MCP tool's name says it leaves everything as it found it.
|
|
372
|
+
*
|
|
373
|
+
* ponytail: judged by names, as `mayWrite` is. A tool that opens with none of these
|
|
374
|
+
* reads as one nobody can answer for, a browser's `navigate` included, and its
|
|
375
|
+
* session keeps a commit on another branch. Add the word if one such proves to move
|
|
376
|
+
* sessions to the wrong pull request.
|
|
377
|
+
*/
|
|
378
|
+
export function mcpOnlyReads(name) {
|
|
379
|
+
return MCP_READ_VERBS.has(nameWords(name.slice(name.lastIndexOf("__") + 2))[0]);
|
|
380
|
+
}
|
|
354
381
|
/** A redirect into a file: `> a.ts`, `>> log`, `2> out`. Not `2>&1`, not `/dev/null`. */
|
|
355
382
|
const REDIRECT_WRITE = /(?:^|[^<>&\d])\d*>>?(?!&)\s*(?!\/dev\/|\/tmp\/)[^\s&|;<>]/;
|
|
356
383
|
const TEE_WRITE = /\btee\s+(?:-a\s+)?(?!\/dev\/|\/tmp\/)[^\s&|;-]/;
|
|
357
384
|
const FILE_MUTATORS = /^\s*(?:sudo\s+)?(?:patch|truncate|mv|cp|git\s+(?:apply|mv|rm|restore|checkout\s+--))\b/;
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
const
|
|
385
|
+
/** What can stand in front of a command and leave it the same command: `do sed -i …`. */
|
|
386
|
+
const LEADING_WORDS = /^\s*(?:(?:sudo|if|then|elif|else|while|until|do)\s+)*/;
|
|
387
|
+
const IN_PLACE_EDITORS = /^(g?sed|g?awk|perl|ruby)(?=\s|$)/;
|
|
388
|
+
/**
|
|
389
|
+
* Theirs is among the flags in front, in a cluster that may hold a digit: `perl -0pi`.
|
|
390
|
+
* Only behind flags that take nothing glued to them, as `-Ilib`, `-rdigest` and
|
|
391
|
+
* `-Mstrict` hold an `i` that is no flag.
|
|
392
|
+
*/
|
|
393
|
+
const PERL_IN_PLACE = /^\S+(?:\s+-\S+)*?\s+-[0-9acdlnpsStTuUvwWXy]*i/;
|
|
394
|
+
/** To awk `-i` loads a library, and the one that rewrites is gawk's `inplace`, unless another flag takes the word. */
|
|
395
|
+
const AWK_IN_PLACE = /(?:^|\s)(?=-)(?<!\s-[efFv]\s+)(?:-i\s*|--include[=\s]\s*)inplace(?:\s|$)/;
|
|
396
|
+
/**
|
|
397
|
+
* What quotes hold, emptied out: an operator in there is an argument, as in
|
|
398
|
+
* `grep "->" a`. An escaped quote opens nothing, and stepping over it first keeps a
|
|
399
|
+
* run of them from each scanning the rest of the text for a close that never comes.
|
|
400
|
+
*/
|
|
401
|
+
function withoutQuoted(segment) {
|
|
402
|
+
return segment.replace(/\\.|'[^']*'|"(?:[^"\\]|\\.)*"/g, (match) => (match.startsWith("\\") ? match : "''"));
|
|
403
|
+
}
|
|
404
|
+
/**
|
|
405
|
+
* Whether a segment, with what its quotes hold emptied out, runs an editor that
|
|
406
|
+
* rewrites its files: `sed 's/ -i //' a.ts` only prints.
|
|
407
|
+
*/
|
|
408
|
+
function editsInPlace(bare) {
|
|
409
|
+
const called = bare.replace(LEADING_WORDS, "");
|
|
410
|
+
const editor = IN_PLACE_EDITORS.exec(called)?.[1];
|
|
411
|
+
if (editor === undefined)
|
|
412
|
+
return false;
|
|
413
|
+
if (editor === "perl" || editor === "ruby")
|
|
414
|
+
return PERL_IN_PLACE.test(called);
|
|
415
|
+
return (editor.endsWith("awk") ? AWK_IN_PLACE : IN_PLACE).test(called);
|
|
416
|
+
}
|
|
417
|
+
const INTERPRETER = /(?:^|[\s;&|(])(?:python[0-9.]*|node|deno|bun|tsx|ruby|perl|php|bash|sh|zsh)\b/;
|
|
418
|
+
/**
|
|
419
|
+
* An interpreter fed a script on stdin: whatever the script writes, the command line
|
|
420
|
+
* cannot say. Looked for ahead of a line's last `<<`, because scanning on from every
|
|
421
|
+
* interpreter word to find one took 20 s over 200 KB of them.
|
|
422
|
+
*/
|
|
423
|
+
function feedsInterpreter(ran) {
|
|
424
|
+
if (!ran.includes("<<"))
|
|
425
|
+
return false;
|
|
426
|
+
return ran.split("\n").some((line) => INTERPRETER.test(line.slice(0, Math.max(line.lastIndexOf("<<"), 0))));
|
|
427
|
+
}
|
|
361
428
|
/**
|
|
362
429
|
* Whether a shell command could have changed a file in the repository.
|
|
363
430
|
*
|
|
@@ -375,18 +442,93 @@ const INTERPRETER_HEREDOC = /(?:^|[\s;&|(])(?:python[0-9.]*|node|deno|bun|tsx|ru
|
|
|
375
442
|
*/
|
|
376
443
|
export function shellWrites(command) {
|
|
377
444
|
const ran = withoutHeredocs(command);
|
|
378
|
-
if (
|
|
445
|
+
if (feedsInterpreter(ran))
|
|
379
446
|
return true;
|
|
380
447
|
for (const segment of commandSegments(ran)) {
|
|
381
|
-
|
|
382
|
-
const bare = segment.replace(/'[^']*'|"(?:[^"\\]|\\.)*"/g, "''");
|
|
448
|
+
const bare = withoutQuoted(segment);
|
|
383
449
|
if (REDIRECT_WRITE.test(bare) || TEE_WRITE.test(bare) || FILE_MUTATORS.test(bare))
|
|
384
450
|
return true;
|
|
385
|
-
if (
|
|
451
|
+
if (editsInPlace(bare))
|
|
386
452
|
return true;
|
|
387
453
|
}
|
|
388
454
|
return false;
|
|
389
455
|
}
|
|
456
|
+
/**
|
|
457
|
+
* A `git` call as a shell runs one: what can stand in front of it, the global
|
|
458
|
+
* flags, then the subcommand. `if FOO=1 git -C app -c a=b commit -m x`. Read with
|
|
459
|
+
* what quotes hold emptied out, so a value with a space in it is one word.
|
|
460
|
+
*/
|
|
461
|
+
const GIT_CALL = /^\s*(?:[({!]\s*|(?:sudo|if|then|elif|else|while|until|do|time|command|env)\s+|\w+=\S*\s+)*git\s+(?:-[Cc]\s+\S+\s+|--?[^\s=]+(?:=\S*)?\s+){0,8}([a-z][\w-]*)/;
|
|
462
|
+
/**
|
|
463
|
+
* The git subcommands a shell command runs, in the order it runs them. By command
|
|
464
|
+
* and not by word: `pytest tests/git/`, `npm --prefix tools/commit run dev` and a
|
|
465
|
+
* message saying "git commit" run none.
|
|
466
|
+
*/
|
|
467
|
+
export function gitCommands(command) {
|
|
468
|
+
if (!command.includes("git"))
|
|
469
|
+
return [];
|
|
470
|
+
const found = [];
|
|
471
|
+
for (const segment of commandSegments(withoutHeredocs(command))) {
|
|
472
|
+
const subcommand = GIT_CALL.exec(withoutQuoted(segment))?.[1];
|
|
473
|
+
if (subcommand !== undefined)
|
|
474
|
+
found.push(subcommand);
|
|
475
|
+
}
|
|
476
|
+
return found;
|
|
477
|
+
}
|
|
478
|
+
/** Besides `SHELL_READERS`, what leaves the tree as it found it: a comment and a loop's own words too. */
|
|
479
|
+
const SHELL_LOOKS = /^(?:#|(?:cd|ls|pwd|echo|printf|date|sleep|true|false|which|sort|uniq|cut|tr|ps|mkdir|export|test|\[|gh|for|done|fi)(?:\s|$))/;
|
|
480
|
+
/** Git commands that put nothing into the tree for a commit to hold. */
|
|
481
|
+
const GIT_LOOKS = new Set([
|
|
482
|
+
"status", "log", "diff", "show", "blame", "reflog", "rev-parse", "rev-list", "ls-files", "describe",
|
|
483
|
+
"config", "remote", "fetch", "push", "branch", "tag", "worktree", "switch", "checkout", "add", "commit",
|
|
484
|
+
]);
|
|
485
|
+
/** What only the shell can turn into a file's name, so the command line never spells it. */
|
|
486
|
+
const SHELL_EXPANDS = /[*?$`{]/;
|
|
487
|
+
/** The one `git checkout` that stands on a branch and takes no file from anywhere. */
|
|
488
|
+
const CHECKOUT_BRANCH = /\bcheckout(?:\s+(?:-[bB]\s+\S+|-[qft]|--(?:quiet|force|track|detach)))*(?:\s+\S+)?\s*$/;
|
|
489
|
+
/** Readers that write when a flag or a second name says so, and the `gh` that moves the tree or fills it. */
|
|
490
|
+
const WRITES_BY_ARGUMENT = /^(?:gh\s+(?:pr\s+checkout|co|release\s+download|run\s+download|repo\s+clone|gist\s+clone)\s|sort\s(?:.*\s)?(?:-[a-zA-Z]*o|--output)|yq\s(?:.*\s)?(?:-[a-zA-Z]*i|--inplace)|(?:uniq|xxd)(?:\s+-\S+)*\s+[^\s-]\S*\s+[^\s-])/;
|
|
491
|
+
/** `sed`'s write, as a command or as the flag of an `s`: `w out`, `1,5w out`, `s/a/b/gw out`. */
|
|
492
|
+
const SED_WRITE = /(?:^|[^A-Za-z])[gpiImMe]*[wW]\s+\S/;
|
|
493
|
+
/**
|
|
494
|
+
* Whether an `awk` or `sed` writes from inside its program, which `withoutQuoted`
|
|
495
|
+
* empties: `awk '{print > "x"}'`, `sed -n 'w x'`. Behind a `print`, as `$1 > 5` is a
|
|
496
|
+
* comparison; a `>` in a printed string reads as a write, and the session keeps the commit.
|
|
497
|
+
*/
|
|
498
|
+
function programWrites(called, segment) {
|
|
499
|
+
const [reader] = /^(?:awk|sed)\b/.exec(called) ?? [];
|
|
500
|
+
if (reader === undefined)
|
|
501
|
+
return false;
|
|
502
|
+
const program = (segment.match(/\\.|'[^']*'|"(?:[^"\\]|\\.)*"/g) ?? []).join("");
|
|
503
|
+
if (reader === "sed")
|
|
504
|
+
return SED_WRITE.test(program);
|
|
505
|
+
const print = program.search(/\bprintf?\b/);
|
|
506
|
+
return /\bsystem\s*\(/.test(program) || (print !== -1 && /[>|]/.test(program.slice(print)));
|
|
507
|
+
}
|
|
508
|
+
/**
|
|
509
|
+
* Whether a shell command leaves nothing behind for a commit to hold, apart from
|
|
510
|
+
* what it writes by name (`shellWrites`). Anything not known here reads as no: a
|
|
511
|
+
* formatter, a build, a script write what nobody can say from the command line, and
|
|
512
|
+
* neither can anyone say which files a write through a glob or a variable took.
|
|
513
|
+
*/
|
|
514
|
+
export function shellOnlyLooks(command) {
|
|
515
|
+
// `commandSegments` lifts a substitution out of its segment, and it may be the target.
|
|
516
|
+
const substitutionsKept = withoutHeredocs(command).replace(/\$\([^()]*\)/g, "$$_$&");
|
|
517
|
+
return commandSegments(substitutionsKept).every((segment) => {
|
|
518
|
+
if (shellWrites(segment) && SHELL_EXPANDS.test(segment.replace(/'[^']*'/g, "")))
|
|
519
|
+
return false;
|
|
520
|
+
const bare = withoutQuoted(segment);
|
|
521
|
+
const subcommand = GIT_CALL.exec(bare)?.[1];
|
|
522
|
+
if (subcommand === "checkout")
|
|
523
|
+
return CHECKOUT_BRANCH.test(bare);
|
|
524
|
+
if (subcommand !== undefined)
|
|
525
|
+
return GIT_LOOKS.has(subcommand) && !/\s--output\b/.test(bare);
|
|
526
|
+
const called = bare.replace(LEADING_WORDS, "");
|
|
527
|
+
if (WRITES_BY_ARGUMENT.test(called) || programWrites(called, segment))
|
|
528
|
+
return false;
|
|
529
|
+
return called.trim() === "" || SHELL_READERS.test(called) || SHELL_LOOKS.test(called);
|
|
530
|
+
});
|
|
531
|
+
}
|
|
390
532
|
/**
|
|
391
533
|
* `sed -i` rewrites the file; whatever else it is, it is not a look at it. GNU
|
|
392
534
|
* takes any backup suffix (`-i.bak`, `-ie`), the flag rides in a cluster
|
|
@@ -395,7 +537,47 @@ export function shellWrites(command) {
|
|
|
395
537
|
const IN_PLACE = /(?:^|\s)(?:-[a-zA-Z]*[iI]\S*|--in-place\S*)(?:\s|$)/;
|
|
396
538
|
/** Tools whose first non-flag argument is a program, not a path. */
|
|
397
539
|
const SCRIPTED = /^\s*(?:sudo\s+)?(?:sed|awk)\b/;
|
|
398
|
-
|
|
540
|
+
/** `main:src/a.ts` names `src/a.ts`; the colon in a path's own marker names no revision. */
|
|
541
|
+
function afterRevision(token) {
|
|
542
|
+
let colon = token.lastIndexOf(":");
|
|
543
|
+
while (colon > 0 && token.endsWith(REDACTION_MARKER, colon))
|
|
544
|
+
colon = token.lastIndexOf(":", colon - 1);
|
|
545
|
+
return token.slice(colon + 1);
|
|
546
|
+
}
|
|
547
|
+
/** The longest path a filesystem takes: a directory past it is none, however a record or a chain of `cd` spells it. */
|
|
548
|
+
const PATH_MAX = 4096;
|
|
549
|
+
/**
|
|
550
|
+
* What reading one session's shell may spend, in characters: its commands, and the
|
|
551
|
+
* paths they spell out. Every word of a command is joined to its directory, so two
|
|
552
|
+
* bytes of command cost a directory's length, and records repeat that as often as a
|
|
553
|
+
* transcript has room: no cap per command bounds it. Past this a command already seen
|
|
554
|
+
* to write still counts, and the rest count as writes nobody can name. The largest
|
|
555
|
+
* real session, with its subagents, spends 13 MB.
|
|
556
|
+
*/
|
|
557
|
+
const SHELL_CHARS = 32 * 1024 * 1024;
|
|
558
|
+
/** What keeping a path costs besides its characters, so that short ones spend `SHELL_CHARS` too. */
|
|
559
|
+
const PATH_COST = 64;
|
|
560
|
+
/** The longest command read, against 39 KB in real sessions, and short of the 8 MB word that overflows the stack of the patterns reading it. */
|
|
561
|
+
const COMMAND_CHARS = 1024 * 1024;
|
|
562
|
+
/** Whether this much still fits the budget, which pays for it if so. */
|
|
563
|
+
export function spend(budget, amount) {
|
|
564
|
+
if (budget.left < amount)
|
|
565
|
+
return false;
|
|
566
|
+
budget.left -= amount;
|
|
567
|
+
return true;
|
|
568
|
+
}
|
|
569
|
+
/**
|
|
570
|
+
* `to` as seen from a directory. Nothing past `PATH_MAX`, where normalising is what
|
|
571
|
+
* costs, as it copies the path so far at each `..`, or past the budget.
|
|
572
|
+
*/
|
|
573
|
+
function pathFrom(dir, to, budget) {
|
|
574
|
+
if (to.startsWith("/"))
|
|
575
|
+
return to.length <= PATH_MAX && spend(budget, to.length) ? posix.normalize(to) : undefined;
|
|
576
|
+
if (dir === undefined || dir.length + to.length >= PATH_MAX || !spend(budget, dir.length + to.length))
|
|
577
|
+
return undefined;
|
|
578
|
+
return posix.join(dir, to);
|
|
579
|
+
}
|
|
580
|
+
export function shellReadPaths(command, budget = { left: SHELL_CHARS }) {
|
|
399
581
|
const paths = [];
|
|
400
582
|
for (const segment of commandSegments(withoutHeredocs(command))) {
|
|
401
583
|
if (!SHELL_READERS.test(segment))
|
|
@@ -418,27 +600,32 @@ export function shellReadPaths(command) {
|
|
|
418
600
|
}
|
|
419
601
|
// A glob stays one: `cat src/*.js` read every file it expanded to, and
|
|
420
602
|
// `detectors.ts:samePath` matches it against the file edited.
|
|
421
|
-
const path =
|
|
603
|
+
const path = afterRevision(token);
|
|
422
604
|
// "/" and "." name no file, and "/" matches every path there is.
|
|
423
605
|
if (/^[./]+$/.test(path))
|
|
424
606
|
continue;
|
|
425
607
|
if (!/\.[A-Za-z0-9]+$/.test(path) && !path.includes("/"))
|
|
426
608
|
continue;
|
|
609
|
+
if (!spend(budget, path.length + PATH_COST))
|
|
610
|
+
return paths;
|
|
427
611
|
paths.push(path.replace(/^\.\//, ""));
|
|
428
612
|
}
|
|
429
613
|
}
|
|
430
614
|
return paths;
|
|
431
615
|
}
|
|
432
|
-
/** A shell word, quotes and all: `'s/a b/c/'` is one word, not two. */
|
|
433
|
-
const SHELL_WORD = /(
|
|
616
|
+
/** A shell word, quotes and all: `'s/a b/c/'` is one word, not two. What is escaped goes first, as in `withoutQuoted`. */
|
|
617
|
+
const SHELL_WORD = /(?:\\.|'[^']*'|"(?:[^"\\]|\\.)*"|[^\s'"])+/g;
|
|
434
618
|
/**
|
|
435
619
|
* Flags whose next word is the script, not a file. Only the bare ones for sed:
|
|
436
620
|
* `sed -ie` is `-i` with the backup suffix "e", while `perl -pe` is `-p -e`.
|
|
437
621
|
*/
|
|
438
622
|
const SED_SCRIPT_FLAG = /^(?:-e|--expression|-f|--file)$/;
|
|
439
623
|
const PERL_SCRIPT_FLAG = /^-[a-zA-Z]*e$/;
|
|
624
|
+
/** awk's flags that take a word that is neither the script nor a file. */
|
|
625
|
+
const AWK_VALUE_FLAG = /^(?:-[ivF]|--(?:include|assign|field-separator))$/;
|
|
440
626
|
/**
|
|
441
|
-
* The files `sed -i` and `
|
|
627
|
+
* The files `sed -i`, `perl -i`, `ruby -i` and `awk -i inplace` rewrite: every
|
|
628
|
+
* argument after the script.
|
|
442
629
|
* What the edit changed stays unseen — a sed script is a program, not the text
|
|
443
630
|
* it replaces — but where it landed is on the command line, and without it a
|
|
444
631
|
* session editing through sed read to the detectors as one that edited nothing.
|
|
@@ -453,63 +640,69 @@ const PERL_SCRIPT_FLAG = /^-[a-zA-Z]*e$/;
|
|
|
453
640
|
* still names no file to the detectors (`shellWrites` knows only that one was
|
|
454
641
|
* written); a glob names files this cannot list.
|
|
455
642
|
*/
|
|
456
|
-
export function shellEditPaths(command, cwd) {
|
|
643
|
+
export function shellEditPaths(command, cwd, budget) {
|
|
644
|
+
return shellEditWords(command, cwd, budget).filter((word) => !/[*?]/.test(word));
|
|
645
|
+
}
|
|
646
|
+
/** As `shellEditPaths`, with a glob left in: to say what a command edits, `.env*` is as good as a file. */
|
|
647
|
+
export function shellEditWords(command, cwd, budget = { left: SHELL_CHARS }) {
|
|
457
648
|
const paths = [];
|
|
458
|
-
let dir = cwd;
|
|
649
|
+
let dir = cwd !== undefined && cwd.length <= PATH_MAX ? cwd : undefined;
|
|
459
650
|
for (const segment of commandSegments(withoutHeredocs(command))) {
|
|
460
651
|
if (/^\s*cd(?:\s|$)/.test(segment)) {
|
|
461
652
|
const to = /^\s*cd\s+(\S+)\s*$/.exec(segment)?.[1]?.replace(/["']/g, "");
|
|
462
|
-
|
|
463
|
-
dir = posix.normalize(to);
|
|
464
|
-
else
|
|
465
|
-
dir = dir === undefined || to === undefined || /^[~$-]/.test(to) ? undefined : posix.join(dir, to);
|
|
653
|
+
dir = to === undefined || /^[~$-]/.test(to) ? undefined : pathFrom(dir, to, budget);
|
|
466
654
|
continue;
|
|
467
655
|
}
|
|
468
|
-
|
|
469
|
-
continue;
|
|
470
|
-
const perl = /^\s*(?:sudo\s+)?perl\b/.test(segment);
|
|
656
|
+
const called = segment.replace(LEADING_WORDS, "");
|
|
471
657
|
// The same test `shellWrites` applies, on the same unquoted text, so the two
|
|
472
|
-
// agree on what was written
|
|
473
|
-
|
|
474
|
-
if (!(perl ? /\bperl\s+-\w*i/.test(bare) : IN_PLACE.test(bare)))
|
|
658
|
+
// agree on what was written.
|
|
659
|
+
if (!IN_PLACE_EDITORS.test(called) || !editsInPlace(withoutQuoted(called)))
|
|
475
660
|
continue;
|
|
661
|
+
const scriptFlag = /^(?:perl|ruby)\b/.test(called) ? PERL_SCRIPT_FLAG : SED_SCRIPT_FLAG;
|
|
662
|
+
const awk = /^g?awk\b/.test(called);
|
|
476
663
|
let scriptSeen = false;
|
|
477
|
-
let
|
|
478
|
-
const words =
|
|
664
|
+
let skipNext = false;
|
|
665
|
+
const words = called.match(SHELL_WORD) ?? [];
|
|
479
666
|
for (const word of words.slice(1)) {
|
|
480
667
|
if (/^\d*[<>|]|^&/.test(word))
|
|
481
668
|
break;
|
|
482
669
|
const token = word.replace(/["']/g, "");
|
|
483
|
-
if (
|
|
484
|
-
|
|
670
|
+
if (skipNext) {
|
|
671
|
+
skipNext = false;
|
|
485
672
|
continue;
|
|
486
673
|
}
|
|
487
|
-
if (
|
|
674
|
+
if (scriptFlag.test(token)) {
|
|
488
675
|
scriptSeen = true;
|
|
489
|
-
|
|
676
|
+
skipNext = true;
|
|
677
|
+
continue;
|
|
678
|
+
}
|
|
679
|
+
if (awk && AWK_VALUE_FLAG.test(token)) {
|
|
680
|
+
skipNext = true;
|
|
490
681
|
continue;
|
|
491
682
|
}
|
|
492
683
|
if (/^--(?:expression|file)=/.test(token))
|
|
493
684
|
scriptSeen = true;
|
|
494
|
-
// An empty word is BSD sed's backup suffix: `sed -i '' 's/a/b/' f`.
|
|
495
|
-
|
|
685
|
+
// An empty word is BSD sed's backup suffix: `sed -i '' 's/a/b/' f`. A bare
|
|
686
|
+
// `$VAR` can be flags as well as the script; quoted, it is one word, and in
|
|
687
|
+
// single quotes `$` is sed's own, the last line: `sed -i '$a K=v' f`.
|
|
688
|
+
if (token.length === 0 || token.startsWith("-") || word.startsWith("$"))
|
|
496
689
|
continue;
|
|
497
690
|
if (!scriptSeen) {
|
|
498
691
|
scriptSeen = true;
|
|
499
692
|
continue;
|
|
500
693
|
}
|
|
501
|
-
if (
|
|
694
|
+
if (token.startsWith("$") || /^[./]+$/.test(token))
|
|
502
695
|
continue;
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
696
|
+
const path = dir === undefined || token.startsWith("~") ? undefined : pathFrom(dir, token, budget);
|
|
697
|
+
if (!spend(budget, path === undefined ? token.length + PATH_COST : PATH_COST))
|
|
698
|
+
return paths;
|
|
699
|
+
paths.push(path ?? token.replace(/^\.\//, ""));
|
|
507
700
|
}
|
|
508
701
|
}
|
|
509
702
|
return paths;
|
|
510
703
|
}
|
|
511
704
|
function pushShellEdits(state, use, command, cwd) {
|
|
512
|
-
for (const filePath of shellEditPaths(command, asString(cwd))) {
|
|
705
|
+
for (const filePath of shellEditPaths(command, asString(cwd), state.shell)) {
|
|
513
706
|
const edit = {
|
|
514
707
|
toolUseId: use.id,
|
|
515
708
|
tool: use.name,
|
|
@@ -548,11 +741,20 @@ function pushToolUse(state, record, block, recordIndex) {
|
|
|
548
741
|
if (LOOKUP_TOOLS.has(name))
|
|
549
742
|
pushLookup(state, use);
|
|
550
743
|
if (name === "Bash" && typeof use.input.command === "string") {
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
744
|
+
const { command } = use.input;
|
|
745
|
+
if (command.length <= COMMAND_CHARS && spend(state.shell, command.length)) {
|
|
746
|
+
for (const path of shellReadPaths(command, state.shell))
|
|
747
|
+
state.filesRead.add(path);
|
|
748
|
+
if (shellWrites(command))
|
|
749
|
+
state.shellWrote = true;
|
|
750
|
+
pushShellEdits(state, use, command, record.cwd);
|
|
751
|
+
}
|
|
752
|
+
else {
|
|
753
|
+
// Not read, which does not say nothing was written. The call keeps no command,
|
|
754
|
+
// so that nothing after this reads it either.
|
|
755
|
+
state.opaqueWrite = true;
|
|
756
|
+
use.input = {};
|
|
757
|
+
}
|
|
556
758
|
}
|
|
557
759
|
if (mayWrite(name, use.input))
|
|
558
760
|
state.opaqueWrite = true;
|
|
@@ -634,12 +836,71 @@ function delegatedUnseen(state) {
|
|
|
634
836
|
!state.delegated.has(use.id) &&
|
|
635
837
|
!state.counted.has(use.id));
|
|
636
838
|
}
|
|
839
|
+
/**
|
|
840
|
+
* Whether the agent ran a commit: what it put on a branch is its work however the
|
|
841
|
+
* files got there, and a formatter, a generator or a script writes nothing
|
|
842
|
+
* `shellWrites` can read. A call that failed committed nothing.
|
|
843
|
+
*/
|
|
844
|
+
function committedFromShell(state) {
|
|
845
|
+
return state.toolUses.some((use) => {
|
|
846
|
+
const { command } = use.input;
|
|
847
|
+
return (use.name === "Bash" &&
|
|
848
|
+
typeof command === "string" &&
|
|
849
|
+
use.result?.isError !== true &&
|
|
850
|
+
gitCommands(command).includes("commit"));
|
|
851
|
+
});
|
|
852
|
+
}
|
|
637
853
|
/**
|
|
638
854
|
* Records of what Claude Code sent to the API, not of what happened: a blob
|
|
639
855
|
* repeats messages already in the session — tool calls and results included —
|
|
640
856
|
* so reading its blocks counted every call it carried a second time.
|
|
641
857
|
*/
|
|
642
858
|
const REQUEST_RECORD_TYPES = new Set(["api-request", "api-request-shape", "api-request-blob"]);
|
|
859
|
+
/**
|
|
860
|
+
* The longest line parsed, against 4.5 MB in 1215 real sessions. Past some 200 MB of
|
|
861
|
+
* `1,1,1,` in one array, parsing it ends the process, which nothing can catch.
|
|
862
|
+
*/
|
|
863
|
+
const LINE_CHARS = 64 * 1024 * 1024;
|
|
864
|
+
/**
|
|
865
|
+
* What one session's objects, arrays, keys and items may come to, counting an object
|
|
866
|
+
* or array as three keys and an item as a third of one. Parsed, the three bytes of
|
|
867
|
+
* `{},` are 56 and with what is kept of the block 200, a key takes 72 in an object of
|
|
868
|
+
* hundreds or of a shape of its own, ten times its line, and the four bytes of `0.5,`
|
|
869
|
+
* in an array are 24. So a small upload that inflates to nothing else outgrows any
|
|
870
|
+
* heap, and it has to be counted before it is parsed.
|
|
871
|
+
*
|
|
872
|
+
* ponytail: one flat count, 14.6 million in the most of 97 real sessions (252 MB), and
|
|
873
|
+
* a colon after a quote or a comma inside a string counts as a key or an item. What
|
|
874
|
+
* comes past it, or past `LINE_CHARS`, reads as a write nobody can name, and is not
|
|
875
|
+
* read. Raise it if sessions of 500 MB as dense as that one turn up, and the heap with
|
|
876
|
+
* it.
|
|
877
|
+
*/
|
|
878
|
+
export const STRUCTURES = 16_000_000;
|
|
879
|
+
const KEYS_PER_STRUCTURE = 3;
|
|
880
|
+
const ITEMS_PER_KEY = 3;
|
|
881
|
+
function structuresIn(line) {
|
|
882
|
+
let count = 0;
|
|
883
|
+
for (const opening of ["{", "["]) {
|
|
884
|
+
for (let at = line.indexOf(opening); at !== -1; at = line.indexOf(opening, at + 1))
|
|
885
|
+
count += KEYS_PER_STRUCTURE;
|
|
886
|
+
}
|
|
887
|
+
for (let at = line.indexOf(":"); at !== -1; at = line.indexOf(":", at + 1)) {
|
|
888
|
+
// Backwards over what JSON lets stand before a colon, which ends at the colon before.
|
|
889
|
+
let before = at - 1;
|
|
890
|
+
while (line[before] === " " || line[before] === "\t" || line[before] === "\r")
|
|
891
|
+
before -= 1;
|
|
892
|
+
if (line[before] === '"')
|
|
893
|
+
count += 1;
|
|
894
|
+
}
|
|
895
|
+
let items = 0;
|
|
896
|
+
for (let at = line.indexOf(","); at !== -1; at = line.indexOf(",", at + 1))
|
|
897
|
+
items += 1;
|
|
898
|
+
return count + Math.floor(items / ITEMS_PER_KEY);
|
|
899
|
+
}
|
|
900
|
+
/** Whether a line of a transcript may be parsed at all, which spends its part of `STRUCTURES` if so. */
|
|
901
|
+
export function mayParse(line, structures) {
|
|
902
|
+
return line.length <= LINE_CHARS && spend(structures, structuresIn(line));
|
|
903
|
+
}
|
|
643
904
|
function pushLine(state, line) {
|
|
644
905
|
state.skipped.lines += 1;
|
|
645
906
|
const trimmed = line.trim();
|
|
@@ -647,6 +908,11 @@ function pushLine(state, line) {
|
|
|
647
908
|
state.skipped.blank += 1;
|
|
648
909
|
return;
|
|
649
910
|
}
|
|
911
|
+
if (!mayParse(trimmed, state.structures)) {
|
|
912
|
+
state.skipped.malformedJson += 1;
|
|
913
|
+
state.opaqueWrite = true;
|
|
914
|
+
return;
|
|
915
|
+
}
|
|
650
916
|
let parsed;
|
|
651
917
|
try {
|
|
652
918
|
parsed = JSON.parse(trimmed);
|
|
@@ -767,12 +1033,14 @@ function finish(state) {
|
|
|
767
1033
|
// Not the same question as editToolUses > 0: a file written from the
|
|
768
1034
|
// shell counts, a call the parser cannot see into is "cannot tell", which
|
|
769
1035
|
// must not read as "edited nothing", and so is a subagent that says it
|
|
770
|
-
// edited, or whose work never came back to say.
|
|
1036
|
+
// edited, or whose work never came back to say. So does a commit the agent
|
|
1037
|
+
// ran, whatever changed the files in it.
|
|
771
1038
|
hasFileEdits: state.editToolUses > 0 ||
|
|
772
1039
|
state.shellWrote ||
|
|
773
1040
|
state.opaqueWrite ||
|
|
774
1041
|
state.delegatedEdits ||
|
|
775
|
-
delegatedUnseen(state)
|
|
1042
|
+
delegatedUnseen(state) ||
|
|
1043
|
+
committedFromShell(state),
|
|
776
1044
|
sidechainRecords,
|
|
777
1045
|
},
|
|
778
1046
|
skipped: state.skipped,
|
|
@@ -799,23 +1067,34 @@ export async function parseTranscriptFile(filePath) {
|
|
|
799
1067
|
export function onMainPath(session, uuid) {
|
|
800
1068
|
return typeof uuid === "string" && session.mainPath.has(uuid);
|
|
801
1069
|
}
|
|
802
|
-
|
|
1070
|
+
/** `walked` holds what is known of records already walked, as all on a way back share what is at its end. */
|
|
1071
|
+
export function onAbandonedBranch(session, uuid, walked = new Map()) {
|
|
803
1072
|
if (typeof uuid !== "string" || session.mainPath.size === 0)
|
|
804
1073
|
return false;
|
|
805
1074
|
if (session.mainPath.has(uuid))
|
|
806
1075
|
return false;
|
|
807
1076
|
const visited = new Set();
|
|
1077
|
+
let abandoned = false;
|
|
808
1078
|
let current = session.byUuid.get(uuid);
|
|
809
1079
|
while (current && typeof current.uuid === "string" && !visited.has(current.uuid)) {
|
|
1080
|
+
const known = walked.get(current.uuid);
|
|
1081
|
+
if (known !== undefined) {
|
|
1082
|
+
abandoned = known;
|
|
1083
|
+
break;
|
|
1084
|
+
}
|
|
810
1085
|
visited.add(current.uuid);
|
|
811
1086
|
const parent = parentOf(current);
|
|
812
1087
|
if (parent === undefined)
|
|
813
|
-
|
|
814
|
-
if (session.mainPath.has(parent))
|
|
815
|
-
|
|
1088
|
+
break;
|
|
1089
|
+
if (session.mainPath.has(parent)) {
|
|
1090
|
+
abandoned = true;
|
|
1091
|
+
break;
|
|
1092
|
+
}
|
|
816
1093
|
current = session.byUuid.get(parent);
|
|
817
1094
|
}
|
|
818
|
-
|
|
1095
|
+
for (const seen of visited)
|
|
1096
|
+
walked.set(seen, abandoned);
|
|
1097
|
+
return abandoned;
|
|
819
1098
|
}
|
|
820
1099
|
export function hasSkips(skipped) {
|
|
821
1100
|
return (skipped.malformedJson > 0 ||
|