claude-dev-env 8.117.3 → 8.117.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/_shared/pr-loop/scripts/pyproject.toml +1 -1
- package/_shared/process-tree/scripts/pyproject.toml +1 -1
- package/bin/cde.mjs +4 -3
- package/bin/cde.test.mjs +87 -1
- package/bin/codex-compat.mjs +4 -3
- package/bin/codex-compat.test.mjs +91 -1
- package/bin/install-plan.mjs +2 -1
- package/bin/install.mjs +24 -7
- package/bin/install.plan.test.mjs +15 -0
- package/bin/install.test.mjs +61 -0
- package/bin/python-probe.mjs +42 -0
- package/bin/python-probe.test.mjs +19 -2
- package/docs/rule-guides/features-start-with-an-eval.md +3 -3
- package/docs/rule-guides/proof-before-pull-request.md +2 -2
- package/hooks/blocking/bash_pre_tool_use_dispatcher.py +11 -4
- package/hooks/blocking/build_eval_before_eval_write_gate.py +10 -4
- package/hooks/blocking/followup_pr_dedupe.py +381 -64
- package/hooks/blocking/gate_child_process.py +187 -0
- package/hooks/blocking/gate_run_reads.py +116 -35
- package/hooks/blocking/merge_review_gate.py +93 -10
- package/hooks/blocking/merge_review_gate_parts/create_folders.py +346 -0
- package/hooks/blocking/merge_review_gate_parts/create_options.py +86 -0
- package/hooks/blocking/merge_review_gate_parts/folder_state.py +107 -0
- package/hooks/blocking/merge_review_gate_parts/open_request.py +199 -64
- package/hooks/blocking/merge_review_gate_parts/quoted_lines.py +283 -0
- package/hooks/blocking/merge_review_gate_parts/wrapped_creates.py +126 -0
- package/hooks/blocking/pr_lifecycle_skill_gate.py +61 -18
- package/hooks/blocking/pre_tool_use_dispatcher.py +25 -3
- package/hooks/blocking/pull_request_proof.py +5 -51
- package/hooks/blocking/pull_request_proof_edit_gate.py +19 -5
- package/hooks/blocking/test_bash_pre_tool_use_dispatcher.py +54 -0
- package/hooks/blocking/test_build_eval_before_eval_write_gate.py +64 -0
- package/hooks/blocking/test_followup_pr_dedupe.py +464 -1
- package/hooks/blocking/test_gate_child_process.py +64 -0
- package/hooks/blocking/test_gate_run_reads.py +364 -4
- package/hooks/blocking/test_merge_review_gate.py +832 -39
- package/hooks/blocking/test_pr_lifecycle_skill_gate.py +130 -4
- package/hooks/blocking/test_pre_tool_use_dispatcher.py +64 -0
- package/hooks/blocking/test_pull_request_proof.py +16 -0
- package/hooks/blocking/test_pull_request_proof_edit_gate.py +60 -0
- package/hooks/blocking/tests/test_create_folders.py +290 -0
- package/hooks/blocking/tests/test_create_options.py +34 -0
- package/hooks/blocking/tests/test_folder_state.py +40 -0
- package/hooks/blocking/tests/test_open_request.py +350 -44
- package/hooks/blocking/tests/test_quoted_lines.py +86 -0
- package/hooks/blocking/tests/test_wrapped_creates.py +42 -0
- package/hooks/features/bash-dispatchers.md +1 -1
- package/hooks/features/conduct-gates.md +7 -7
- package/hooks/features/write-edit-blocking.md +3 -3
- package/hooks/hooks_constants/build_eval_before_eval_write_gate_constants.py +5 -0
- package/hooks/hooks_constants/followup_pr_dedupe_constants.py +34 -1
- package/hooks/hooks_constants/gate_fault.py +52 -0
- package/hooks/hooks_constants/gate_fault_constants.py +13 -0
- package/hooks/hooks_constants/gate_run_reads_constants.py +9 -1
- package/hooks/hooks_constants/merge_review_gate_constants.py +94 -1
- package/hooks/hooks_constants/piped_pytest_blocker_constants.py +2 -0
- package/hooks/hooks_constants/pr_lifecycle_skill_gate_constants.py +6 -1
- package/hooks/hooks_constants/pre_tool_use_stdin.py +23 -3
- package/hooks/hooks_constants/shell_command_pipeline.py +68 -8
- package/hooks/hooks_constants/shell_command_segments.py +2 -2
- package/hooks/hooks_constants/shell_command_wrappers.py +3 -2
- package/hooks/hooks_constants/test_followup_pr_dedupe_constants.py +7 -0
- package/hooks/hooks_constants/test_gate_fault.py +41 -0
- package/hooks/hooks_constants/test_piped_pytest_blocker_constants.py +5 -0
- package/hooks/hooks_constants/test_pr_lifecycle_skill_gate_constants.py +10 -0
- package/hooks/hooks_constants/test_pre_tool_use_stdin.py +97 -0
- package/hooks/hooks_constants/test_shell_command_pipeline.py +55 -0
- package/hooks/hooks_constants/test_shell_command_segments.py +15 -0
- package/hooks/hooks_constants/test_shell_command_wrappers.py +6 -0
- package/hooks/hooks_constants/transcript_skill_scan_constants.py +6 -0
- package/hooks/pyproject.toml +2 -3
- package/hooks/test_python_floor.py +197 -0
- package/hooks/test_transcript_skill_scan.py +324 -0
- package/hooks/transcript_skill_scan.py +233 -2
- package/package.json +1 -1
|
@@ -1,20 +1,29 @@
|
|
|
1
1
|
"""Tests for skill invocation state across transcript compactions."""
|
|
2
2
|
|
|
3
|
+
import errno
|
|
4
|
+
import io
|
|
3
5
|
import json
|
|
6
|
+
import random
|
|
4
7
|
import sys
|
|
5
8
|
from pathlib import Path
|
|
9
|
+
from typing import BinaryIO
|
|
10
|
+
|
|
11
|
+
import pytest
|
|
6
12
|
|
|
7
13
|
HOOKS_DIRECTORY = Path(__file__).resolve().parent
|
|
8
14
|
if str(HOOKS_DIRECTORY) not in sys.path:
|
|
9
15
|
sys.path.insert(0, str(HOOKS_DIRECTORY))
|
|
10
16
|
|
|
11
17
|
from transcript_skill_scan import (
|
|
18
|
+
ScanCaps,
|
|
12
19
|
TranscriptToolCall,
|
|
13
20
|
invokes_skill_with_argument,
|
|
14
21
|
is_skill_loaded_after_last_compaction,
|
|
22
|
+
is_skill_loaded_after_last_compaction_newest_first,
|
|
15
23
|
is_skill_loaded_since_last_prompt,
|
|
16
24
|
skill_invocation_status,
|
|
17
25
|
tool_calls_since_last_prompt,
|
|
26
|
+
transcript_lines_newest_first,
|
|
18
27
|
)
|
|
19
28
|
|
|
20
29
|
|
|
@@ -221,3 +230,318 @@ def should_list_each_tool_call_since_the_last_prompt_with_its_outcome() -> None:
|
|
|
221
230
|
TranscriptToolCall("Edit", {}, False),
|
|
222
231
|
TranscriptToolCall("Write", {}, False),
|
|
223
232
|
]
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
NEWEST_FIRST_SKILL_NAMES = ("pr-lifecycle",)
|
|
236
|
+
NEWEST_FIRST_MARKERS = ("<command-name>/pr-lifecycle</command-name>",)
|
|
237
|
+
ALL_GENERATED_LINE_KINDS = (
|
|
238
|
+
_skill_entry("pr-lifecycle"),
|
|
239
|
+
_skill_entry("plugin:pr-lifecycle"),
|
|
240
|
+
json.dumps({"type": "system", "subtype": "compact_boundary"}),
|
|
241
|
+
json.dumps(
|
|
242
|
+
{
|
|
243
|
+
"type": "assistant",
|
|
244
|
+
"subtype": "compact_boundary",
|
|
245
|
+
"message": {"content": [{"type": "tool_use", "name": "Skill", "input": {"skill": "pr-lifecycle"}}]},
|
|
246
|
+
}
|
|
247
|
+
),
|
|
248
|
+
json.dumps({"type": "user", "message": {"content": "<command-name>/pr-lifecycle</command-name>"}}),
|
|
249
|
+
json.dumps({"type": "user", "message": {"content": "Please run pr-lifecycle later."}}),
|
|
250
|
+
json.dumps({"type": "user", "message": {"content": [{"type": "tool_result", "content": "pr-lifecycle"}]}}),
|
|
251
|
+
_skill_entry("other"),
|
|
252
|
+
json.dumps({"type": "user", "message": {"content": "Unrelated."}}),
|
|
253
|
+
"pr-lifecycle {",
|
|
254
|
+
)
|
|
255
|
+
EXHAUSTIVE_SEQUENCE_LENGTH = 4
|
|
256
|
+
GENERATED_SEQUENCE_COUNT = 3000
|
|
257
|
+
LONGEST_GENERATED_SEQUENCE = 30
|
|
258
|
+
GENERATED_SEQUENCE_SEED = 2338
|
|
259
|
+
ALL_TINY_BLOCK_SIZES = (1, 2, 3, 7, 64, 1 << 20)
|
|
260
|
+
ALL_READER_CONTENTS = (
|
|
261
|
+
b"",
|
|
262
|
+
b"\n",
|
|
263
|
+
b"one",
|
|
264
|
+
b"one\n",
|
|
265
|
+
b"one\ntwo",
|
|
266
|
+
b"one\ntwo\n",
|
|
267
|
+
b"\n\nmiddle\n\n",
|
|
268
|
+
b"short\n" + b"x" * 200 + b"\nend",
|
|
269
|
+
"café ☃ \U0001f600\nnext é\n".encode("utf-8"),
|
|
270
|
+
b"bad \xff\xfe byte\n\xc3\nhalf\xe2\x98\n",
|
|
271
|
+
b"carriage\r\nreturn\rinside\n",
|
|
272
|
+
b"lone\rcarriage\rreturns\r",
|
|
273
|
+
b"\r",
|
|
274
|
+
b"\r\r\n\n\r",
|
|
275
|
+
b"ab\r\ncd\r\nef\r\n",
|
|
276
|
+
b"x\ry\r\nz",
|
|
277
|
+
b"abcdef\r\nghijk\rlmnopq\n",
|
|
278
|
+
)
|
|
279
|
+
ALL_LINE_ENDINGS = ("\n", "\r\n", "\r")
|
|
280
|
+
ALL_SEQUENCE_BLOCK_SIZES = (1, 2, 3, 7, 64)
|
|
281
|
+
MIXED_ENDING_SEQUENCE_COUNT = 400
|
|
282
|
+
STOP_AFTER_BLOCK_COUNT = 2
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _newest_first_load(all_lines: list[str]) -> bool:
|
|
286
|
+
return is_skill_loaded_after_last_compaction_newest_first(
|
|
287
|
+
list(reversed(all_lines)), NEWEST_FIRST_SKILL_NAMES, NEWEST_FIRST_MARKERS
|
|
288
|
+
)
|
|
289
|
+
|
|
290
|
+
|
|
291
|
+
def _oldest_first_load(all_lines: list[str]) -> bool:
|
|
292
|
+
return is_skill_loaded_after_last_compaction(all_lines, NEWEST_FIRST_SKILL_NAMES, NEWEST_FIRST_MARKERS)
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def test_newest_first_load_should_take_the_newest_load_or_compaction() -> None:
|
|
296
|
+
load, plugin_load, compaction, compaction_that_also_loads, slash_command = ALL_GENERATED_LINE_KINDS[:5]
|
|
297
|
+
unrelated = ALL_GENERATED_LINE_KINDS[-2]
|
|
298
|
+
assert _newest_first_load([compaction, load, unrelated])
|
|
299
|
+
assert _newest_first_load([load, compaction, slash_command])
|
|
300
|
+
assert not _newest_first_load([load, compaction])
|
|
301
|
+
assert not _newest_first_load([plugin_load, compaction_that_also_loads])
|
|
302
|
+
assert not _newest_first_load([unrelated])
|
|
303
|
+
assert not _newest_first_load([])
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def test_newest_first_load_should_match_the_oldest_first_answer_on_every_short_sequence() -> None:
|
|
307
|
+
all_sequences: list[list[str]] = [[]]
|
|
308
|
+
for _ in range(EXHAUSTIVE_SEQUENCE_LENGTH):
|
|
309
|
+
all_sequences = [
|
|
310
|
+
each_sequence + [each_kind] for each_sequence in all_sequences for each_kind in ALL_GENERATED_LINE_KINDS
|
|
311
|
+
]
|
|
312
|
+
for each_sequence in all_sequences:
|
|
313
|
+
assert _newest_first_load(each_sequence) == _oldest_first_load(each_sequence), each_sequence
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def test_newest_first_load_should_match_the_oldest_first_answer_on_generated_long_sequences() -> None:
|
|
317
|
+
generator = random.Random(GENERATED_SEQUENCE_SEED)
|
|
318
|
+
all_answers: set[bool] = set()
|
|
319
|
+
for _ in range(GENERATED_SEQUENCE_COUNT):
|
|
320
|
+
all_lines = [
|
|
321
|
+
generator.choice(ALL_GENERATED_LINE_KINDS) for _ in range(generator.randint(0, LONGEST_GENERATED_SEQUENCE))
|
|
322
|
+
]
|
|
323
|
+
oldest_first_answer = _oldest_first_load(all_lines)
|
|
324
|
+
assert _newest_first_load(all_lines) == oldest_first_answer, all_lines
|
|
325
|
+
all_answers.add(oldest_first_answer)
|
|
326
|
+
assert all_answers == {True, False}
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def _lines_read_oldest_first(path: Path) -> list[str]:
|
|
330
|
+
with open(path, encoding="utf-8", errors="replace") as transcript:
|
|
331
|
+
return [each_line.removesuffix("\n") for each_line in transcript]
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def _lines_read_newest_first(path: Path, block_size: int) -> list[str]:
|
|
335
|
+
with open(path, "rb") as transcript:
|
|
336
|
+
return list(transcript_lines_newest_first(transcript, "utf-8", "replace", block_size))
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def test_lines_newest_first_should_return_each_line_of_the_file_newest_first(tmp_path: Path) -> None:
|
|
340
|
+
transcript_path = tmp_path / "session.jsonl"
|
|
341
|
+
for each_content in ALL_READER_CONTENTS:
|
|
342
|
+
transcript_path.write_bytes(each_content)
|
|
343
|
+
expected_lines = list(reversed(_lines_read_oldest_first(transcript_path)))
|
|
344
|
+
all_answers = {
|
|
345
|
+
each_block_size: _lines_read_newest_first(transcript_path, each_block_size)
|
|
346
|
+
for each_block_size in ALL_TINY_BLOCK_SIZES
|
|
347
|
+
}
|
|
348
|
+
assert all_answers == dict.fromkeys(ALL_TINY_BLOCK_SIZES, expected_lines), each_content
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def test_lines_newest_first_should_split_a_carriage_return_pair_across_a_block_edge(tmp_path: Path) -> None:
|
|
352
|
+
transcript_path = tmp_path / "session.jsonl"
|
|
353
|
+
transcript_path.write_bytes(b"ab\r\ncd\n")
|
|
354
|
+
assert _lines_read_newest_first(transcript_path, 4) == ["cd", "ab"]
|
|
355
|
+
transcript_path.write_bytes(b"ab\rcd\n")
|
|
356
|
+
assert _lines_read_newest_first(transcript_path, 3) == ["cd", "ab"]
|
|
357
|
+
transcript_path.write_bytes(b"ab\r\rcd")
|
|
358
|
+
assert _lines_read_newest_first(transcript_path, 2) == ["cd", "", "ab"]
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def _old_gate_answer(path: Path) -> bool:
|
|
362
|
+
return _oldest_first_load(_lines_read_oldest_first(path))
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def _new_gate_answer(path: Path, block_size: int) -> bool:
|
|
366
|
+
with open(path, "rb") as transcript:
|
|
367
|
+
all_lines = transcript_lines_newest_first(transcript, "utf-8", "replace", block_size)
|
|
368
|
+
return is_skill_loaded_after_last_compaction_newest_first(all_lines, NEWEST_FIRST_SKILL_NAMES, NEWEST_FIRST_MARKERS)
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def test_newest_first_scan_should_answer_as_the_old_gate_on_a_lone_carriage_return(tmp_path: Path) -> None:
|
|
372
|
+
load, _, compaction = ALL_GENERATED_LINE_KINDS[:3]
|
|
373
|
+
transcript_path = tmp_path / "session.jsonl"
|
|
374
|
+
transcript_path.write_bytes(f'{load}\n{compaction}\r{{"x":1}}\n'.encode())
|
|
375
|
+
assert not _old_gate_answer(transcript_path)
|
|
376
|
+
assert [_new_gate_answer(transcript_path, each_size) for each_size in ALL_SEQUENCE_BLOCK_SIZES] == [False] * 5
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def _mixed_ending_transcript(generator: random.Random) -> bytes:
|
|
380
|
+
all_lines = [generator.choice(ALL_GENERATED_LINE_KINDS) for _ in range(generator.randint(0, 12))]
|
|
381
|
+
return "".join(each_line + generator.choice(ALL_LINE_ENDINGS) for each_line in all_lines).encode("utf-8")
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def test_newest_first_scan_should_answer_as_the_old_gate_with_every_line_ending(tmp_path: Path) -> None:
|
|
385
|
+
generator = random.Random(GENERATED_SEQUENCE_SEED)
|
|
386
|
+
transcript_path = tmp_path / "session.jsonl"
|
|
387
|
+
all_answers: set[bool] = set()
|
|
388
|
+
for _ in range(MIXED_ENDING_SEQUENCE_COUNT):
|
|
389
|
+
transcript_path.write_bytes(_mixed_ending_transcript(generator))
|
|
390
|
+
old_answer = _old_gate_answer(transcript_path)
|
|
391
|
+
all_new_answers = {_new_gate_answer(transcript_path, each_size) for each_size in ALL_SEQUENCE_BLOCK_SIZES}
|
|
392
|
+
assert all_new_answers == {old_answer}, transcript_path.read_bytes()
|
|
393
|
+
all_answers.add(old_answer)
|
|
394
|
+
assert all_answers == {True, False}
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def test_lines_newest_first_should_stop_between_blocks_once_told_to_stop(tmp_path: Path) -> None:
|
|
398
|
+
transcript_path = tmp_path / "session.jsonl"
|
|
399
|
+
transcript_path.write_bytes(b"a\nb\nc\n" + b"x" * 40)
|
|
400
|
+
all_asks: list[int] = []
|
|
401
|
+
|
|
402
|
+
def keeps_reading() -> bool:
|
|
403
|
+
all_asks.append(len(all_asks))
|
|
404
|
+
return len(all_asks) <= STOP_AFTER_BLOCK_COUNT
|
|
405
|
+
|
|
406
|
+
with open(transcript_path, "rb") as transcript:
|
|
407
|
+
assert list(transcript_lines_newest_first(transcript, "utf-8", "replace", 4, keeps_reading)) == []
|
|
408
|
+
assert transcript.tell() == transcript_path.stat().st_size - 4
|
|
409
|
+
transcript_path.write_bytes(b"a\nb\nc\n")
|
|
410
|
+
all_asks.clear()
|
|
411
|
+
with open(transcript_path, "rb") as transcript:
|
|
412
|
+
assert list(transcript_lines_newest_first(transcript, "utf-8", "replace", 2, keeps_reading)) == ["c"]
|
|
413
|
+
assert len(all_asks) == STOP_AFTER_BLOCK_COUNT + 1
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def test_lines_newest_first_should_give_a_recent_load_before_reading_the_whole_file(tmp_path: Path) -> None:
|
|
417
|
+
transcript_path = tmp_path / "session.jsonl"
|
|
418
|
+
all_old_lines = [json.dumps({"type": "user", "message": {"content": f"line {each_index}"}}) for each_index in range(2000)]
|
|
419
|
+
transcript_path.write_text("\n".join([*all_old_lines, _skill_entry("pr-lifecycle")]) + "\n", encoding="utf-8")
|
|
420
|
+
with open(transcript_path, "rb") as transcript:
|
|
421
|
+
all_lines = transcript_lines_newest_first(transcript, "utf-8", "replace", 256)
|
|
422
|
+
assert is_skill_loaded_after_last_compaction_newest_first(all_lines, NEWEST_FIRST_SKILL_NAMES, NEWEST_FIRST_MARKERS)
|
|
423
|
+
assert transcript.tell() > transcript_path.stat().st_size - 1024
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
SMALL_LINE_BYTE_CAP = 256
|
|
427
|
+
SMALL_COPY_BYTE_CAP = 4096
|
|
428
|
+
ALL_CAP_BLOCK_SIZES = (1, 5, 16, 1000)
|
|
429
|
+
ALL_COPY_BLOCK_SIZES = (1, 7, 64, 4096, 5000)
|
|
430
|
+
|
|
431
|
+
|
|
432
|
+
class UnseekableTranscript(io.BytesIO):
|
|
433
|
+
def seekable(self) -> bool:
|
|
434
|
+
return False
|
|
435
|
+
|
|
436
|
+
def seek(self, *all_arguments: int) -> int:
|
|
437
|
+
raise io.UnsupportedOperation("seek")
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
class EndSeekRefusedTranscript(io.BytesIO):
|
|
441
|
+
def seek(self, offset: int, whence: int = io.SEEK_SET) -> int:
|
|
442
|
+
if whence == io.SEEK_END:
|
|
443
|
+
raise OSError(errno.EINVAL, "Invalid argument")
|
|
444
|
+
return super().seek(offset, whence)
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
def _capped_lines(
|
|
448
|
+
transcript: BinaryIO, block_size: int, all_cuts: list[int], copy_byte_cap: int = SMALL_COPY_BYTE_CAP
|
|
449
|
+
) -> list[str]:
|
|
450
|
+
caps = ScanCaps(SMALL_LINE_BYTE_CAP, copy_byte_cap, lambda: all_cuts.append(1))
|
|
451
|
+
return list(transcript_lines_newest_first(transcript, "utf-8", "replace", block_size, caps=caps))
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def _capped_load(content: bytes, block_size: int, all_cuts: list[int], is_seekable: bool = True) -> bool:
|
|
455
|
+
transcript = io.BytesIO(content) if is_seekable else UnseekableTranscript(content)
|
|
456
|
+
all_lines = _capped_lines(transcript, block_size, all_cuts)
|
|
457
|
+
return is_skill_loaded_after_last_compaction_newest_first(all_lines, NEWEST_FIRST_SKILL_NAMES, NEWEST_FIRST_MARKERS)
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def _load_line() -> bytes:
|
|
461
|
+
return _skill_entry("pr-lifecycle").encode()
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def test_load_older_than_a_line_over_the_cap_should_not_be_found_and_should_count_a_cut() -> None:
|
|
465
|
+
content = _load_line() + b"\n" + b"y" * (SMALL_LINE_BYTE_CAP + 1) + b"\n"
|
|
466
|
+
for each_block_size in ALL_CAP_BLOCK_SIZES:
|
|
467
|
+
all_cuts: list[int] = []
|
|
468
|
+
assert not _capped_load(content, each_block_size, all_cuts)
|
|
469
|
+
assert all_cuts == [1]
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def test_load_newer_than_a_line_over_the_cap_should_be_found() -> None:
|
|
473
|
+
content = b"x\n" + b"y" * (SMALL_LINE_BYTE_CAP * 3) + b"\n" + _load_line() + b"\n"
|
|
474
|
+
for each_block_size in ALL_CAP_BLOCK_SIZES:
|
|
475
|
+
assert _capped_load(content, each_block_size, [])
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
def test_line_at_exactly_the_cap_should_be_read() -> None:
|
|
479
|
+
content = _load_line() + b"\n" + b"y" * SMALL_LINE_BYTE_CAP + b"\r\n" + b"z" * SMALL_LINE_BYTE_CAP
|
|
480
|
+
for each_block_size in ALL_CAP_BLOCK_SIZES:
|
|
481
|
+
all_cuts: list[int] = []
|
|
482
|
+
assert _capped_load(content, each_block_size, all_cuts)
|
|
483
|
+
assert all_cuts == []
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
def test_unseekable_transcript_should_be_read_newest_first_after_a_forward_copy() -> None:
|
|
487
|
+
content = b"a\nb\r\nc\rd"
|
|
488
|
+
all_lines = transcript_lines_newest_first(UnseekableTranscript(content), "utf-8", "replace", 3)
|
|
489
|
+
assert list(all_lines) == ["d", "c", "b", "a"]
|
|
490
|
+
assert _capped_load(b"x\n" + _load_line() + b"\n", 7, [], is_seekable=False)
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def test_unseekable_transcript_should_give_no_lines_once_the_forward_copy_is_stopped() -> None:
|
|
494
|
+
all_asks: list[int] = []
|
|
495
|
+
|
|
496
|
+
def keeps_reading() -> bool:
|
|
497
|
+
all_asks.append(len(all_asks))
|
|
498
|
+
return len(all_asks) <= STOP_AFTER_BLOCK_COUNT
|
|
499
|
+
|
|
500
|
+
transcript = UnseekableTranscript(_load_line() + b"\n" + b"x" * 40)
|
|
501
|
+
assert list(transcript_lines_newest_first(transcript, "utf-8", "replace", 4, keeps_reading)) == []
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
def test_unseekable_transcript_should_stop_the_copy_at_a_line_over_the_cap() -> None:
|
|
505
|
+
all_cuts: list[int] = []
|
|
506
|
+
content = b"y" * (SMALL_LINE_BYTE_CAP * 4) + b"\n" + _load_line() + b"\n"
|
|
507
|
+
assert not _capped_load(content, 16, all_cuts, is_seekable=False)
|
|
508
|
+
assert all_cuts == [1]
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
def _copy_sized_stream(total_bytes: int) -> bytes:
|
|
512
|
+
load_part = _load_line() + b"\n"
|
|
513
|
+
filler_line = b"f" * 99 + b"\n"
|
|
514
|
+
filler = filler_line * ((total_bytes - len(load_part)) // len(filler_line) + 1)
|
|
515
|
+
return (load_part + filler)[:total_bytes]
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
def test_transcript_whose_end_seek_fails_should_be_read_newest_first_after_a_forward_copy() -> None:
|
|
519
|
+
content = b"x\n" + _load_line() + b"\nnewest"
|
|
520
|
+
all_lines = _capped_lines(EndSeekRefusedTranscript(content), 7, [])
|
|
521
|
+
assert all_lines[0] == "newest"
|
|
522
|
+
assert is_skill_loaded_after_last_compaction_newest_first(all_lines, NEWEST_FIRST_SKILL_NAMES, NEWEST_FIRST_MARKERS)
|
|
523
|
+
|
|
524
|
+
|
|
525
|
+
def test_procfs_transcript_should_be_read_after_a_forward_copy() -> None:
|
|
526
|
+
if sys.platform != "linux":
|
|
527
|
+
pytest.skip("procfs is a Linux file system")
|
|
528
|
+
with open("/proc/version", "rb") as transcript:
|
|
529
|
+
all_lines = list(transcript_lines_newest_first(transcript, "utf-8", "replace"))
|
|
530
|
+
assert all_lines[0].startswith("Linux version")
|
|
531
|
+
|
|
532
|
+
|
|
533
|
+
def test_stream_one_byte_over_the_copy_cap_should_give_no_lines_and_count_a_cut() -> None:
|
|
534
|
+
content = _copy_sized_stream(SMALL_COPY_BYTE_CAP + 1)
|
|
535
|
+
for each_block_size in ALL_COPY_BLOCK_SIZES:
|
|
536
|
+
all_cuts: list[int] = []
|
|
537
|
+
assert _capped_lines(UnseekableTranscript(content), each_block_size, all_cuts) == []
|
|
538
|
+
assert all_cuts == [1]
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
def test_stream_at_exactly_the_copy_cap_should_be_read() -> None:
|
|
542
|
+
content = _copy_sized_stream(SMALL_COPY_BYTE_CAP)
|
|
543
|
+
for each_block_size in ALL_COPY_BLOCK_SIZES:
|
|
544
|
+
all_cuts: list[int] = []
|
|
545
|
+
all_lines = _capped_lines(EndSeekRefusedTranscript(content), each_block_size, all_cuts)
|
|
546
|
+
assert all_cuts == []
|
|
547
|
+
assert is_skill_loaded_after_last_compaction_newest_first(all_lines, NEWEST_FIRST_SKILL_NAMES, NEWEST_FIRST_MARKERS)
|
|
@@ -1,23 +1,33 @@
|
|
|
1
1
|
"""Find a skill invocation after the last transcript compaction, or one with a given first argument anywhere.
|
|
2
2
|
|
|
3
|
+
A reader gives a large transcript's lines newest first, so a recent invocation answers without reading the whole file.
|
|
4
|
+
|
|
3
5
|
It also lists the tool calls made since the last user prompt, each with whether its result came back without an error.
|
|
4
6
|
"""
|
|
5
7
|
|
|
6
8
|
from __future__ import annotations
|
|
7
9
|
|
|
10
|
+
import io
|
|
8
11
|
import json
|
|
9
12
|
import re
|
|
10
|
-
from collections.abc import Collection, Iterable, Iterator
|
|
11
|
-
from
|
|
13
|
+
from collections.abc import Callable, Collection, Iterable, Iterator
|
|
14
|
+
from io import SEEK_END
|
|
15
|
+
from typing import BinaryIO, NamedTuple
|
|
12
16
|
|
|
13
17
|
from hooks_constants.transcript_skill_scan_constants import (
|
|
18
|
+
ALL_LINE_BREAK_BYTES,
|
|
14
19
|
ASSISTANT_ENTRY_TYPE,
|
|
20
|
+
CARRIAGE_RETURN_BYTE,
|
|
15
21
|
COMMAND_ARGUMENTS_PATTERN,
|
|
16
22
|
COMPACT_BOUNDARY_SUBTYPE,
|
|
23
|
+
FORWARD_COPY_BYTE_CAP,
|
|
17
24
|
IS_ERROR_FIELD,
|
|
18
25
|
META_ENTRY_FIELD,
|
|
26
|
+
NEWLINE_BYTE,
|
|
19
27
|
SKILL_TOOL_NAME,
|
|
20
28
|
TOOL_RESULT_BLOCK_TYPE,
|
|
29
|
+
TRANSCRIPT_BLOCK_BYTES,
|
|
30
|
+
TRANSCRIPT_LINE_BYTE_CAP,
|
|
21
31
|
TOOL_USE_BLOCK_TYPE,
|
|
22
32
|
TOOL_USE_ID_FIELD,
|
|
23
33
|
USER_ENTRY_TYPE,
|
|
@@ -171,6 +181,227 @@ def is_skill_loaded_after_last_compaction(
|
|
|
171
181
|
return skill_invocation_status(all_transcript_lines, skill_names, slash_command_markers)[1]
|
|
172
182
|
|
|
173
183
|
|
|
184
|
+
def is_skill_loaded_after_last_compaction_newest_first(
|
|
185
|
+
all_lines_newest_first: Iterable[str],
|
|
186
|
+
skill_names: Collection[str],
|
|
187
|
+
slash_command_markers: Collection[str],
|
|
188
|
+
) -> bool:
|
|
189
|
+
"""Return whether a matching invocation follows the last compact boundary, reading the newest line first.
|
|
190
|
+
|
|
191
|
+
::
|
|
192
|
+
|
|
193
|
+
newest first: load, ..., compact boundary -> True
|
|
194
|
+
newest first: compact boundary, ..., load -> False
|
|
195
|
+
no load and no compact boundary -> False
|
|
196
|
+
|
|
197
|
+
The first load or compact boundary met decides. The answer equals
|
|
198
|
+
``is_skill_loaded_after_last_compaction`` on the same lines in file order.
|
|
199
|
+
|
|
200
|
+
Args:
|
|
201
|
+
all_lines_newest_first: JSON transcript entries, one per line, newest first.
|
|
202
|
+
skill_names: Accepted skill names, with plugin prefixes accepted.
|
|
203
|
+
slash_command_markers: Accepted user command tags.
|
|
204
|
+
"""
|
|
205
|
+
for each_line in all_lines_newest_first:
|
|
206
|
+
all_entry_fields = _relevant_entry(each_line, skill_names)
|
|
207
|
+
if all_entry_fields is None:
|
|
208
|
+
continue
|
|
209
|
+
if all_entry_fields.get("subtype") == COMPACT_BOUNDARY_SUBTYPE:
|
|
210
|
+
return False
|
|
211
|
+
if _invokes_skill(all_entry_fields, skill_names, slash_command_markers):
|
|
212
|
+
return True
|
|
213
|
+
return False
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _always_reads() -> bool:
|
|
217
|
+
return True
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _notes_nothing() -> None:
|
|
221
|
+
return None
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
class ScanCaps(NamedTuple):
|
|
225
|
+
"""The byte caps a newest-first read holds to, and what it calls when one is passed.
|
|
226
|
+
|
|
227
|
+
::
|
|
228
|
+
|
|
229
|
+
a line over line_byte_cap -> the read stops; on_cap_passed runs once
|
|
230
|
+
a copied stream over forward_copy_byte_cap -> no lines; on_cap_passed runs once
|
|
231
|
+
"""
|
|
232
|
+
|
|
233
|
+
line_byte_cap: int = TRANSCRIPT_LINE_BYTE_CAP
|
|
234
|
+
forward_copy_byte_cap: int = FORWARD_COPY_BYTE_CAP
|
|
235
|
+
on_cap_passed: Callable[[], None] = _notes_nothing
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def _block_ending_at(
|
|
239
|
+
transcript: BinaryIO, end: int, block_size: int
|
|
240
|
+
) -> tuple[int, bytes]:
|
|
241
|
+
start = max(0, end - block_size)
|
|
242
|
+
transcript.seek(start)
|
|
243
|
+
block = transcript.read(end - start)
|
|
244
|
+
if start > 0 and block.startswith(NEWLINE_BYTE):
|
|
245
|
+
transcript.seek(start - 1)
|
|
246
|
+
if transcript.read(1) == CARRIAGE_RETURN_BYTE:
|
|
247
|
+
return start - 1, CARRIAGE_RETURN_BYTE + block
|
|
248
|
+
return start, block
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _blocks_newest_first(
|
|
252
|
+
transcript: BinaryIO, block_size: int, keeps_reading: Callable[[], bool]
|
|
253
|
+
) -> Iterator[tuple[int, bytes]]:
|
|
254
|
+
end = transcript.seek(0, SEEK_END)
|
|
255
|
+
while end > 0 and keeps_reading():
|
|
256
|
+
end, block = _block_ending_at(transcript, end, block_size)
|
|
257
|
+
yield end, block
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def _block_pieces(block: bytes) -> list[bytes]:
|
|
261
|
+
all_pieces = block.splitlines()
|
|
262
|
+
if block.endswith(ALL_LINE_BREAK_BYTES):
|
|
263
|
+
all_pieces.append(b"")
|
|
264
|
+
return all_pieces
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _complete_lines(
|
|
268
|
+
all_pieces: list[bytes], all_newer_parts: list[bytes], is_file_end: bool
|
|
269
|
+
) -> list[bytes]:
|
|
270
|
+
newest_line = all_pieces[-1] + b"".join(reversed(all_newer_parts))
|
|
271
|
+
all_older_lines = list(reversed(all_pieces[1:-1]))
|
|
272
|
+
return (
|
|
273
|
+
[newest_line, *all_older_lines]
|
|
274
|
+
if newest_line or not is_file_end
|
|
275
|
+
else all_older_lines
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def _lines_within_cap(
|
|
280
|
+
all_lines: list[bytes], may_pass_cap: bool, caps: ScanCaps
|
|
281
|
+
) -> tuple[list[bytes], bool]:
|
|
282
|
+
if not may_pass_cap:
|
|
283
|
+
return all_lines, True
|
|
284
|
+
for each_index, each_line in enumerate(all_lines):
|
|
285
|
+
if len(each_line) > caps.line_byte_cap:
|
|
286
|
+
return all_lines[:each_index], False
|
|
287
|
+
return all_lines, True
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _byte_lines_newest_first(
|
|
291
|
+
transcript: BinaryIO,
|
|
292
|
+
block_size: int,
|
|
293
|
+
keeps_reading: Callable[[], bool],
|
|
294
|
+
caps: ScanCaps,
|
|
295
|
+
) -> Iterator[bytes]:
|
|
296
|
+
all_newer_parts: list[bytes] = []
|
|
297
|
+
newer_byte_count = 0
|
|
298
|
+
is_file_end = True
|
|
299
|
+
oldest_start = 0
|
|
300
|
+
for each_start, each_block in _blocks_newest_first(
|
|
301
|
+
transcript, block_size, keeps_reading
|
|
302
|
+
):
|
|
303
|
+
oldest_start = each_start
|
|
304
|
+
all_pieces = _block_pieces(each_block)
|
|
305
|
+
is_within_cap = True
|
|
306
|
+
if len(all_pieces) > 1:
|
|
307
|
+
all_lines = _complete_lines(all_pieces, all_newer_parts, is_file_end)
|
|
308
|
+
may_pass_cap = len(each_block) + newer_byte_count > caps.line_byte_cap
|
|
309
|
+
all_lines, is_within_cap = _lines_within_cap(all_lines, may_pass_cap, caps)
|
|
310
|
+
yield from all_lines
|
|
311
|
+
is_file_end, all_newer_parts, newer_byte_count = False, [], 0
|
|
312
|
+
all_newer_parts.append(all_pieces[0])
|
|
313
|
+
newer_byte_count += len(all_pieces[0])
|
|
314
|
+
if not is_within_cap or newer_byte_count > caps.line_byte_cap:
|
|
315
|
+
caps.on_cap_passed()
|
|
316
|
+
return
|
|
317
|
+
oldest_line = b"".join(reversed(all_newer_parts))
|
|
318
|
+
if oldest_start == 0 and (oldest_line or not is_file_end):
|
|
319
|
+
yield oldest_line
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def _open_line_byte_count(block: bytes, earlier_open_byte_count: int) -> int:
|
|
323
|
+
last_break = max(block.rfind(NEWLINE_BYTE), block.rfind(CARRIAGE_RETURN_BYTE))
|
|
324
|
+
return (
|
|
325
|
+
earlier_open_byte_count + len(block)
|
|
326
|
+
if last_break < 0
|
|
327
|
+
else len(block) - last_break - 1
|
|
328
|
+
)
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def _forward_copy(
|
|
332
|
+
transcript: BinaryIO,
|
|
333
|
+
block_size: int,
|
|
334
|
+
keeps_reading: Callable[[], bool],
|
|
335
|
+
caps: ScanCaps,
|
|
336
|
+
) -> BinaryIO:
|
|
337
|
+
copy = io.BytesIO()
|
|
338
|
+
open_line_byte_count = 0
|
|
339
|
+
while keeps_reading():
|
|
340
|
+
block = transcript.read(block_size)
|
|
341
|
+
if not block:
|
|
342
|
+
return copy
|
|
343
|
+
open_line_byte_count = _open_line_byte_count(block, open_line_byte_count)
|
|
344
|
+
is_over_copy_cap = copy.tell() + len(block) > caps.forward_copy_byte_cap
|
|
345
|
+
if is_over_copy_cap or open_line_byte_count > caps.line_byte_cap:
|
|
346
|
+
caps.on_cap_passed()
|
|
347
|
+
break
|
|
348
|
+
copy.write(block)
|
|
349
|
+
return io.BytesIO()
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def _seeks_to_end(transcript: BinaryIO) -> bool:
|
|
353
|
+
if not transcript.seekable():
|
|
354
|
+
return False
|
|
355
|
+
try:
|
|
356
|
+
transcript.seek(0, SEEK_END)
|
|
357
|
+
except OSError:
|
|
358
|
+
return False
|
|
359
|
+
return True
|
|
360
|
+
|
|
361
|
+
|
|
362
|
+
def _seekable_byte_lines(
|
|
363
|
+
transcript: BinaryIO,
|
|
364
|
+
block_size: int,
|
|
365
|
+
keeps_reading: Callable[[], bool],
|
|
366
|
+
caps: ScanCaps,
|
|
367
|
+
) -> Iterator[bytes]:
|
|
368
|
+
if not _seeks_to_end(transcript):
|
|
369
|
+
transcript = _forward_copy(transcript, block_size, keeps_reading, caps)
|
|
370
|
+
yield from _byte_lines_newest_first(transcript, block_size, keeps_reading, caps)
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def transcript_lines_newest_first(
|
|
374
|
+
transcript: BinaryIO,
|
|
375
|
+
encoding: str,
|
|
376
|
+
errors: str,
|
|
377
|
+
block_size: int = TRANSCRIPT_BLOCK_BYTES,
|
|
378
|
+
keeps_reading: Callable[[], bool] = _always_reads,
|
|
379
|
+
caps: ScanCaps | None = None,
|
|
380
|
+
) -> Iterator[str]:
|
|
381
|
+
"""Yield each line of a binary transcript, newest first, split where text mode splits it.
|
|
382
|
+
|
|
383
|
+
::
|
|
384
|
+
|
|
385
|
+
b"one\\ntwo\\r\\nthree\\rfour" -> "four", "three", "two", "one"
|
|
386
|
+
|
|
387
|
+
``\\n``, ``\\r\\n`` and a lone ``\\r`` end lines. A no from ``keeps_reading``, or a passed
|
|
388
|
+
cap, stops the reader. A transcript that cannot seek to its end is copied forward first.
|
|
389
|
+
|
|
390
|
+
Args:
|
|
391
|
+
transcript: The transcript opened in binary mode.
|
|
392
|
+
encoding: The text encoding of the transcript.
|
|
393
|
+
errors: The decode errors policy, such as ``"replace"``.
|
|
394
|
+
block_size: The bytes read from the file at a time.
|
|
395
|
+
keeps_reading: Whether the reader may read one more block.
|
|
396
|
+
caps: The byte caps, or None for the default ``ScanCaps()``.
|
|
397
|
+
"""
|
|
398
|
+
scan_caps = ScanCaps() if caps is None else caps
|
|
399
|
+
all_byte_lines = _seekable_byte_lines(
|
|
400
|
+
transcript, block_size, keeps_reading, scan_caps
|
|
401
|
+
)
|
|
402
|
+
return (each_line.decode(encoding, errors) for each_line in all_byte_lines)
|
|
403
|
+
|
|
404
|
+
|
|
174
405
|
def _is_user_prompt(all_entry_fields: dict[str, object]) -> bool:
|
|
175
406
|
message = all_entry_fields.get("message")
|
|
176
407
|
if all_entry_fields.get("type") != USER_ENTRY_TYPE or all_entry_fields.get(META_ENTRY_FIELD):
|