@ictechgy/context-guard 0.4.16 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +74 -0
- package/README.ko.md +91 -1
- package/README.md +95 -1
- package/docs/distribution.md +100 -0
- package/package.json +5 -1
- package/plugins/context-guard/.claude-plugin/plugin.json +1 -1
- package/plugins/context-guard/README.ko.md +34 -1
- package/plugins/context-guard/README.md +36 -1
- package/plugins/context-guard/bin/bash_reference_policy.py +967 -0
- package/plugins/context-guard/bin/context-guard-artifact +1 -1
- package/plugins/context-guard/bin/context-guard-bench +4216 -103
- package/plugins/context-guard/bin/context-guard-failed-nudge +95 -23
- package/plugins/context-guard/bin/context-guard-guard-read +6 -2
- package/plugins/context-guard/bin/context-guard-mcp +2 -1
- package/plugins/context-guard/bin/context-guard-pack +2086 -142
- package/plugins/context-guard/bin/context-guard-rewrite-bash +497 -45
- package/plugins/context-guard/bin/context-guard-sanitize-output +178 -22
- package/plugins/context-guard/bin/context-guard-setup +901 -28
- package/plugins/context-guard/bin/context-guard-statusline +33 -2
- package/plugins/context-guard/bin/context-guard-statusline-merged +71 -20
- package/plugins/context-guard/bin/context-guard-task-memory +635 -0
- package/plugins/context-guard/bin/context-guard-trim-output +706 -35
- package/plugins/context-guard/lib/context_guard_commands.py +25 -1
- package/plugins/context-guard/lib/context_pack_git_boundary.py +19 -0
- package/plugins/context-guard/lib/context_pack_identity.py +115 -0
- package/plugins/context-guard/lib/context_pack_receipts.py +9 -0
- package/plugins/context-guard/lib/context_pack_rendering.py +12 -0
- package/plugins/context-guard/lib/context_pack_scanning.py +10 -0
- package/plugins/context-guard/lib/context_pack_selection.py +6 -0
- package/plugins/context-guard/lib/credential_policy.py +10 -2
|
@@ -23,6 +23,7 @@ import posixpath
|
|
|
23
23
|
from pathlib import Path
|
|
24
24
|
import re
|
|
25
25
|
import shlex
|
|
26
|
+
import signal
|
|
26
27
|
import stat
|
|
27
28
|
import subprocess
|
|
28
29
|
import sys
|
|
@@ -48,7 +49,11 @@ AUTO_SCHEMA_VERSION = "contextguard.pack-auto.v1"
|
|
|
48
49
|
AUTO_EXPLAIN_SCHEMA_VERSION = "contextguard.pack-auto-explain.v1"
|
|
49
50
|
REPO_MAP_SCHEMA_VERSION = "contextguard.pack-repo-map.v1"
|
|
50
51
|
ADAPTIVE_K_SCHEMA_VERSION = "contextguard.pack-adaptive-k.v1"
|
|
52
|
+
ADAPTIVE_K_APPLICATION_SCHEMA_VERSION = "contextguard.pack-adaptive-k-application.v1"
|
|
51
53
|
SYMBOL_MEMORY_SCHEMA_VERSION = "contextguard.pack-symbol-memory.v1"
|
|
54
|
+
GRAPH_APPLICATION_SCHEMA_VERSION = "contextguard.pack-graph-application.v1"
|
|
55
|
+
SELF_FINANCING_SELECTION_SCHEMA_VERSION = "contextguard.pack-self-financing-selection.v1"
|
|
56
|
+
SELECTION_PLAN_SCHEMA_VERSION = "contextguard.pack-selection-plan.v1"
|
|
52
57
|
CONTENT_ADDRESS_SCHEMA_VERSION = "contextguard.pack-content-address.v1"
|
|
53
58
|
ROLLING_DELTA_SCHEMA_VERSION = "contextguard.pack-rolling-delta.v1"
|
|
54
59
|
SKETCH_DUPLICATE_SHINGLE_WIDTH = 5
|
|
@@ -65,10 +70,24 @@ DEFAULT_SUGGEST_CONTEXT_LINES = 20
|
|
|
65
70
|
MAX_SUGGEST_CONTEXT_LINES = 120
|
|
66
71
|
SUGGEST_WHOLE_FILE_MAX_LINES = 120
|
|
67
72
|
MAX_SUGGEST_INPUT_BYTES = 256_000
|
|
73
|
+
MAX_GIT_DIFF_STDERR_BYTES = 16_000
|
|
74
|
+
GIT_DIFF_TIMEOUT_SECONDS = 10.0
|
|
68
75
|
MAX_QUERY_SCAN_FILES = 2_000
|
|
69
76
|
MAX_QUERY_SCAN_BYTES_PER_FILE = 200_000
|
|
70
77
|
MAX_GIT_LS_FILES_OUTPUT_BYTES = MAX_QUERY_SCAN_FILES * 512
|
|
71
78
|
GIT_LS_FILES_READ_CHUNK_BYTES = 64 * 1024
|
|
79
|
+
MAX_GIT_ATTR_INPUT_BYTES = MAX_GIT_LS_FILES_OUTPUT_BYTES
|
|
80
|
+
MAX_GIT_ATTR_OUTPUT_BYTES = MAX_GIT_ATTR_INPUT_BYTES * 2
|
|
81
|
+
GIT_ATTR_TIMEOUT_SECONDS = 10.0
|
|
82
|
+
MAX_QUERY_WALK_DIRS = 2_000
|
|
83
|
+
MAX_QUERY_WALK_ENTRIES = 10_000
|
|
84
|
+
MAX_QUERY_WALK_DEPTH = 32
|
|
85
|
+
MAX_QUERY_WALK_SECONDS = 2.0
|
|
86
|
+
MAX_SOURCE_INPUT_BYTES = 4_000_000
|
|
87
|
+
MAX_SOURCE_INPUT_LINES = 100_000
|
|
88
|
+
MAX_SOURCE_LINE_BYTES = 256_000
|
|
89
|
+
MAX_TOTAL_SOURCE_INPUT_BYTES = 16_000_000
|
|
90
|
+
MAX_TOTAL_SOURCE_INPUT_LINES = 400_000
|
|
72
91
|
MAX_REPO_MAP_FILES = 1_000
|
|
73
92
|
MAX_REPO_MAP_SCAN_FILES = 160
|
|
74
93
|
MAX_REPO_MAP_BYTES_PER_FILE = 120_000
|
|
@@ -85,6 +104,8 @@ MAX_ADAPTIVE_K_VERIFICATION_HINTS = 12
|
|
|
85
104
|
ADAPTIVE_K_POLICIES = ("balanced", "recall", "precision")
|
|
86
105
|
MAX_SYMBOL_MEMORY_ITEMS = 12
|
|
87
106
|
MAX_SYMBOL_MEMORY_GRAPH_ITEMS = 12
|
|
107
|
+
MAX_GRAPH_APPLICATION_SOURCES = 4
|
|
108
|
+
MAX_GRAPH_APPLICATION_LINES = 80
|
|
88
109
|
PACK_DIR = ".context-guard/packs"
|
|
89
110
|
REDACTED_PATH_COMPONENT = "[REDACTED-PATH-COMPONENT]"
|
|
90
111
|
ALLOWED_FIRST_ABSOLUTE_SYMLINKS = {
|
|
@@ -178,6 +199,97 @@ class ResolvedSource:
|
|
|
178
199
|
selected_lines: list[str]
|
|
179
200
|
total_lines: int
|
|
180
201
|
redacted_lines: int
|
|
202
|
+
total_lines_exact: bool = True
|
|
203
|
+
input_bytes_read: int = 0
|
|
204
|
+
input_lines_read: int = 0
|
|
205
|
+
sanitized_through_line: int = 0
|
|
206
|
+
input_limit_reason: str | None = None
|
|
207
|
+
redacted_lines_exact: bool = True
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
@dataclass(frozen=True)
|
|
211
|
+
class _SourceScanResult:
|
|
212
|
+
selected_lines: tuple[str, ...]
|
|
213
|
+
total_lines: int
|
|
214
|
+
redacted_lines: int
|
|
215
|
+
total_lines_exact: bool
|
|
216
|
+
input_bytes_read: int
|
|
217
|
+
input_lines_read: int
|
|
218
|
+
sanitized_through_line: int
|
|
219
|
+
limit_reason: str | None
|
|
220
|
+
selection_complete: bool
|
|
221
|
+
redacted_lines_exact: bool
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
@dataclass(frozen=True)
|
|
225
|
+
class _SourceSnapshot:
|
|
226
|
+
identity: tuple[int, int, int, int, int, int, int, int]
|
|
227
|
+
display_path: str
|
|
228
|
+
redacted_path: bool
|
|
229
|
+
requested_lines: LineRange
|
|
230
|
+
selected_lines: tuple[str, ...]
|
|
231
|
+
total_lines: int
|
|
232
|
+
redacted_lines: int
|
|
233
|
+
total_lines_exact: bool
|
|
234
|
+
input_bytes_read: int
|
|
235
|
+
input_lines_read: int
|
|
236
|
+
sanitized_through_line: int
|
|
237
|
+
input_limit_reason: str | None
|
|
238
|
+
redacted_lines_exact: bool
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
class _SourceInputBudget:
|
|
242
|
+
def __init__(self) -> None:
|
|
243
|
+
self.bytes_read = 0
|
|
244
|
+
self.lines_read = 0
|
|
245
|
+
self.bytes_attempted = 0
|
|
246
|
+
self.lines_attempted = 0
|
|
247
|
+
self.bytes_charged = 0
|
|
248
|
+
self.lines_charged = 0
|
|
249
|
+
self.capped = (
|
|
250
|
+
MAX_TOTAL_SOURCE_INPUT_BYTES <= 0
|
|
251
|
+
or MAX_TOTAL_SOURCE_INPUT_LINES <= 0
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
def remaining_bytes(self) -> int:
|
|
255
|
+
return max(0, MAX_TOTAL_SOURCE_INPUT_BYTES - self.bytes_charged)
|
|
256
|
+
|
|
257
|
+
def remaining_lines(self) -> int:
|
|
258
|
+
return max(0, MAX_TOTAL_SOURCE_INPUT_LINES - self.lines_charged)
|
|
259
|
+
|
|
260
|
+
def record_read(self, bytes_count: int) -> str | None:
|
|
261
|
+
bytes_remaining = self.remaining_bytes()
|
|
262
|
+
lines_remaining = self.remaining_lines()
|
|
263
|
+
self.bytes_read += bytes_count
|
|
264
|
+
self.lines_read += 1
|
|
265
|
+
self.bytes_attempted += bytes_count
|
|
266
|
+
self.lines_attempted += 1
|
|
267
|
+
self.bytes_charged = min(
|
|
268
|
+
MAX_TOTAL_SOURCE_INPUT_BYTES,
|
|
269
|
+
self.bytes_charged + bytes_count,
|
|
270
|
+
)
|
|
271
|
+
self.lines_charged = min(
|
|
272
|
+
MAX_TOTAL_SOURCE_INPUT_LINES,
|
|
273
|
+
self.lines_charged + 1,
|
|
274
|
+
)
|
|
275
|
+
self.capped = (
|
|
276
|
+
self.bytes_charged >= MAX_TOTAL_SOURCE_INPUT_BYTES
|
|
277
|
+
or self.lines_charged >= MAX_TOTAL_SOURCE_INPUT_LINES
|
|
278
|
+
)
|
|
279
|
+
if lines_remaining <= 0:
|
|
280
|
+
return "cumulative_input_lines_exceeded"
|
|
281
|
+
if bytes_count > bytes_remaining:
|
|
282
|
+
return "cumulative_input_bytes_exceeded"
|
|
283
|
+
return None
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
class _SourceSnapshotCache:
|
|
287
|
+
def __init__(self) -> None:
|
|
288
|
+
self.entries: dict[tuple[str, str, str], _SourceSnapshot] = {}
|
|
289
|
+
|
|
290
|
+
@staticmethod
|
|
291
|
+
def key(rel: Path, requested: LineRange | None, context: str) -> tuple[str, str, str]:
|
|
292
|
+
return (rel.as_posix(), requested.identity() if requested is not None else "all", context)
|
|
181
293
|
|
|
182
294
|
|
|
183
295
|
@dataclass
|
|
@@ -354,12 +466,33 @@ def sanitize_source_lines(
|
|
|
354
466
|
context: str = "source_code",
|
|
355
467
|
private_roots: tuple[str, ...] = (),
|
|
356
468
|
) -> tuple[list[str], int, int]:
|
|
357
|
-
"""
|
|
469
|
+
"""Compatibility wrapper for the bounded source scanner."""
|
|
470
|
+
scan = _scan_source_lines(
|
|
471
|
+
handle,
|
|
472
|
+
requested,
|
|
473
|
+
context=context,
|
|
474
|
+
private_roots=private_roots,
|
|
475
|
+
input_budget=_SourceInputBudget(),
|
|
476
|
+
)
|
|
477
|
+
return list(scan.selected_lines), scan.total_lines, scan.redacted_lines
|
|
478
|
+
|
|
358
479
|
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
480
|
+
def _scan_source_lines(
|
|
481
|
+
handle: Any,
|
|
482
|
+
requested: LineRange | None,
|
|
483
|
+
*,
|
|
484
|
+
context: str,
|
|
485
|
+
private_roots: tuple[str, ...],
|
|
486
|
+
input_budget: _SourceInputBudget,
|
|
487
|
+
expected_size_bytes: int | None = None,
|
|
488
|
+
) -> _SourceScanResult:
|
|
489
|
+
"""Read with byte/line caps and sanitize only the required prefix.
|
|
490
|
+
|
|
491
|
+
Stateful sanitizers still see every line through ``requested.end``. The
|
|
492
|
+
remaining tail is counted without invoking the sanitizer so range requests
|
|
493
|
+
do not pay sanitizer cost for irrelevant content. If counting reaches a
|
|
494
|
+
cap, the selected range remains usable and the total is explicitly marked
|
|
495
|
+
as a lower bound.
|
|
363
496
|
"""
|
|
364
497
|
sanitizer = load_line_sanitizer(
|
|
365
498
|
context=context,
|
|
@@ -368,16 +501,95 @@ def sanitize_source_lines(
|
|
|
368
501
|
selected: list[str] = []
|
|
369
502
|
redacted = 0
|
|
370
503
|
total_lines = 0
|
|
504
|
+
input_bytes = 0
|
|
505
|
+
input_lines = 0
|
|
371
506
|
collect_all = requested is None
|
|
372
507
|
start = requested.start if requested is not None else 1
|
|
373
508
|
end = requested.end if requested is not None else 0
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
509
|
+
total_lines_exact = False
|
|
510
|
+
limit_reason: str | None = None
|
|
511
|
+
redacted_lines_exact = True
|
|
512
|
+
iterator = None if callable(getattr(handle, "readline", None)) else iter(handle)
|
|
513
|
+
|
|
514
|
+
while True:
|
|
515
|
+
boundary_reason: str | None = None
|
|
516
|
+
if total_lines >= MAX_SOURCE_INPUT_LINES:
|
|
517
|
+
boundary_reason = "source_input_lines_exceeded"
|
|
518
|
+
elif input_budget.remaining_lines() <= 0:
|
|
519
|
+
boundary_reason = "cumulative_input_lines_exceeded"
|
|
520
|
+
source_remaining = MAX_SOURCE_INPUT_BYTES - input_bytes
|
|
521
|
+
cumulative_remaining = input_budget.remaining_bytes()
|
|
522
|
+
if boundary_reason is None and source_remaining <= 0:
|
|
523
|
+
boundary_reason = "source_input_bytes_exceeded"
|
|
524
|
+
elif boundary_reason is None and cumulative_remaining <= 0:
|
|
525
|
+
boundary_reason = "cumulative_input_bytes_exceeded"
|
|
526
|
+
if boundary_reason is not None:
|
|
527
|
+
if expected_size_bytes is not None:
|
|
528
|
+
try:
|
|
529
|
+
if handle.tell() == expected_size_bytes:
|
|
530
|
+
total_lines_exact = True
|
|
531
|
+
break
|
|
532
|
+
except (AttributeError, OSError):
|
|
533
|
+
pass
|
|
534
|
+
limit_reason = boundary_reason
|
|
535
|
+
break
|
|
536
|
+
read_char_cap = min(MAX_SOURCE_LINE_BYTES, source_remaining, cumulative_remaining)
|
|
537
|
+
try:
|
|
538
|
+
if iterator is None:
|
|
539
|
+
raw_line = handle.readline(read_char_cap + 1)
|
|
540
|
+
else:
|
|
541
|
+
raw_line = next(iterator, "")
|
|
542
|
+
except (OSError, UnicodeError):
|
|
543
|
+
limit_reason = "unsafe_path"
|
|
544
|
+
break
|
|
545
|
+
if raw_line == "":
|
|
546
|
+
total_lines_exact = True
|
|
547
|
+
break
|
|
548
|
+
raw_bytes = byte_len(raw_line)
|
|
549
|
+
cumulative_reason = input_budget.record_read(raw_bytes)
|
|
550
|
+
input_bytes += raw_bytes
|
|
551
|
+
input_lines += 1
|
|
552
|
+
if len(raw_line) > MAX_SOURCE_LINE_BYTES or raw_bytes > MAX_SOURCE_LINE_BYTES:
|
|
553
|
+
limit_reason = "source_line_bytes_exceeded"
|
|
554
|
+
break
|
|
555
|
+
if raw_bytes > source_remaining:
|
|
556
|
+
limit_reason = "source_input_bytes_exceeded"
|
|
557
|
+
break
|
|
558
|
+
if cumulative_reason is not None:
|
|
559
|
+
limit_reason = cumulative_reason
|
|
560
|
+
break
|
|
561
|
+
total_lines += 1
|
|
562
|
+
|
|
563
|
+
must_sanitize = collect_all or total_lines <= end
|
|
564
|
+
if must_sanitize:
|
|
565
|
+
sanitized, did_redact = sanitizer.sanitize(raw_line) # type: ignore[attr-defined]
|
|
566
|
+
if did_redact:
|
|
567
|
+
redacted += 1
|
|
568
|
+
if collect_all or start <= total_lines <= end:
|
|
569
|
+
selected.append(sanitized)
|
|
570
|
+
else:
|
|
571
|
+
redacted_lines_exact = False
|
|
572
|
+
if SECRET_CONTENT_RE.search(raw_line):
|
|
573
|
+
redacted += 1
|
|
574
|
+
|
|
575
|
+
selection_complete = (
|
|
576
|
+
total_lines_exact
|
|
577
|
+
if collect_all
|
|
578
|
+
else total_lines >= end or (total_lines_exact and total_lines >= start)
|
|
579
|
+
)
|
|
580
|
+
sanitized_through = total_lines if collect_all else min(total_lines, end)
|
|
581
|
+
return _SourceScanResult(
|
|
582
|
+
selected_lines=tuple(selected),
|
|
583
|
+
total_lines=total_lines,
|
|
584
|
+
redacted_lines=redacted,
|
|
585
|
+
total_lines_exact=total_lines_exact,
|
|
586
|
+
input_bytes_read=input_bytes,
|
|
587
|
+
input_lines_read=input_lines,
|
|
588
|
+
sanitized_through_line=sanitized_through,
|
|
589
|
+
limit_reason=limit_reason,
|
|
590
|
+
selection_complete=selection_complete,
|
|
591
|
+
redacted_lines_exact=redacted_lines_exact,
|
|
592
|
+
)
|
|
381
593
|
|
|
382
594
|
|
|
383
595
|
def byte_len(text: str) -> int:
|
|
@@ -1165,7 +1377,69 @@ def open_regular_under_root(root: Path, rel: Path) -> tuple[Any | None, str]:
|
|
|
1165
1377
|
return None, "unsafe_path"
|
|
1166
1378
|
|
|
1167
1379
|
|
|
1168
|
-
def
|
|
1380
|
+
def _open_source_identity(handle: Any) -> tuple[int, int, int, int, int, int, int, int] | None:
|
|
1381
|
+
try:
|
|
1382
|
+
st = os.fstat(handle.fileno())
|
|
1383
|
+
except (AttributeError, OSError):
|
|
1384
|
+
return None
|
|
1385
|
+
return (
|
|
1386
|
+
st.st_dev,
|
|
1387
|
+
st.st_ino,
|
|
1388
|
+
st.st_mode,
|
|
1389
|
+
st.st_uid,
|
|
1390
|
+
st.st_nlink,
|
|
1391
|
+
st.st_size,
|
|
1392
|
+
int(getattr(st, "st_mtime_ns", int(st.st_mtime * 1_000_000_000))),
|
|
1393
|
+
int(getattr(st, "st_ctime_ns", int(st.st_ctime * 1_000_000_000))),
|
|
1394
|
+
)
|
|
1395
|
+
|
|
1396
|
+
|
|
1397
|
+
def _input_limit_metadata(reason: str) -> dict[str, Any]:
|
|
1398
|
+
caps = {
|
|
1399
|
+
"source_line_bytes_exceeded": ("source_line_bytes", MAX_SOURCE_LINE_BYTES),
|
|
1400
|
+
"source_input_bytes_exceeded": ("source_bytes", MAX_SOURCE_INPUT_BYTES),
|
|
1401
|
+
"source_input_lines_exceeded": ("source_lines", MAX_SOURCE_INPUT_LINES),
|
|
1402
|
+
"cumulative_input_bytes_exceeded": ("cumulative_bytes", MAX_TOTAL_SOURCE_INPUT_BYTES),
|
|
1403
|
+
"cumulative_input_lines_exceeded": ("cumulative_lines", MAX_TOTAL_SOURCE_INPUT_LINES),
|
|
1404
|
+
}
|
|
1405
|
+
kind, cap = caps.get(reason, ("unknown", 0))
|
|
1406
|
+
return {"kind": kind, "cap_bytes" if "bytes" in kind else "cap_lines": cap}
|
|
1407
|
+
|
|
1408
|
+
|
|
1409
|
+
def _snapshot_to_source(
|
|
1410
|
+
snapshot: _SourceSnapshot,
|
|
1411
|
+
*,
|
|
1412
|
+
root: Path,
|
|
1413
|
+
rel: Path,
|
|
1414
|
+
spec: SourceSpec,
|
|
1415
|
+
) -> ResolvedSource:
|
|
1416
|
+
return ResolvedSource(
|
|
1417
|
+
spec=spec,
|
|
1418
|
+
abs_path=root / rel,
|
|
1419
|
+
display_path=snapshot.display_path,
|
|
1420
|
+
redacted_path=snapshot.redacted_path,
|
|
1421
|
+
requested_lines=spec.lines or snapshot.requested_lines,
|
|
1422
|
+
selected_lines=list(snapshot.selected_lines),
|
|
1423
|
+
total_lines=snapshot.total_lines,
|
|
1424
|
+
redacted_lines=snapshot.redacted_lines,
|
|
1425
|
+
total_lines_exact=snapshot.total_lines_exact,
|
|
1426
|
+
input_bytes_read=snapshot.input_bytes_read,
|
|
1427
|
+
input_lines_read=snapshot.input_lines_read,
|
|
1428
|
+
sanitized_through_line=snapshot.sanitized_through_line,
|
|
1429
|
+
input_limit_reason=snapshot.input_limit_reason,
|
|
1430
|
+
redacted_lines_exact=snapshot.redacted_lines_exact,
|
|
1431
|
+
)
|
|
1432
|
+
|
|
1433
|
+
|
|
1434
|
+
def resolve_source(
|
|
1435
|
+
root: Path,
|
|
1436
|
+
spec: SourceSpec,
|
|
1437
|
+
*,
|
|
1438
|
+
source_cache: _SourceSnapshotCache | None = None,
|
|
1439
|
+
input_budget: _SourceInputBudget | None = None,
|
|
1440
|
+
expected_identity: tuple[int, int, int, int, int, int, int, int] | None = None,
|
|
1441
|
+
require_cached: bool = False,
|
|
1442
|
+
) -> tuple[ResolvedSource | None, dict[str, Any] | None]:
|
|
1169
1443
|
if spec.lines is not None and spec.lines.start < 1:
|
|
1170
1444
|
return None, omission(spec, "invalid_lines")
|
|
1171
1445
|
rel, reason = lexical_rel(spec.path)
|
|
@@ -1175,10 +1449,45 @@ def resolve_source(root: Path, spec: SourceSpec) -> tuple[ResolvedSource | None,
|
|
|
1175
1449
|
handle, reason = open_regular_under_root(root, rel)
|
|
1176
1450
|
if handle is None:
|
|
1177
1451
|
return None, omission(spec, reason, path=display, redacted_path=redacted_path)
|
|
1452
|
+
scan_budget = input_budget if input_budget is not None else _SourceInputBudget()
|
|
1453
|
+
requested = spec.lines
|
|
1454
|
+
cache_key = _SourceSnapshotCache.key(rel, requested, spec.sanitization_context)
|
|
1178
1455
|
try:
|
|
1179
1456
|
with handle:
|
|
1180
|
-
|
|
1181
|
-
|
|
1457
|
+
before_identity = _open_source_identity(handle)
|
|
1458
|
+
if expected_identity is not None and before_identity != expected_identity:
|
|
1459
|
+
return None, omission(
|
|
1460
|
+
spec,
|
|
1461
|
+
"graph_source_changed_since_repo_map_snapshot",
|
|
1462
|
+
path=display,
|
|
1463
|
+
redacted_path=redacted_path,
|
|
1464
|
+
)
|
|
1465
|
+
cached = source_cache.entries.get(cache_key) if source_cache is not None else None
|
|
1466
|
+
if require_cached and cached is None:
|
|
1467
|
+
return None, omission(
|
|
1468
|
+
spec,
|
|
1469
|
+
"graph_source_snapshot_unavailable",
|
|
1470
|
+
path=display,
|
|
1471
|
+
redacted_path=redacted_path,
|
|
1472
|
+
)
|
|
1473
|
+
if cached is not None:
|
|
1474
|
+
if before_identity != cached.identity:
|
|
1475
|
+
return None, omission(
|
|
1476
|
+
spec,
|
|
1477
|
+
"source_changed_during_auto",
|
|
1478
|
+
path=display,
|
|
1479
|
+
redacted_path=redacted_path,
|
|
1480
|
+
)
|
|
1481
|
+
source = _snapshot_to_source(cached, root=root, rel=rel, spec=spec)
|
|
1482
|
+
if _open_source_identity(handle) != before_identity:
|
|
1483
|
+
return None, omission(
|
|
1484
|
+
spec,
|
|
1485
|
+
"source_changed_during_auto",
|
|
1486
|
+
path=display,
|
|
1487
|
+
redacted_path=redacted_path,
|
|
1488
|
+
)
|
|
1489
|
+
return source, None
|
|
1490
|
+
scan = _scan_source_lines(
|
|
1182
1491
|
handle,
|
|
1183
1492
|
requested,
|
|
1184
1493
|
context=spec.sanitization_context,
|
|
@@ -1187,17 +1496,38 @@ def resolve_source(root: Path, spec: SourceSpec) -> tuple[ResolvedSource | None,
|
|
|
1187
1496
|
if spec.sanitization_context == "filesystem_listing"
|
|
1188
1497
|
else ()
|
|
1189
1498
|
),
|
|
1499
|
+
input_budget=scan_budget,
|
|
1500
|
+
expected_size_bytes=before_identity[5] if before_identity is not None else None,
|
|
1190
1501
|
)
|
|
1502
|
+
after_identity = _open_source_identity(handle)
|
|
1191
1503
|
except OSError:
|
|
1192
1504
|
return None, omission(spec, "unsafe_path", path=display, redacted_path=redacted_path)
|
|
1505
|
+
if before_identity is not None and after_identity != before_identity:
|
|
1506
|
+
return None, omission(spec, "source_changed_during_read", path=display, redacted_path=redacted_path)
|
|
1507
|
+
if scan.limit_reason == "unsafe_path":
|
|
1508
|
+
return None, omission(spec, "unsafe_path", path=display, redacted_path=redacted_path)
|
|
1509
|
+
if scan.limit_reason is not None and not scan.selection_complete:
|
|
1510
|
+
item = omission(spec, scan.limit_reason, path=display, redacted_path=redacted_path)
|
|
1511
|
+
item["input_limit"] = _input_limit_metadata(scan.limit_reason)
|
|
1512
|
+
item["input_observed"] = {
|
|
1513
|
+
"bytes": scan.input_bytes_read,
|
|
1514
|
+
"lines": scan.input_lines_read,
|
|
1515
|
+
"bytes_attempted": scan.input_bytes_read,
|
|
1516
|
+
"lines_attempted": scan.input_lines_read,
|
|
1517
|
+
"capped": True,
|
|
1518
|
+
}
|
|
1519
|
+
return None, item
|
|
1520
|
+
selected = list(scan.selected_lines)
|
|
1521
|
+
total_lines = scan.total_lines
|
|
1522
|
+
redacted_lines = scan.redacted_lines
|
|
1193
1523
|
if total_lines <= 0:
|
|
1194
1524
|
return None, omission(spec, "empty_source", path=display, redacted_path=redacted_path)
|
|
1195
1525
|
requested = requested or LineRange(1, total_lines)
|
|
1196
|
-
if requested.start > total_lines:
|
|
1526
|
+
if scan.total_lines_exact and requested.start > total_lines:
|
|
1197
1527
|
return None, omission(spec, "empty_source", path=display, redacted_path=redacted_path)
|
|
1198
1528
|
if not selected:
|
|
1199
1529
|
return None, omission(spec, "empty_source", path=display, redacted_path=redacted_path)
|
|
1200
|
-
|
|
1530
|
+
source = ResolvedSource(
|
|
1201
1531
|
spec=spec,
|
|
1202
1532
|
abs_path=root / rel,
|
|
1203
1533
|
display_path=display,
|
|
@@ -1206,7 +1536,33 @@ def resolve_source(root: Path, spec: SourceSpec) -> tuple[ResolvedSource | None,
|
|
|
1206
1536
|
selected_lines=selected,
|
|
1207
1537
|
total_lines=total_lines,
|
|
1208
1538
|
redacted_lines=redacted_lines,
|
|
1209
|
-
|
|
1539
|
+
total_lines_exact=scan.total_lines_exact,
|
|
1540
|
+
input_bytes_read=scan.input_bytes_read,
|
|
1541
|
+
input_lines_read=scan.input_lines_read,
|
|
1542
|
+
sanitized_through_line=scan.sanitized_through_line,
|
|
1543
|
+
input_limit_reason=scan.limit_reason,
|
|
1544
|
+
redacted_lines_exact=scan.redacted_lines_exact,
|
|
1545
|
+
)
|
|
1546
|
+
if source_cache is not None and before_identity is not None:
|
|
1547
|
+
snapshot = _SourceSnapshot(
|
|
1548
|
+
identity=before_identity,
|
|
1549
|
+
display_path=display,
|
|
1550
|
+
redacted_path=redacted_path,
|
|
1551
|
+
requested_lines=requested,
|
|
1552
|
+
selected_lines=tuple(selected),
|
|
1553
|
+
total_lines=total_lines,
|
|
1554
|
+
redacted_lines=redacted_lines,
|
|
1555
|
+
total_lines_exact=scan.total_lines_exact,
|
|
1556
|
+
input_bytes_read=scan.input_bytes_read,
|
|
1557
|
+
input_lines_read=scan.input_lines_read,
|
|
1558
|
+
sanitized_through_line=scan.sanitized_through_line,
|
|
1559
|
+
input_limit_reason=scan.limit_reason,
|
|
1560
|
+
redacted_lines_exact=scan.redacted_lines_exact,
|
|
1561
|
+
)
|
|
1562
|
+
source_cache.entries[cache_key] = snapshot
|
|
1563
|
+
canonical_key = _SourceSnapshotCache.key(rel, source_selected_range(source), spec.sanitization_context)
|
|
1564
|
+
source_cache.entries[canonical_key] = snapshot
|
|
1565
|
+
return source, None
|
|
1210
1566
|
|
|
1211
1567
|
|
|
1212
1568
|
def retrieval_cli(root_arg: str, display_path: str, lines: LineRange) -> str:
|
|
@@ -1245,31 +1601,99 @@ def retrieval_for(root_arg: str, display_path: str, lines: LineRange, *, redacte
|
|
|
1245
1601
|
return retrieval_cli(safe_root, display_path, lines), None
|
|
1246
1602
|
|
|
1247
1603
|
|
|
1248
|
-
|
|
1249
|
-
|
|
1604
|
+
def markdown_metadata_text(value: object) -> str:
|
|
1605
|
+
out: list[str] = []
|
|
1606
|
+
for char in str(value):
|
|
1607
|
+
code = ord(char)
|
|
1608
|
+
if not char.isprintable():
|
|
1609
|
+
out.append(f"\\u{code:04X}" if code <= 0xFFFF else f"\\U{code:08X}")
|
|
1610
|
+
elif char == "&":
|
|
1611
|
+
out.append("&")
|
|
1612
|
+
elif char == "<":
|
|
1613
|
+
out.append("<")
|
|
1614
|
+
elif char == ">":
|
|
1615
|
+
out.append(">")
|
|
1616
|
+
elif char in {"[", "]", "(", ")", "!"}:
|
|
1617
|
+
out.append("\\" + char)
|
|
1618
|
+
elif char in {"`", "\\"}:
|
|
1619
|
+
out.append("\\" + char)
|
|
1620
|
+
else:
|
|
1621
|
+
out.append(char)
|
|
1622
|
+
return "".join(out)
|
|
1623
|
+
|
|
1624
|
+
|
|
1625
|
+
def markdown_inline_code(value: object) -> str:
|
|
1626
|
+
text = "".join(
|
|
1627
|
+
(f"\\u{ord(char):04X}" if ord(char) <= 0xFFFF else f"\\U{ord(char):08X}")
|
|
1628
|
+
if not char.isprintable()
|
|
1629
|
+
else char
|
|
1630
|
+
for char in str(value)
|
|
1631
|
+
)
|
|
1632
|
+
max_run = max((len(match.group(0)) for match in re.finditer(r"`+", text)), default=0)
|
|
1633
|
+
delimiter = "`" * max(1, max_run + 1)
|
|
1634
|
+
padding = " " if text.startswith("`") or text.endswith("`") else ""
|
|
1635
|
+
return f"{delimiter}{padding}{text}{padding}{delimiter}"
|
|
1636
|
+
|
|
1637
|
+
|
|
1638
|
+
def markdown_block_delimiters(lines: list[str]) -> tuple[str, str]:
|
|
1639
|
+
max_run = 0
|
|
1640
|
+
for line in lines:
|
|
1641
|
+
line_max = max((len(match.group(0)) for match in re.finditer(r"`+", line)), default=0)
|
|
1642
|
+
max_run = max(max_run, line_max)
|
|
1643
|
+
fence = "`" * max(3, max_run + 1)
|
|
1644
|
+
return f"\n\n{fence}text\n", f"{fence}\n\n"
|
|
1250
1645
|
|
|
1251
1646
|
|
|
1252
1647
|
def render_block_header(source: ResolvedSource, *, root_arg: str, status: str, included: LineRange) -> str:
|
|
1253
|
-
title = source.spec.label or source.display_path
|
|
1648
|
+
title = markdown_metadata_text(source.spec.label or source.display_path)
|
|
1254
1649
|
requested = source.requested_lines or LineRange(1, source.total_lines)
|
|
1255
1650
|
retrieval, retrieval_omitted_reason = retrieval_for(root_arg, source.display_path, included, redacted_path=source.redacted_path)
|
|
1256
1651
|
header = [
|
|
1257
1652
|
f"## {title}",
|
|
1258
|
-
f"Source:
|
|
1653
|
+
f"Source: {markdown_inline_code(source.display_path)}",
|
|
1259
1654
|
f"Priority: {source.spec.priority}",
|
|
1260
1655
|
f"Status: {status}",
|
|
1261
1656
|
f"Included lines: {included.start}:{included.end}",
|
|
1262
1657
|
f"Requested lines: {requested.start}:{requested.end}",
|
|
1263
1658
|
]
|
|
1264
1659
|
if retrieval:
|
|
1265
|
-
header.append(f"Retrieval:
|
|
1660
|
+
header.append(f"Retrieval: {markdown_inline_code(retrieval)}")
|
|
1266
1661
|
elif retrieval_omitted_reason:
|
|
1267
1662
|
header.append(f"Retrieval omitted: {retrieval_omitted_reason}")
|
|
1268
1663
|
return "\n".join(header)
|
|
1269
1664
|
|
|
1270
1665
|
|
|
1271
1666
|
def render_block(source: ResolvedSource, lines: list[str], *, root_arg: str, status: str, included: LineRange) -> str:
|
|
1272
|
-
|
|
1667
|
+
block_open, block_close = markdown_block_delimiters(lines)
|
|
1668
|
+
return render_block_header(source, root_arg=root_arg, status=status, included=included) + block_open + "".join(lines) + ("" if not lines or lines[-1].endswith("\n") else "\n") + block_close
|
|
1669
|
+
|
|
1670
|
+
|
|
1671
|
+
def source_input_metadata(source: ResolvedSource) -> dict[str, Any]:
|
|
1672
|
+
item: dict[str, Any] = {
|
|
1673
|
+
"bytes_read": source.input_bytes_read,
|
|
1674
|
+
"lines_read": source.input_lines_read,
|
|
1675
|
+
"bytes_attempted": source.input_bytes_read,
|
|
1676
|
+
"lines_attempted": source.input_lines_read,
|
|
1677
|
+
"capped": source.input_limit_reason is not None,
|
|
1678
|
+
"total_lines_exact": source.total_lines_exact,
|
|
1679
|
+
"truncated": not source.total_lines_exact,
|
|
1680
|
+
"sanitized_through_line": source.sanitized_through_line,
|
|
1681
|
+
"redacted_lines_exact": source.redacted_lines_exact,
|
|
1682
|
+
"limits": {
|
|
1683
|
+
"source_bytes": MAX_SOURCE_INPUT_BYTES,
|
|
1684
|
+
"source_lines": MAX_SOURCE_INPUT_LINES,
|
|
1685
|
+
"source_line_bytes": MAX_SOURCE_LINE_BYTES,
|
|
1686
|
+
"cumulative_bytes": MAX_TOTAL_SOURCE_INPUT_BYTES,
|
|
1687
|
+
"cumulative_lines": MAX_TOTAL_SOURCE_INPUT_LINES,
|
|
1688
|
+
},
|
|
1689
|
+
}
|
|
1690
|
+
if source.total_lines_exact:
|
|
1691
|
+
item["total_lines"] = source.total_lines
|
|
1692
|
+
else:
|
|
1693
|
+
item["total_lines_lower_bound"] = source.total_lines
|
|
1694
|
+
if source.input_limit_reason is not None:
|
|
1695
|
+
item["limit_reason"] = source.input_limit_reason
|
|
1696
|
+
return item
|
|
1273
1697
|
|
|
1274
1698
|
|
|
1275
1699
|
def source_metadata(source: ResolvedSource, *, status: str, lines: list[str], included: LineRange, root_arg: str) -> dict[str, Any]:
|
|
@@ -1285,6 +1709,7 @@ def source_metadata(source: ResolvedSource, *, status: str, lines: list[str], in
|
|
|
1285
1709
|
}
|
|
1286
1710
|
if source.spec.label:
|
|
1287
1711
|
item["label"] = source.spec.label
|
|
1712
|
+
item["input"] = source_input_metadata(source)
|
|
1288
1713
|
retrieval, retrieval_omitted_reason = retrieval_for(root_arg, source.display_path, included, redacted_path=source.redacted_path)
|
|
1289
1714
|
if retrieval:
|
|
1290
1715
|
item["retrieval_cli"] = retrieval
|
|
@@ -1299,7 +1724,11 @@ def budget_omission(source: ResolvedSource, *, root_arg: str) -> dict[str, Any]:
|
|
|
1299
1724
|
requested = source.requested_lines or LineRange(1, source.total_lines)
|
|
1300
1725
|
item = omission(source.spec, "budget_exhausted", path=source.display_path, redacted_path=source.redacted_path)
|
|
1301
1726
|
item["requested_lines"] = requested.as_dict()
|
|
1302
|
-
|
|
1727
|
+
if source.total_lines_exact:
|
|
1728
|
+
item["total_lines"] = source.total_lines
|
|
1729
|
+
else:
|
|
1730
|
+
item["total_lines_lower_bound"] = source.total_lines
|
|
1731
|
+
item["input"] = source_input_metadata(source)
|
|
1303
1732
|
retrieval, retrieval_omitted_reason = retrieval_for(root_arg, source.display_path, requested, redacted_path=source.redacted_path)
|
|
1304
1733
|
if retrieval:
|
|
1305
1734
|
item["retrieval_cli"] = retrieval
|
|
@@ -1335,7 +1764,8 @@ def render_block_byte_len(
|
|
|
1335
1764
|
body_bytes = line_prefixes[line_count]
|
|
1336
1765
|
if line_count > 0 and not source.selected_lines[line_count - 1].endswith("\n"):
|
|
1337
1766
|
body_bytes += 1
|
|
1338
|
-
|
|
1767
|
+
block_open, block_close = markdown_block_delimiters(source.selected_lines[:line_count])
|
|
1768
|
+
return byte_len(render_block_header(source, root_arg=root_arg, status=status, included=included)) + byte_len(block_open) + body_bytes + byte_len(block_close)
|
|
1339
1769
|
|
|
1340
1770
|
|
|
1341
1771
|
def fit_partial_lines(
|
|
@@ -1834,7 +2264,13 @@ def build_pack(
|
|
|
1834
2264
|
store_artifact: bool,
|
|
1835
2265
|
delta_from_pack_id: str | None = None,
|
|
1836
2266
|
sketch_duplicate_veto: bool = False,
|
|
2267
|
+
_source_cache: _SourceSnapshotCache | None = None,
|
|
2268
|
+
_input_budget: _SourceInputBudget | None = None,
|
|
2269
|
+
_required_snapshot_sources: set[tuple[str, str]] | None = None,
|
|
2270
|
+
_expected_source_identities: dict[str, tuple[int, int, int, int, int, int, int, int]] | None = None,
|
|
2271
|
+
_snapshot_rejections: dict[tuple[str, str], dict[str, Any]] | None = None,
|
|
1837
2272
|
) -> dict[str, Any]:
|
|
2273
|
+
input_budget = _input_budget if _input_budget is not None else _SourceInputBudget()
|
|
1838
2274
|
seen: set[tuple[str, str]] = set()
|
|
1839
2275
|
resolved: list[ResolvedSource] = []
|
|
1840
2276
|
paired_candidates: list[_PairedCandidate] = []
|
|
@@ -1861,7 +2297,55 @@ def build_pack(
|
|
|
1861
2297
|
continue
|
|
1862
2298
|
if rel is not None:
|
|
1863
2299
|
seen.add(identity)
|
|
1864
|
-
|
|
2300
|
+
rel_path = rel.as_posix() if rel is not None else ""
|
|
2301
|
+
require_cached = bool(
|
|
2302
|
+
_required_snapshot_sources
|
|
2303
|
+
and (rel_path, identity_lines) in _required_snapshot_sources
|
|
2304
|
+
)
|
|
2305
|
+
expected_identity = (
|
|
2306
|
+
_expected_source_identities.get(rel_path)
|
|
2307
|
+
if require_cached and _expected_source_identities is not None
|
|
2308
|
+
else None
|
|
2309
|
+
)
|
|
2310
|
+
if require_cached and expected_identity is None:
|
|
2311
|
+
display, redacted = display_rel_path(rel_path)
|
|
2312
|
+
omitted_item = omission(
|
|
2313
|
+
spec,
|
|
2314
|
+
"graph_source_not_in_repo_map_snapshot",
|
|
2315
|
+
path=display,
|
|
2316
|
+
redacted_path=redacted,
|
|
2317
|
+
)
|
|
2318
|
+
omitted.append(omitted_item)
|
|
2319
|
+
canonical_specs.append({
|
|
2320
|
+
"path": display,
|
|
2321
|
+
"priority": spec.priority,
|
|
2322
|
+
"lines": identity_lines,
|
|
2323
|
+
"status": omitted_item.get("reason"),
|
|
2324
|
+
})
|
|
2325
|
+
continue
|
|
2326
|
+
cached_rejection = (
|
|
2327
|
+
_snapshot_rejections.get((rel_path, identity_lines))
|
|
2328
|
+
if require_cached and _snapshot_rejections is not None
|
|
2329
|
+
else None
|
|
2330
|
+
)
|
|
2331
|
+
if cached_rejection is not None:
|
|
2332
|
+
omitted_item = copy.deepcopy(cached_rejection)
|
|
2333
|
+
omitted.append(omitted_item)
|
|
2334
|
+
canonical_specs.append({
|
|
2335
|
+
"path": omitted_item.get("path"),
|
|
2336
|
+
"priority": spec.priority,
|
|
2337
|
+
"lines": identity_lines,
|
|
2338
|
+
"status": omitted_item.get("reason"),
|
|
2339
|
+
})
|
|
2340
|
+
continue
|
|
2341
|
+
source, omitted_item = resolve_source(
|
|
2342
|
+
root,
|
|
2343
|
+
spec,
|
|
2344
|
+
source_cache=_source_cache,
|
|
2345
|
+
input_budget=input_budget,
|
|
2346
|
+
expected_identity=expected_identity,
|
|
2347
|
+
require_cached=require_cached,
|
|
2348
|
+
)
|
|
1865
2349
|
if omitted_item is not None:
|
|
1866
2350
|
omitted.append(omitted_item)
|
|
1867
2351
|
canonical_specs.append({"path": omitted_item.get("path"), "priority": spec.priority, "lines": identity_lines, "status": omitted_item.get("reason")})
|
|
@@ -1936,7 +2420,27 @@ def build_pack(
|
|
|
1936
2420
|
"sources": {"total": len(specs), "included": len(included) - partial_count, "partial": partial_count, "omitted": len(omitted_sorted)},
|
|
1937
2421
|
"included_sources": included,
|
|
1938
2422
|
"omitted_sources": omitted_sorted,
|
|
1939
|
-
"redaction": {
|
|
2423
|
+
"redaction": {
|
|
2424
|
+
"redacted_lines": redacted_lines,
|
|
2425
|
+
"redacted_lines_exact": all(source.redacted_lines_exact for source in all_resolved),
|
|
2426
|
+
"redacted_before_pack": True,
|
|
2427
|
+
},
|
|
2428
|
+
"input": {
|
|
2429
|
+
"bytes_read": input_budget.bytes_read,
|
|
2430
|
+
"lines_read": input_budget.lines_read,
|
|
2431
|
+
"bytes_attempted": input_budget.bytes_attempted,
|
|
2432
|
+
"lines_attempted": input_budget.lines_attempted,
|
|
2433
|
+
"bytes_charged": input_budget.bytes_charged,
|
|
2434
|
+
"lines_charged": input_budget.lines_charged,
|
|
2435
|
+
"capped": input_budget.capped,
|
|
2436
|
+
"limits": {
|
|
2437
|
+
"source_bytes": MAX_SOURCE_INPUT_BYTES,
|
|
2438
|
+
"source_lines": MAX_SOURCE_INPUT_LINES,
|
|
2439
|
+
"source_line_bytes": MAX_SOURCE_LINE_BYTES,
|
|
2440
|
+
"cumulative_bytes": MAX_TOTAL_SOURCE_INPUT_BYTES,
|
|
2441
|
+
"cumulative_lines": MAX_TOTAL_SOURCE_INPUT_LINES,
|
|
2442
|
+
},
|
|
2443
|
+
},
|
|
1940
2444
|
"artifact": {"stored": False, "path": None, "bytes": 0, "capped": False, "cap_bytes": MAX_RECEIPT_BYTES},
|
|
1941
2445
|
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
1942
2446
|
}
|
|
@@ -1982,7 +2486,12 @@ def slice_source(root: Path, *, raw_path: str, lines: LineRange) -> tuple[dict[s
|
|
|
1982
2486
|
"query": {"type": "lines", "start": lines.start, "end": min(lines.end, source.total_lines), "returned_lines": len(source.selected_lines)},
|
|
1983
2487
|
"content": content,
|
|
1984
2488
|
"bytes": byte_len(content),
|
|
1985
|
-
"redaction": {
|
|
2489
|
+
"redaction": {
|
|
2490
|
+
"redacted_lines": source.redacted_lines,
|
|
2491
|
+
"redacted_lines_exact": source.redacted_lines_exact,
|
|
2492
|
+
"redacted_before_pack": True,
|
|
2493
|
+
},
|
|
2494
|
+
"input": source_input_metadata(source),
|
|
1986
2495
|
}
|
|
1987
2496
|
return payload, 0
|
|
1988
2497
|
|
|
@@ -2052,32 +2561,245 @@ def add_suggest_candidate(
|
|
|
2052
2561
|
)
|
|
2053
2562
|
|
|
2054
2563
|
|
|
2564
|
+
def trusted_git_executable() -> str:
|
|
2565
|
+
if os.name != "posix":
|
|
2566
|
+
raise OSError("Git execution is unavailable on this platform")
|
|
2567
|
+
executable_names = ("git",)
|
|
2568
|
+
for directory in os.defpath.split(os.pathsep):
|
|
2569
|
+
if not directory:
|
|
2570
|
+
continue
|
|
2571
|
+
for name in executable_names:
|
|
2572
|
+
candidate = Path(directory) / name
|
|
2573
|
+
try:
|
|
2574
|
+
if candidate.is_file() and os.access(candidate, os.X_OK):
|
|
2575
|
+
return str(candidate)
|
|
2576
|
+
except OSError:
|
|
2577
|
+
continue
|
|
2578
|
+
raise OSError("trusted system git executable unavailable")
|
|
2579
|
+
|
|
2580
|
+
|
|
2581
|
+
def guarded_git_environment() -> dict[str, str]:
|
|
2582
|
+
return {
|
|
2583
|
+
"PATH": os.defpath,
|
|
2584
|
+
"LANG": "C",
|
|
2585
|
+
"LC_ALL": "C",
|
|
2586
|
+
"GIT_CONFIG_GLOBAL": os.devnull,
|
|
2587
|
+
"GIT_CONFIG_SYSTEM": os.devnull,
|
|
2588
|
+
"GIT_CONFIG_NOSYSTEM": "1",
|
|
2589
|
+
"GIT_ATTR_NOSYSTEM": "1",
|
|
2590
|
+
"GIT_TERMINAL_PROMPT": "0",
|
|
2591
|
+
"GIT_ASKPASS": os.devnull,
|
|
2592
|
+
"SSH_ASKPASS": os.devnull,
|
|
2593
|
+
"GCM_INTERACTIVE": "Never",
|
|
2594
|
+
"GIT_NO_LAZY_FETCH": "1",
|
|
2595
|
+
"GIT_OPTIONAL_LOCKS": "0",
|
|
2596
|
+
"GIT_PAGER": "cat",
|
|
2597
|
+
"PAGER": "cat",
|
|
2598
|
+
}
|
|
2599
|
+
|
|
2600
|
+
|
|
2601
|
+
def guarded_git_command(root: Path, *args: str) -> list[str]:
|
|
2602
|
+
return [
|
|
2603
|
+
trusted_git_executable(),
|
|
2604
|
+
"-c",
|
|
2605
|
+
"core.fsmonitor=false",
|
|
2606
|
+
"-c",
|
|
2607
|
+
f"core.hooksPath={os.devnull}",
|
|
2608
|
+
"-c",
|
|
2609
|
+
f"core.attributesFile={os.devnull}",
|
|
2610
|
+
"-c",
|
|
2611
|
+
"credential.helper=",
|
|
2612
|
+
"-c",
|
|
2613
|
+
"core.askPass=",
|
|
2614
|
+
"-c",
|
|
2615
|
+
"credential.interactive=never",
|
|
2616
|
+
"-c",
|
|
2617
|
+
"filter.unset.clean=",
|
|
2618
|
+
"-c",
|
|
2619
|
+
"filter.unset.process=",
|
|
2620
|
+
"-c",
|
|
2621
|
+
"filter.unset.required=false",
|
|
2622
|
+
"-c",
|
|
2623
|
+
"filter.unspecified.clean=",
|
|
2624
|
+
"-c",
|
|
2625
|
+
"filter.unspecified.process=",
|
|
2626
|
+
"-c",
|
|
2627
|
+
"filter.unspecified.required=false",
|
|
2628
|
+
"-C",
|
|
2629
|
+
str(root),
|
|
2630
|
+
*args,
|
|
2631
|
+
]
|
|
2632
|
+
|
|
2633
|
+
|
|
2634
|
+
def _signal_process_group(proc: subprocess.Popen[Any], *, force: bool) -> None:
|
|
2635
|
+
requested_signal = getattr(signal, "SIGKILL", signal.SIGTERM) if force else signal.SIGTERM
|
|
2636
|
+
if os.name == "posix" and hasattr(os, "killpg"):
|
|
2637
|
+
try:
|
|
2638
|
+
os.killpg(proc.pid, requested_signal)
|
|
2639
|
+
return
|
|
2640
|
+
except ProcessLookupError:
|
|
2641
|
+
return
|
|
2642
|
+
except OSError:
|
|
2643
|
+
pass
|
|
2644
|
+
if proc.poll() is not None:
|
|
2645
|
+
return
|
|
2646
|
+
try:
|
|
2647
|
+
if force:
|
|
2648
|
+
proc.kill()
|
|
2649
|
+
else:
|
|
2650
|
+
proc.terminate()
|
|
2651
|
+
except OSError:
|
|
2652
|
+
pass
|
|
2653
|
+
|
|
2654
|
+
|
|
2655
|
+
def _run_process_capped(
|
|
2656
|
+
command: list[str],
|
|
2657
|
+
*,
|
|
2658
|
+
stdout_cap: int,
|
|
2659
|
+
stderr_cap: int,
|
|
2660
|
+
timeout_seconds: float,
|
|
2661
|
+
environment: dict[str, str] | None = None,
|
|
2662
|
+
stdin_data: bytes | None = None,
|
|
2663
|
+
stdin_cap_bytes: int | None = None,
|
|
2664
|
+
) -> tuple[int, bytes, bytes, bool, bool]:
|
|
2665
|
+
if stdin_data is not None and (
|
|
2666
|
+
stdin_cap_bytes is None or len(stdin_data) > stdin_cap_bytes
|
|
2667
|
+
):
|
|
2668
|
+
raise PackError("process stdin exceeds cap")
|
|
2669
|
+
proc = subprocess.Popen(
|
|
2670
|
+
command,
|
|
2671
|
+
stdin=subprocess.PIPE if stdin_data is not None else subprocess.DEVNULL,
|
|
2672
|
+
stdout=subprocess.PIPE,
|
|
2673
|
+
stderr=subprocess.PIPE,
|
|
2674
|
+
text=False,
|
|
2675
|
+
start_new_session=os.name == "posix",
|
|
2676
|
+
env=environment,
|
|
2677
|
+
)
|
|
2678
|
+
buffers: dict[str, list[bytes]] = {"stdout": [], "stderr": []}
|
|
2679
|
+
capped = {"stdout": False, "stderr": False}
|
|
2680
|
+
stop = threading.Event()
|
|
2681
|
+
|
|
2682
|
+
def drain(name: str, stream: Any, cap: int) -> None:
|
|
2683
|
+
total = 0
|
|
2684
|
+
try:
|
|
2685
|
+
while not stop.is_set() and total <= cap:
|
|
2686
|
+
chunk = stream.read(min(64 * 1024, cap + 1 - total))
|
|
2687
|
+
if not chunk:
|
|
2688
|
+
break
|
|
2689
|
+
buffers[name].append(chunk)
|
|
2690
|
+
total += len(chunk)
|
|
2691
|
+
if total > cap:
|
|
2692
|
+
capped[name] = True
|
|
2693
|
+
stop.set()
|
|
2694
|
+
_signal_process_group(proc, force=False)
|
|
2695
|
+
break
|
|
2696
|
+
finally:
|
|
2697
|
+
try:
|
|
2698
|
+
stream.close()
|
|
2699
|
+
except OSError:
|
|
2700
|
+
pass
|
|
2701
|
+
|
|
2702
|
+
threads = [
|
|
2703
|
+
threading.Thread(target=drain, args=("stdout", proc.stdout, stdout_cap), daemon=True),
|
|
2704
|
+
threading.Thread(target=drain, args=("stderr", proc.stderr, stderr_cap), daemon=True),
|
|
2705
|
+
]
|
|
2706
|
+
if stdin_data is not None:
|
|
2707
|
+
def write_stdin() -> None:
|
|
2708
|
+
try:
|
|
2709
|
+
assert proc.stdin is not None
|
|
2710
|
+
view = memoryview(stdin_data)
|
|
2711
|
+
for offset in range(0, len(view), 64 * 1024):
|
|
2712
|
+
if stop.is_set():
|
|
2713
|
+
break
|
|
2714
|
+
proc.stdin.write(view[offset : offset + 64 * 1024])
|
|
2715
|
+
proc.stdin.flush()
|
|
2716
|
+
except (BrokenPipeError, OSError, ValueError):
|
|
2717
|
+
pass
|
|
2718
|
+
finally:
|
|
2719
|
+
if proc.stdin is not None:
|
|
2720
|
+
try:
|
|
2721
|
+
proc.stdin.close()
|
|
2722
|
+
except OSError:
|
|
2723
|
+
pass
|
|
2724
|
+
|
|
2725
|
+
threads.append(threading.Thread(target=write_stdin, daemon=True))
|
|
2726
|
+
for thread in threads:
|
|
2727
|
+
thread.start()
|
|
2728
|
+
timed_out = False
|
|
2729
|
+
try:
|
|
2730
|
+
proc.wait(timeout=timeout_seconds)
|
|
2731
|
+
except subprocess.TimeoutExpired:
|
|
2732
|
+
timed_out = True
|
|
2733
|
+
stop.set()
|
|
2734
|
+
_signal_process_group(proc, force=False)
|
|
2735
|
+
try:
|
|
2736
|
+
proc.wait(timeout=0.2)
|
|
2737
|
+
except subprocess.TimeoutExpired:
|
|
2738
|
+
_signal_process_group(proc, force=True)
|
|
2739
|
+
try:
|
|
2740
|
+
proc.wait(timeout=2)
|
|
2741
|
+
except subprocess.TimeoutExpired:
|
|
2742
|
+
pass
|
|
2743
|
+
if capped["stdout"] or capped["stderr"] or timed_out:
|
|
2744
|
+
_signal_process_group(proc, force=True)
|
|
2745
|
+
for thread in threads:
|
|
2746
|
+
thread.join(0.5)
|
|
2747
|
+
if any(thread.is_alive() for thread in threads):
|
|
2748
|
+
stop.set()
|
|
2749
|
+
_signal_process_group(proc, force=True)
|
|
2750
|
+
for thread in threads:
|
|
2751
|
+
thread.join(0.2)
|
|
2752
|
+
stdout = b"".join(buffers["stdout"])[:stdout_cap]
|
|
2753
|
+
stderr = b"".join(buffers["stderr"])[:stderr_cap]
|
|
2754
|
+
return proc.returncode if proc.returncode is not None else -1, stdout, stderr, capped["stdout"], capped["stderr"] or timed_out
|
|
2755
|
+
|
|
2756
|
+
|
|
2055
2757
|
def run_git_diff(root: Path, diff_ref: str) -> str:
|
|
2056
2758
|
ref = diff_ref.strip()
|
|
2057
2759
|
if not ref:
|
|
2058
2760
|
raise PackError("empty --diff")
|
|
2059
|
-
|
|
2761
|
+
git_args = [
|
|
2762
|
+
"diff",
|
|
2763
|
+
"--no-ext-diff",
|
|
2764
|
+
"--no-textconv",
|
|
2765
|
+
"--ignore-submodules=all",
|
|
2766
|
+
"--unified=3",
|
|
2767
|
+
]
|
|
2060
2768
|
if ref in {"staged", "--staged", "cached", "--cached"}:
|
|
2061
|
-
|
|
2769
|
+
git_args.append("--cached")
|
|
2062
2770
|
elif ref in {"worktree", "unstaged", "working-tree"}:
|
|
2063
2771
|
pass
|
|
2064
2772
|
elif ref.startswith("-"):
|
|
2065
2773
|
raise PackError("invalid --diff: revision must not start with '-'")
|
|
2066
2774
|
else:
|
|
2067
|
-
|
|
2775
|
+
git_args.append(ref)
|
|
2068
2776
|
try:
|
|
2069
|
-
|
|
2777
|
+
reject_configured_git_filters(root)
|
|
2778
|
+
command = guarded_git_command(root, *git_args)
|
|
2779
|
+
returncode, stdout, stderr, stdout_capped, stderr_capped_or_timeout = _run_process_capped(
|
|
2780
|
+
command,
|
|
2781
|
+
stdout_cap=MAX_SUGGEST_INPUT_BYTES,
|
|
2782
|
+
stderr_cap=MAX_GIT_DIFF_STDERR_BYTES,
|
|
2783
|
+
timeout_seconds=GIT_DIFF_TIMEOUT_SECONDS,
|
|
2784
|
+
environment=guarded_git_environment(),
|
|
2785
|
+
)
|
|
2070
2786
|
except (OSError, UnicodeError, subprocess.TimeoutExpired) as exc:
|
|
2071
2787
|
raise PackError(f"could not read diff: {exc.__class__.__name__}") from exc
|
|
2072
|
-
if
|
|
2788
|
+
if stdout_capped:
|
|
2789
|
+
raise PackError(f"could not read diff: diff output exceeds cap ({MAX_SUGGEST_INPUT_BYTES} bytes)")
|
|
2790
|
+
if stderr_capped_or_timeout:
|
|
2791
|
+
raise PackError("could not read diff: stderr cap or timeout exceeded")
|
|
2792
|
+
stdout_text = stdout.decode("utf-8", "replace")
|
|
2793
|
+
stderr_text = stderr.decode("utf-8", "replace")
|
|
2794
|
+
if returncode != 0:
|
|
2073
2795
|
detail = sanitize_text(
|
|
2074
|
-
|
|
2796
|
+
stderr_text or stdout_text or "git diff failed",
|
|
2075
2797
|
context="command_search_diff",
|
|
2076
2798
|
)[0].strip().splitlines()
|
|
2077
2799
|
message = detail[0] if detail else "git diff failed"
|
|
2078
2800
|
raise PackError(f"could not read diff: {cap_label(message, default='git diff failed', limit=160)}")
|
|
2079
2801
|
return sanitize_text(
|
|
2080
|
-
|
|
2802
|
+
stdout_text,
|
|
2081
2803
|
context="command_search_diff",
|
|
2082
2804
|
)[0]
|
|
2083
2805
|
|
|
@@ -2190,100 +2912,269 @@ def collect_output_candidates(
|
|
|
2190
2912
|
return candidates, omitted
|
|
2191
2913
|
|
|
2192
2914
|
|
|
2193
|
-
def
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
try:
|
|
2205
|
-
while total <= limit:
|
|
2206
|
-
chunk = proc.stdout.read(min(GIT_LS_FILES_READ_CHUNK_BYTES, limit + 1 - total))
|
|
2207
|
-
if not chunk:
|
|
2208
|
-
break
|
|
2209
|
-
chunks.append(chunk)
|
|
2210
|
-
total += len(chunk)
|
|
2211
|
-
if total > limit:
|
|
2212
|
-
capped = True
|
|
2213
|
-
break
|
|
2214
|
-
finally:
|
|
2215
|
-
if capped and proc.poll() is None:
|
|
2216
|
-
try:
|
|
2217
|
-
proc.terminate()
|
|
2218
|
-
except OSError:
|
|
2219
|
-
pass
|
|
2220
|
-
try:
|
|
2221
|
-
proc.stdout.close()
|
|
2222
|
-
except OSError:
|
|
2223
|
-
pass
|
|
2915
|
+
def _read_git_stdout_capped(
|
|
2916
|
+
proc: subprocess.Popen[bytes],
|
|
2917
|
+
limit: int,
|
|
2918
|
+
timeout_seconds: float,
|
|
2919
|
+
) -> tuple[bytes, bool]:
|
|
2920
|
+
if proc.stdout is None:
|
|
2921
|
+
return b"", False
|
|
2922
|
+
chunks: list[bytes] = []
|
|
2923
|
+
total = 0
|
|
2924
|
+
capped = False
|
|
2925
|
+
timed_out = False
|
|
2224
2926
|
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2927
|
+
def reader() -> None:
|
|
2928
|
+
nonlocal total, capped
|
|
2929
|
+
try:
|
|
2930
|
+
while total <= limit:
|
|
2931
|
+
chunk = proc.stdout.read(min(GIT_LS_FILES_READ_CHUNK_BYTES, limit + 1 - total))
|
|
2932
|
+
if not chunk:
|
|
2933
|
+
break
|
|
2934
|
+
chunks.append(chunk)
|
|
2935
|
+
total += len(chunk)
|
|
2936
|
+
if total > limit:
|
|
2937
|
+
capped = True
|
|
2938
|
+
break
|
|
2939
|
+
finally:
|
|
2940
|
+
if capped:
|
|
2941
|
+
_signal_process_group(proc, force=False)
|
|
2230
2942
|
try:
|
|
2231
|
-
proc.
|
|
2943
|
+
proc.stdout.close()
|
|
2232
2944
|
except OSError:
|
|
2233
2945
|
pass
|
|
2946
|
+
|
|
2947
|
+
thread = threading.Thread(target=reader, daemon=True)
|
|
2948
|
+
thread.start()
|
|
2949
|
+
thread.join(timeout_seconds)
|
|
2950
|
+
if thread.is_alive():
|
|
2951
|
+
timed_out = True
|
|
2952
|
+
_signal_process_group(proc, force=False)
|
|
2953
|
+
try:
|
|
2954
|
+
proc.wait(timeout=0.2 if timed_out else 2)
|
|
2955
|
+
except subprocess.TimeoutExpired:
|
|
2956
|
+
_signal_process_group(proc, force=True)
|
|
2234
2957
|
try:
|
|
2235
2958
|
proc.wait(timeout=2)
|
|
2236
2959
|
except subprocess.TimeoutExpired:
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
|
|
2246
|
-
raw_output
|
|
2247
|
-
|
|
2248
|
-
|
|
2960
|
+
pass
|
|
2961
|
+
if capped or timed_out:
|
|
2962
|
+
_signal_process_group(proc, force=True)
|
|
2963
|
+
thread.join(0.5)
|
|
2964
|
+
raw_output = b"".join(chunks)[:limit]
|
|
2965
|
+
complete = (
|
|
2966
|
+
proc.returncode == 0
|
|
2967
|
+
and not capped
|
|
2968
|
+
and not timed_out
|
|
2969
|
+
and (not raw_output or raw_output.endswith(b"\0"))
|
|
2970
|
+
)
|
|
2971
|
+
return raw_output, complete
|
|
2972
|
+
|
|
2249
2973
|
|
|
2250
|
-
|
|
2251
|
-
git_returncode: int | None = None
|
|
2974
|
+
def _git_ls_files_raw(root: Path) -> tuple[bytes, bool, int | None]:
|
|
2252
2975
|
try:
|
|
2253
2976
|
proc = subprocess.Popen(
|
|
2254
|
-
|
|
2977
|
+
guarded_git_command(root, "ls-files", "-z"),
|
|
2978
|
+
stdin=subprocess.DEVNULL,
|
|
2255
2979
|
stdout=subprocess.PIPE,
|
|
2256
2980
|
stderr=subprocess.DEVNULL,
|
|
2257
2981
|
text=False,
|
|
2982
|
+
start_new_session=os.name == "posix",
|
|
2983
|
+
env=guarded_git_environment(),
|
|
2258
2984
|
)
|
|
2259
|
-
raw,
|
|
2260
|
-
|
|
2985
|
+
raw, complete = _read_git_stdout_capped(
|
|
2986
|
+
proc,
|
|
2987
|
+
MAX_GIT_LS_FILES_OUTPUT_BYTES,
|
|
2988
|
+
10,
|
|
2989
|
+
)
|
|
2990
|
+
return raw, complete, proc.returncode
|
|
2261
2991
|
except (OSError, subprocess.TimeoutExpired):
|
|
2262
|
-
|
|
2992
|
+
return b"", False, None
|
|
2993
|
+
|
|
2994
|
+
|
|
2995
|
+
def _iter_nul_fields(raw: bytes):
|
|
2996
|
+
view = memoryview(raw)
|
|
2997
|
+
start = 0
|
|
2998
|
+
while start < len(raw):
|
|
2999
|
+
end = raw.find(b"\0", start)
|
|
3000
|
+
if end < 0:
|
|
3001
|
+
return
|
|
3002
|
+
yield view[start:end]
|
|
3003
|
+
start = end + 1
|
|
3004
|
+
|
|
3005
|
+
|
|
3006
|
+
def git_ls_files(root: Path, diagnostics: dict[str, Any] | None = None) -> list[str]:
|
|
3007
|
+
raw, git_complete, git_returncode = _git_ls_files_raw(root)
|
|
2263
3008
|
if raw:
|
|
2264
3009
|
if not raw.endswith(b"\0"):
|
|
2265
3010
|
raw = raw.rsplit(b"\0", 1)[0] if b"\0" in raw else b""
|
|
2266
|
-
|
|
2267
|
-
|
|
3011
|
+
retained_parts: list[bytes] = []
|
|
3012
|
+
file_cap_reached = False
|
|
3013
|
+
for part_view in _iter_nul_fields(raw):
|
|
3014
|
+
if not part_view:
|
|
3015
|
+
continue
|
|
3016
|
+
if len(retained_parts) >= MAX_QUERY_SCAN_FILES:
|
|
3017
|
+
file_cap_reached = True
|
|
3018
|
+
break
|
|
3019
|
+
retained_parts.append(bytes(part_view))
|
|
3020
|
+
if diagnostics is not None:
|
|
3021
|
+
diagnostics.update({
|
|
3022
|
+
"mode": "git",
|
|
3023
|
+
"truncated": not git_complete or file_cap_reached,
|
|
3024
|
+
"truncation_reason": (
|
|
3025
|
+
"git_output_cap_or_timeout"
|
|
3026
|
+
if not git_complete
|
|
3027
|
+
else "file_cap" if file_cap_reached else None
|
|
3028
|
+
),
|
|
3029
|
+
})
|
|
3030
|
+
return [part.decode("utf-8", "replace") for part in retained_parts]
|
|
3031
|
+
if git_returncode == 0:
|
|
3032
|
+
if diagnostics is not None:
|
|
3033
|
+
diagnostics.update({"mode": "git", "truncated": False, "truncation_reason": None})
|
|
3034
|
+
return []
|
|
3035
|
+
if git_returncode is not None and git_returncode < 0:
|
|
3036
|
+
if diagnostics is not None:
|
|
3037
|
+
diagnostics.update({
|
|
3038
|
+
"mode": "git",
|
|
3039
|
+
"truncated": True,
|
|
3040
|
+
"truncation_reason": "git_output_cap_or_timeout",
|
|
3041
|
+
})
|
|
2268
3042
|
return []
|
|
2269
3043
|
out: list[str] = []
|
|
2270
3044
|
skip_dirs = {".git", ".omx", ".context-guard", "node_modules", "dist", "build", "__pycache__"}
|
|
2271
|
-
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
|
|
3045
|
+
started = time.monotonic()
|
|
3046
|
+
visited_dirs = 0
|
|
3047
|
+
visited_entries = 0
|
|
3048
|
+
truncation_reason: str | None = None
|
|
3049
|
+
pending: deque[tuple[Path, int]] = deque([(root, 0)])
|
|
3050
|
+
while pending:
|
|
3051
|
+
if time.monotonic() - started > MAX_QUERY_WALK_SECONDS:
|
|
3052
|
+
truncation_reason = "time_cap"
|
|
3053
|
+
break
|
|
3054
|
+
if visited_dirs >= MAX_QUERY_WALK_DIRS:
|
|
3055
|
+
truncation_reason = "directory_cap"
|
|
3056
|
+
break
|
|
3057
|
+
current_path, depth = pending.popleft()
|
|
3058
|
+
visited_dirs += 1
|
|
3059
|
+
try:
|
|
3060
|
+
iterator = os.scandir(current_path)
|
|
3061
|
+
except OSError:
|
|
3062
|
+
truncation_reason = "unsafe_path"
|
|
3063
|
+
break
|
|
3064
|
+
child_dirs: list[Path] = []
|
|
3065
|
+
try:
|
|
3066
|
+
with iterator:
|
|
3067
|
+
for entry in iterator:
|
|
3068
|
+
if time.monotonic() - started > MAX_QUERY_WALK_SECONDS:
|
|
3069
|
+
truncation_reason = "time_cap"
|
|
3070
|
+
break
|
|
3071
|
+
if visited_entries >= MAX_QUERY_WALK_ENTRIES:
|
|
3072
|
+
truncation_reason = "entry_cap"
|
|
3073
|
+
break
|
|
3074
|
+
visited_entries += 1
|
|
3075
|
+
name = entry.name
|
|
3076
|
+
try:
|
|
3077
|
+
is_dir = entry.is_dir(follow_symlinks=False)
|
|
3078
|
+
is_file = entry.is_file(follow_symlinks=False)
|
|
3079
|
+
except OSError:
|
|
3080
|
+
continue
|
|
3081
|
+
if is_dir:
|
|
3082
|
+
if name in skip_dirs or name.startswith(".pytest"):
|
|
3083
|
+
continue
|
|
3084
|
+
if depth >= MAX_QUERY_WALK_DEPTH:
|
|
3085
|
+
truncation_reason = truncation_reason or "depth_cap"
|
|
3086
|
+
continue
|
|
3087
|
+
child_dirs.append(current_path / name)
|
|
3088
|
+
elif is_file:
|
|
3089
|
+
try:
|
|
3090
|
+
rel = (current_path / name).relative_to(root).as_posix()
|
|
3091
|
+
except ValueError:
|
|
3092
|
+
truncation_reason = "unsafe_path"
|
|
3093
|
+
break
|
|
3094
|
+
out.append(rel)
|
|
3095
|
+
if len(out) >= MAX_QUERY_SCAN_FILES:
|
|
3096
|
+
truncation_reason = "file_cap"
|
|
3097
|
+
break
|
|
3098
|
+
except OSError:
|
|
3099
|
+
truncation_reason = "unsafe_path"
|
|
3100
|
+
break
|
|
3101
|
+
if truncation_reason in {"time_cap", "entry_cap", "file_cap", "unsafe_path"}:
|
|
3102
|
+
break
|
|
3103
|
+
for child in reversed(sorted(child_dirs, key=lambda path: path.name)):
|
|
3104
|
+
pending.appendleft((child, depth + 1))
|
|
3105
|
+
if diagnostics is not None:
|
|
3106
|
+
diagnostics.update({
|
|
3107
|
+
"mode": "walk",
|
|
3108
|
+
"truncated": truncation_reason is not None,
|
|
3109
|
+
"truncation_reason": truncation_reason,
|
|
3110
|
+
"visited_dirs": min(visited_dirs, MAX_QUERY_WALK_DIRS),
|
|
3111
|
+
"visited_entries": min(visited_entries, MAX_QUERY_WALK_ENTRIES),
|
|
3112
|
+
})
|
|
2279
3113
|
return out
|
|
2280
3114
|
|
|
2281
3115
|
|
|
2282
|
-
def
|
|
3116
|
+
def reject_configured_git_filters(root: Path) -> None:
|
|
3117
|
+
raw_paths, complete, returncode = _git_ls_files_raw(root)
|
|
3118
|
+
if returncode != 0 or not complete:
|
|
3119
|
+
raise PackError("could not verify git filters: tracked path scan failed or truncated")
|
|
3120
|
+
if not raw_paths:
|
|
3121
|
+
return
|
|
3122
|
+
if len(raw_paths) > MAX_GIT_ATTR_INPUT_BYTES:
|
|
3123
|
+
raise PackError("could not verify git filters: tracked path input exceeds cap")
|
|
3124
|
+
|
|
3125
|
+
try:
|
|
3126
|
+
command = guarded_git_command(
|
|
3127
|
+
root,
|
|
3128
|
+
"check-attr",
|
|
3129
|
+
"-z",
|
|
3130
|
+
"--stdin",
|
|
3131
|
+
"filter",
|
|
3132
|
+
)
|
|
3133
|
+
returncode, stdout, _stderr, stdout_capped, failed_or_timed_out = _run_process_capped(
|
|
3134
|
+
command,
|
|
3135
|
+
stdout_cap=MAX_GIT_ATTR_OUTPUT_BYTES,
|
|
3136
|
+
stderr_cap=MAX_GIT_DIFF_STDERR_BYTES,
|
|
3137
|
+
timeout_seconds=GIT_ATTR_TIMEOUT_SECONDS,
|
|
3138
|
+
environment=guarded_git_environment(),
|
|
3139
|
+
stdin_data=raw_paths,
|
|
3140
|
+
stdin_cap_bytes=MAX_GIT_ATTR_INPUT_BYTES,
|
|
3141
|
+
)
|
|
3142
|
+
except (OSError, UnicodeError, subprocess.TimeoutExpired) as exc:
|
|
3143
|
+
raise PackError(f"could not verify git filters: {exc.__class__.__name__}") from exc
|
|
3144
|
+
if stdout_capped or failed_or_timed_out or returncode != 0:
|
|
3145
|
+
raise PackError("could not verify git filters: check-attr failed or exceeded cap")
|
|
3146
|
+
if not stdout.endswith(b"\0"):
|
|
3147
|
+
raise PackError("could not verify git filters: malformed check-attr output")
|
|
3148
|
+
output_fields = iter(_iter_nul_fields(stdout))
|
|
3149
|
+
for expected_path in _iter_nul_fields(raw_paths):
|
|
3150
|
+
try:
|
|
3151
|
+
path = next(output_fields)
|
|
3152
|
+
attribute = next(output_fields)
|
|
3153
|
+
value = next(output_fields)
|
|
3154
|
+
except StopIteration as exc:
|
|
3155
|
+
raise PackError("could not verify git filters: incomplete check-attr output") from exc
|
|
3156
|
+
if path != expected_path or bytes(attribute) != b"filter":
|
|
3157
|
+
raise PackError("could not verify git filters: mismatched check-attr output")
|
|
3158
|
+
if bytes(value) not in {b"unspecified", b"unset"}:
|
|
3159
|
+
raise PackError("git diff blocked: configured filter attribute")
|
|
3160
|
+
try:
|
|
3161
|
+
next(output_fields)
|
|
3162
|
+
except StopIteration:
|
|
3163
|
+
return
|
|
3164
|
+
raise PackError("could not verify git filters: excess check-attr output")
|
|
3165
|
+
|
|
3166
|
+
|
|
3167
|
+
def collect_query_candidates(
|
|
3168
|
+
root: Path,
|
|
3169
|
+
query_terms: set[str],
|
|
3170
|
+
context_lines: int,
|
|
3171
|
+
*,
|
|
3172
|
+
diagnostics: dict[str, Any] | None = None,
|
|
3173
|
+
) -> list[SuggestCandidate]:
|
|
2283
3174
|
if not query_terms:
|
|
2284
3175
|
return []
|
|
2285
3176
|
candidates: list[SuggestCandidate] = []
|
|
2286
|
-
for rel_path in git_ls_files(root):
|
|
3177
|
+
for rel_path in git_ls_files(root, diagnostics):
|
|
2287
3178
|
rel, reason = lexical_rel(rel_path)
|
|
2288
3179
|
if rel is None or reason:
|
|
2289
3180
|
continue
|
|
@@ -2373,16 +3264,28 @@ def suggested_source_payload(source: ResolvedSource, candidate: SuggestCandidate
|
|
|
2373
3264
|
return payload
|
|
2374
3265
|
|
|
2375
3266
|
|
|
2376
|
-
def normalize_suggest_source(
|
|
3267
|
+
def normalize_suggest_source(
|
|
3268
|
+
root: Path,
|
|
3269
|
+
candidate: SuggestCandidate,
|
|
3270
|
+
*,
|
|
3271
|
+
source_cache: _SourceSnapshotCache | None = None,
|
|
3272
|
+
input_budget: _SourceInputBudget | None = None,
|
|
3273
|
+
) -> tuple[ResolvedSource | None, dict[str, Any] | None]:
|
|
3274
|
+
effective_lines = candidate.lines or LineRange(1, SUGGEST_WHOLE_FILE_MAX_LINES)
|
|
2377
3275
|
spec = SourceSpec(
|
|
2378
3276
|
path=candidate.path,
|
|
2379
3277
|
priority=candidate.score,
|
|
2380
|
-
lines=
|
|
3278
|
+
lines=effective_lines,
|
|
2381
3279
|
label=candidate.label,
|
|
2382
3280
|
input_index=candidate.input_index,
|
|
2383
3281
|
origin="suggest",
|
|
2384
3282
|
)
|
|
2385
|
-
source, omitted_item = resolve_source(
|
|
3283
|
+
source, omitted_item = resolve_source(
|
|
3284
|
+
root,
|
|
3285
|
+
spec,
|
|
3286
|
+
source_cache=source_cache,
|
|
3287
|
+
input_budget=input_budget,
|
|
3288
|
+
)
|
|
2386
3289
|
if omitted_item is not None:
|
|
2387
3290
|
omitted_item["reason"] = omitted_item.get("reason") or candidate.reason
|
|
2388
3291
|
omitted_item["suggest_reason"] = candidate.reason
|
|
@@ -2390,20 +3293,6 @@ def normalize_suggest_source(root: Path, candidate: SuggestCandidate) -> tuple[R
|
|
|
2390
3293
|
assert source is not None
|
|
2391
3294
|
if source.redacted_path:
|
|
2392
3295
|
return None, omission(spec, "redacted_path", path=source.display_path, redacted_path=True)
|
|
2393
|
-
if spec.lines is None and source.total_lines > SUGGEST_WHOLE_FILE_MAX_LINES:
|
|
2394
|
-
capped = SourceSpec(
|
|
2395
|
-
path=candidate.path,
|
|
2396
|
-
priority=candidate.score,
|
|
2397
|
-
lines=LineRange(1, min(SUGGEST_WHOLE_FILE_MAX_LINES, source.total_lines)),
|
|
2398
|
-
label=candidate.label,
|
|
2399
|
-
input_index=candidate.input_index,
|
|
2400
|
-
origin="suggest",
|
|
2401
|
-
)
|
|
2402
|
-
source, omitted_item = resolve_source(root, capped)
|
|
2403
|
-
if omitted_item is not None:
|
|
2404
|
-
omitted_item["suggest_reason"] = candidate.reason
|
|
2405
|
-
return None, omitted_item
|
|
2406
|
-
assert source is not None
|
|
2407
3296
|
return source, None
|
|
2408
3297
|
|
|
2409
3298
|
|
|
@@ -2943,7 +3832,15 @@ def build_adaptive_k_advisory(
|
|
|
2943
3832
|
}
|
|
2944
3833
|
|
|
2945
3834
|
|
|
2946
|
-
def suggest_pack(
|
|
3835
|
+
def suggest_pack(
|
|
3836
|
+
root: Path,
|
|
3837
|
+
args: argparse.Namespace,
|
|
3838
|
+
*,
|
|
3839
|
+
root_arg: str,
|
|
3840
|
+
_source_cache: _SourceSnapshotCache | None = None,
|
|
3841
|
+
_input_budget: _SourceInputBudget | None = None,
|
|
3842
|
+
) -> tuple[dict[str, Any], int]:
|
|
3843
|
+
input_budget = _input_budget if _input_budget is not None else _SourceInputBudget()
|
|
2947
3844
|
query_text, _query_redactions = sanitize_text(args.query or "")
|
|
2948
3845
|
query = " ".join(query_text.split())
|
|
2949
3846
|
query_terms = suggest_tokens(query)
|
|
@@ -2973,7 +3870,23 @@ def suggest_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tupl
|
|
|
2973
3870
|
candidates.extend(test_candidates)
|
|
2974
3871
|
omitted.extend(output_omitted)
|
|
2975
3872
|
omitted.extend(test_omitted)
|
|
2976
|
-
|
|
3873
|
+
query_scan: dict[str, Any] = {}
|
|
3874
|
+
candidates.extend(
|
|
3875
|
+
collect_query_candidates(
|
|
3876
|
+
root,
|
|
3877
|
+
query_terms,
|
|
3878
|
+
context_lines,
|
|
3879
|
+
diagnostics=query_scan,
|
|
3880
|
+
)
|
|
3881
|
+
)
|
|
3882
|
+
if query_scan.get("truncated"):
|
|
3883
|
+
omitted.append({
|
|
3884
|
+
"path": "repository",
|
|
3885
|
+
"status": "omitted",
|
|
3886
|
+
"reason": "query_scan_truncated",
|
|
3887
|
+
"scan_truncation_reason": query_scan.get("truncation_reason"),
|
|
3888
|
+
"priority": 0,
|
|
3889
|
+
})
|
|
2977
3890
|
|
|
2978
3891
|
candidates.sort(key=lambda item: (-item.score, item.input_index, item.path, item.lines.identity() if item.lines else "0:0"))
|
|
2979
3892
|
seen: set[tuple[str, str]] = set()
|
|
@@ -3000,7 +3913,12 @@ def suggest_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tupl
|
|
|
3000
3913
|
continue
|
|
3001
3914
|
if rel is not None:
|
|
3002
3915
|
seen.add(identity)
|
|
3003
|
-
source, omitted_item = normalize_suggest_source(
|
|
3916
|
+
source, omitted_item = normalize_suggest_source(
|
|
3917
|
+
root,
|
|
3918
|
+
candidate,
|
|
3919
|
+
source_cache=_source_cache,
|
|
3920
|
+
input_budget=input_budget,
|
|
3921
|
+
)
|
|
3004
3922
|
if omitted_item is not None:
|
|
3005
3923
|
omitted_item["priority"] = candidate.score
|
|
3006
3924
|
omitted_item["suggest_reason"] = candidate.reason
|
|
@@ -3040,7 +3958,12 @@ def suggest_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tupl
|
|
|
3040
3958
|
input_index=candidate.input_index,
|
|
3041
3959
|
origin="suggest",
|
|
3042
3960
|
)
|
|
3043
|
-
source, omitted_item = resolve_source(
|
|
3961
|
+
source, omitted_item = resolve_source(
|
|
3962
|
+
root,
|
|
3963
|
+
partial_spec,
|
|
3964
|
+
source_cache=_source_cache,
|
|
3965
|
+
input_budget=input_budget,
|
|
3966
|
+
)
|
|
3044
3967
|
if omitted_item is not None:
|
|
3045
3968
|
omitted_item["priority"] = candidate.score
|
|
3046
3969
|
omitted_item["suggest_reason"] = candidate.reason
|
|
@@ -3099,6 +4022,17 @@ def suggest_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tupl
|
|
|
3099
4022
|
"Byte and token values are pack-size proxies, not billing claims.",
|
|
3100
4023
|
],
|
|
3101
4024
|
}
|
|
4025
|
+
if query_scan:
|
|
4026
|
+
payload["query_scan"] = {
|
|
4027
|
+
**query_scan,
|
|
4028
|
+
"limits": {
|
|
4029
|
+
"files": MAX_QUERY_SCAN_FILES,
|
|
4030
|
+
"directories": MAX_QUERY_WALK_DIRS,
|
|
4031
|
+
"entries": MAX_QUERY_WALK_ENTRIES,
|
|
4032
|
+
"depth": MAX_QUERY_WALK_DEPTH,
|
|
4033
|
+
"seconds": MAX_QUERY_WALK_SECONDS,
|
|
4034
|
+
},
|
|
4035
|
+
}
|
|
3102
4036
|
if build_hint_omitted_reason:
|
|
3103
4037
|
payload["build_hint_omitted_reason"] = build_hint_omitted_reason
|
|
3104
4038
|
if getattr(args, "adaptive_k", False):
|
|
@@ -3124,6 +4058,57 @@ def line_range_identity(value: object) -> str:
|
|
|
3124
4058
|
return str(value)
|
|
3125
4059
|
|
|
3126
4060
|
|
|
4061
|
+
def apply_adaptive_k_manifest(
|
|
4062
|
+
manifest: dict[str, Any],
|
|
4063
|
+
advisory: dict[str, Any],
|
|
4064
|
+
) -> tuple[dict[str, Any], dict[str, Any]]:
|
|
4065
|
+
raw_sources = manifest.get("sources", [])
|
|
4066
|
+
sources = copy.deepcopy(raw_sources) if isinstance(raw_sources, list) else []
|
|
4067
|
+
gates = advisory.get("regression_gates", {})
|
|
4068
|
+
gates_passed = isinstance(gates, dict) and gates.get("status") == "pass"
|
|
4069
|
+
recommended_k = max(0, int(advisory.get("recommended_k", 0) or 0))
|
|
4070
|
+
protected_prefixes = ("file:", "output:", "test-output:", "diff:")
|
|
4071
|
+
protected_indexes = {
|
|
4072
|
+
index
|
|
4073
|
+
for index, item in enumerate(sources)
|
|
4074
|
+
if isinstance(item, dict)
|
|
4075
|
+
and str(item.get("label", "")).startswith(protected_prefixes)
|
|
4076
|
+
}
|
|
4077
|
+
retained: list[dict[str, Any]] = []
|
|
4078
|
+
if gates_passed:
|
|
4079
|
+
target_count = max(recommended_k, len(protected_indexes))
|
|
4080
|
+
for index, item in enumerate(sources):
|
|
4081
|
+
if not isinstance(item, dict):
|
|
4082
|
+
continue
|
|
4083
|
+
if index in protected_indexes or len(retained) < target_count:
|
|
4084
|
+
retained.append(item)
|
|
4085
|
+
else:
|
|
4086
|
+
retained = [item for item in sources if isinstance(item, dict)]
|
|
4087
|
+
omitted_count = len(sources) - len(retained)
|
|
4088
|
+
status = "applied" if gates_passed and omitted_count else "no_change"
|
|
4089
|
+
if not gates_passed:
|
|
4090
|
+
status = "gate_failed"
|
|
4091
|
+
applied_manifest = {"version": 1, "sources": retained}
|
|
4092
|
+
return applied_manifest, {
|
|
4093
|
+
"schema_version": ADAPTIVE_K_APPLICATION_SCHEMA_VERSION,
|
|
4094
|
+
"mode": "explicit_opt_in",
|
|
4095
|
+
"status": status,
|
|
4096
|
+
"recommended_k": recommended_k,
|
|
4097
|
+
"input_source_count": len(sources),
|
|
4098
|
+
"applied_source_count": len(retained),
|
|
4099
|
+
"omitted_source_count": omitted_count,
|
|
4100
|
+
"regression_gates_passed": gates_passed,
|
|
4101
|
+
"explicit_sources_retained": all(
|
|
4102
|
+
sources[index] in retained for index in protected_indexes
|
|
4103
|
+
),
|
|
4104
|
+
"claim_boundary": {
|
|
4105
|
+
"deterministic_local_only": True,
|
|
4106
|
+
"exact_source_fallback_retained": True,
|
|
4107
|
+
"provider_token_or_cost_savings_claim_allowed": False,
|
|
4108
|
+
},
|
|
4109
|
+
}
|
|
4110
|
+
|
|
4111
|
+
|
|
3127
4112
|
def copy_explain_fields(item: dict[str, Any], fields: tuple[str, ...]) -> dict[str, Any]:
|
|
3128
4113
|
out: dict[str, Any] = {}
|
|
3129
4114
|
for field in fields:
|
|
@@ -3193,7 +4178,12 @@ def is_repo_map_text_path(path: str) -> bool:
|
|
|
3193
4178
|
return Path(path).suffix.lower() in REPO_MAP_TEXT_EXTENSIONS
|
|
3194
4179
|
|
|
3195
4180
|
|
|
3196
|
-
def read_repo_map_text(
|
|
4181
|
+
def read_repo_map_text(
|
|
4182
|
+
root: Path,
|
|
4183
|
+
rel_path: str,
|
|
4184
|
+
*,
|
|
4185
|
+
source_identities_out: dict[str, tuple[int, int, int, int, int, int, int, int]] | None = None,
|
|
4186
|
+
) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
|
|
3197
4187
|
rel, reason = lexical_rel(rel_path)
|
|
3198
4188
|
if rel is None:
|
|
3199
4189
|
return None, {"path": repo_map_safe_raw_path_label(rel_path), "reason": reason}
|
|
@@ -3205,14 +4195,24 @@ def read_repo_map_text(root: Path, rel_path: str) -> tuple[dict[str, Any] | None
|
|
|
3205
4195
|
return None, {"path": display, "reason": open_reason, "retrieval_omitted_reason": "redacted_path" if redacted_path else None}
|
|
3206
4196
|
try:
|
|
3207
4197
|
with handle:
|
|
4198
|
+
before_identity = _open_source_identity(handle)
|
|
3208
4199
|
text = handle.read(MAX_REPO_MAP_BYTES_PER_FILE + 1)
|
|
4200
|
+
after_identity = _open_source_identity(handle)
|
|
3209
4201
|
except (OSError, UnicodeError):
|
|
3210
4202
|
return None, {"path": display, "reason": "unsafe_path", "retrieval_omitted_reason": "redacted_path" if redacted_path else None}
|
|
4203
|
+
if before_identity is None or after_identity != before_identity:
|
|
4204
|
+
return None, {
|
|
4205
|
+
"path": display,
|
|
4206
|
+
"reason": "source_changed_during_repo_map",
|
|
4207
|
+
"retrieval_omitted_reason": "redacted_path" if redacted_path else None,
|
|
4208
|
+
}
|
|
3211
4209
|
capped = byte_len(text) > MAX_REPO_MAP_BYTES_PER_FILE
|
|
3212
4210
|
if capped:
|
|
3213
4211
|
text = text.encode("utf-8", errors="replace")[:MAX_REPO_MAP_BYTES_PER_FILE].decode("utf-8", errors="ignore")
|
|
3214
4212
|
risk_counts = secret_risk_counts(text)
|
|
3215
4213
|
sanitized_text, redacted_lines = sanitize_text(text)
|
|
4214
|
+
if source_identities_out is not None:
|
|
4215
|
+
source_identities_out[rel.as_posix()] = before_identity
|
|
3216
4216
|
return {
|
|
3217
4217
|
"path": display,
|
|
3218
4218
|
"raw_path": rel.as_posix(),
|
|
@@ -3251,7 +4251,13 @@ def repo_map_scan_paths(paths: list[str], *, seed_paths: set[str], query_terms:
|
|
|
3251
4251
|
return [path for _index, path in ranked[:MAX_REPO_MAP_SCAN_FILES]]
|
|
3252
4252
|
|
|
3253
4253
|
|
|
3254
|
-
def repo_map_records(
|
|
4254
|
+
def repo_map_records(
|
|
4255
|
+
root: Path,
|
|
4256
|
+
*,
|
|
4257
|
+
seed_paths: set[str],
|
|
4258
|
+
query_terms: set[str],
|
|
4259
|
+
source_identities_out: dict[str, tuple[int, int, int, int, int, int, int, int]] | None = None,
|
|
4260
|
+
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], dict[str, Any]]:
|
|
3255
4261
|
paths = git_ls_files(root)
|
|
3256
4262
|
candidate_paths = paths[:MAX_REPO_MAP_FILES]
|
|
3257
4263
|
path_cap_reached = len(paths) > MAX_REPO_MAP_FILES
|
|
@@ -3260,7 +4266,14 @@ def repo_map_records(root: Path, *, seed_paths: set[str], query_terms: set[str])
|
|
|
3260
4266
|
records: list[dict[str, Any]] = []
|
|
3261
4267
|
omitted: list[dict[str, Any]] = []
|
|
3262
4268
|
for rel_path in scan_paths:
|
|
3263
|
-
|
|
4269
|
+
if source_identities_out is None:
|
|
4270
|
+
record, omission_item = read_repo_map_text(root, rel_path)
|
|
4271
|
+
else:
|
|
4272
|
+
record, omission_item = read_repo_map_text(
|
|
4273
|
+
root,
|
|
4274
|
+
rel_path,
|
|
4275
|
+
source_identities_out=source_identities_out,
|
|
4276
|
+
)
|
|
3264
4277
|
if record is not None:
|
|
3265
4278
|
records.append(record)
|
|
3266
4279
|
elif omission_item is not None and omission_item.get("reason") != "unsupported_file_type":
|
|
@@ -3522,9 +4535,18 @@ def build_graph_rank(
|
|
|
3522
4535
|
query_terms: set[str],
|
|
3523
4536
|
seed_paths: set[str],
|
|
3524
4537
|
secret_scan: dict[str, Any],
|
|
4538
|
+
complete_secret_paths: set[str] | None = None,
|
|
3525
4539
|
) -> list[dict[str, Any]]:
|
|
3526
4540
|
signature_paths = {str(item.get("path", "")) for item in signatures}
|
|
3527
|
-
secret_paths =
|
|
4541
|
+
secret_paths = (
|
|
4542
|
+
complete_secret_paths
|
|
4543
|
+
if complete_secret_paths is not None
|
|
4544
|
+
else {
|
|
4545
|
+
str(item.get("path", ""))
|
|
4546
|
+
for item in secret_scan.get("files_with_risks", [])
|
|
4547
|
+
if isinstance(item, dict)
|
|
4548
|
+
}
|
|
4549
|
+
)
|
|
3528
4550
|
degree: dict[str, int] = {}
|
|
3529
4551
|
for edge in edges:
|
|
3530
4552
|
degree[edge["from"]] = degree.get(edge["from"], 0) + 1
|
|
@@ -3621,13 +4643,27 @@ def build_repo_map_payload(
|
|
|
3621
4643
|
build_payload: dict[str, Any],
|
|
3622
4644
|
*,
|
|
3623
4645
|
root_arg: str,
|
|
4646
|
+
complete_secret_paths_out: set[str] | None = None,
|
|
4647
|
+
source_identities_out: dict[str, tuple[int, int, int, int, int, int, int, int]] | None = None,
|
|
3624
4648
|
) -> dict[str, Any]:
|
|
3625
4649
|
query_terms = suggest_tokens(str(suggest_payload.get("query", "")))
|
|
3626
4650
|
seed_paths = repo_map_seed_paths(args, suggest_payload, build_payload)
|
|
3627
|
-
records, omitted, caps = repo_map_records(
|
|
4651
|
+
records, omitted, caps = repo_map_records(
|
|
4652
|
+
root,
|
|
4653
|
+
seed_paths=seed_paths,
|
|
4654
|
+
query_terms=query_terms,
|
|
4655
|
+
source_identities_out=source_identities_out,
|
|
4656
|
+
)
|
|
3628
4657
|
record_by_path = {str(record["path"]): record for record in records}
|
|
3629
4658
|
signatures = extract_signatures(records)
|
|
3630
4659
|
secret_scan = build_secret_scan(records)
|
|
4660
|
+
complete_secret_paths = {
|
|
4661
|
+
str(record.get("path", ""))
|
|
4662
|
+
for record in records
|
|
4663
|
+
if record.get("secret_risk_counts")
|
|
4664
|
+
}
|
|
4665
|
+
if complete_secret_paths_out is not None:
|
|
4666
|
+
complete_secret_paths_out.update(complete_secret_paths)
|
|
3631
4667
|
edges = collect_import_edges(records)
|
|
3632
4668
|
graph_rank = build_graph_rank(
|
|
3633
4669
|
records,
|
|
@@ -3636,6 +4672,7 @@ def build_repo_map_payload(
|
|
|
3636
4672
|
query_terms=query_terms,
|
|
3637
4673
|
seed_paths=seed_paths,
|
|
3638
4674
|
secret_scan=secret_scan,
|
|
4675
|
+
complete_secret_paths=complete_secret_paths,
|
|
3639
4676
|
)
|
|
3640
4677
|
retrieval = repo_map_retrieval(record_by_path, signatures, graph_rank, root_arg=root_arg)
|
|
3641
4678
|
tree = build_token_tree(records)
|
|
@@ -3685,7 +4722,417 @@ def line_identity_from_dict(value: object) -> str:
|
|
|
3685
4722
|
return f"{value.get('start')}:{value.get('end')}"
|
|
3686
4723
|
|
|
3687
4724
|
|
|
3688
|
-
def
|
|
4725
|
+
def frozen_source_content_sha256(
|
|
4726
|
+
path: str,
|
|
4727
|
+
lines: object,
|
|
4728
|
+
source_cache: _SourceSnapshotCache,
|
|
4729
|
+
) -> str | None:
|
|
4730
|
+
rel, _reason = lexical_rel(path)
|
|
4731
|
+
if rel is None:
|
|
4732
|
+
return None
|
|
4733
|
+
snapshot = source_cache.entries.get(
|
|
4734
|
+
(rel.as_posix(), line_identity_from_dict(lines), "source_code")
|
|
4735
|
+
)
|
|
4736
|
+
if snapshot is None:
|
|
4737
|
+
return None
|
|
4738
|
+
return sha256_text("".join(snapshot.selected_lines))
|
|
4739
|
+
|
|
4740
|
+
|
|
4741
|
+
def frozen_source_identity(
|
|
4742
|
+
path: str,
|
|
4743
|
+
lines: object,
|
|
4744
|
+
identities: dict[str, tuple[int, int, int, int, int, int, int, int]],
|
|
4745
|
+
source_cache: _SourceSnapshotCache,
|
|
4746
|
+
) -> str:
|
|
4747
|
+
identity = identities.get(path)
|
|
4748
|
+
content_sha256 = frozen_source_content_sha256(path, lines, source_cache)
|
|
4749
|
+
material = json.dumps(
|
|
4750
|
+
{
|
|
4751
|
+
"content_sha256": content_sha256,
|
|
4752
|
+
"lines": line_identity_from_dict(lines),
|
|
4753
|
+
"path": path,
|
|
4754
|
+
"stat": identity,
|
|
4755
|
+
},
|
|
4756
|
+
ensure_ascii=False,
|
|
4757
|
+
sort_keys=True,
|
|
4758
|
+
separators=(",", ":"),
|
|
4759
|
+
)
|
|
4760
|
+
return f"sha256:{sha256_text(material)}"
|
|
4761
|
+
|
|
4762
|
+
|
|
4763
|
+
def exact_source_fallback(
|
|
4764
|
+
root_arg: str,
|
|
4765
|
+
path: str,
|
|
4766
|
+
lines: object,
|
|
4767
|
+
*,
|
|
4768
|
+
expected_content_sha256: str | None,
|
|
4769
|
+
unavailable_reason: str | None = None,
|
|
4770
|
+
) -> dict[str, Any]:
|
|
4771
|
+
if unavailable_reason is not None:
|
|
4772
|
+
return {"kind": "unavailable", "reason": unavailable_reason}
|
|
4773
|
+
rel, _reason = lexical_rel(path)
|
|
4774
|
+
safe_root = safe_root_arg_for_retrieval(root_arg)
|
|
4775
|
+
if (
|
|
4776
|
+
rel is None
|
|
4777
|
+
or safe_root is None
|
|
4778
|
+
or repo_map_path_has_sensitive_evidence(path)
|
|
4779
|
+
or not isinstance(lines, dict)
|
|
4780
|
+
or not isinstance(lines.get("start"), int)
|
|
4781
|
+
or isinstance(lines.get("start"), bool)
|
|
4782
|
+
or not isinstance(lines.get("end"), int)
|
|
4783
|
+
or isinstance(lines.get("end"), bool)
|
|
4784
|
+
or lines["start"] < 1
|
|
4785
|
+
or lines["end"] < lines["start"]
|
|
4786
|
+
or expected_content_sha256 is None
|
|
4787
|
+
):
|
|
4788
|
+
return {"kind": "unavailable", "reason": "exact_snapshot_unavailable"}
|
|
4789
|
+
line_identity = line_identity_from_dict(lines)
|
|
4790
|
+
args = [
|
|
4791
|
+
"context-guard-pack", "slice", "--root", safe_root,
|
|
4792
|
+
"--path", rel.as_posix(), "--lines", line_identity, "--json",
|
|
4793
|
+
]
|
|
4794
|
+
return {
|
|
4795
|
+
"kind": "exact_source_slice",
|
|
4796
|
+
"command": " ".join(shlex.quote(part) for part in args),
|
|
4797
|
+
"expected_content_sha256": expected_content_sha256,
|
|
4798
|
+
"path": rel.as_posix(),
|
|
4799
|
+
"lines": copy.deepcopy(lines),
|
|
4800
|
+
}
|
|
4801
|
+
|
|
4802
|
+
|
|
4803
|
+
def self_financing_candidate_receipt(
|
|
4804
|
+
*, phase: str, source: dict[str, Any], reason: str, status: str,
|
|
4805
|
+
secret_decision: str, identities: dict[str, tuple[int, int, int, int, int, int, int, int]],
|
|
4806
|
+
root_arg: str, source_cache: _SourceSnapshotCache, byte_delta: int = 0,
|
|
4807
|
+
removed_sources: list[dict[str, Any]] | None = None,
|
|
4808
|
+
) -> dict[str, Any]:
|
|
4809
|
+
path = str(source.get("path", ""))
|
|
4810
|
+
content_sha256 = frozen_source_content_sha256(
|
|
4811
|
+
path, source.get("lines"), source_cache
|
|
4812
|
+
)
|
|
4813
|
+
return {
|
|
4814
|
+
"phase": phase,
|
|
4815
|
+
"status": status,
|
|
4816
|
+
"path": path,
|
|
4817
|
+
"lines": copy.deepcopy(source.get("lines")),
|
|
4818
|
+
"reason": reason,
|
|
4819
|
+
"hop_count": 1 if phase == "graph" else 0,
|
|
4820
|
+
"frozen_identity": frozen_source_identity(
|
|
4821
|
+
path, source.get("lines"), identities, source_cache
|
|
4822
|
+
),
|
|
4823
|
+
"byte_delta": byte_delta,
|
|
4824
|
+
"secret_risk": {"decision": secret_decision, "signal": "bounded_local_pattern_scan"},
|
|
4825
|
+
"exact_fallback": exact_source_fallback(
|
|
4826
|
+
root_arg,
|
|
4827
|
+
path,
|
|
4828
|
+
source.get("lines"),
|
|
4829
|
+
expected_content_sha256=content_sha256,
|
|
4830
|
+
unavailable_reason="secret_risk" if secret_decision == "reject" else None,
|
|
4831
|
+
),
|
|
4832
|
+
"removed_sources": copy.deepcopy(removed_sources or []),
|
|
4833
|
+
}
|
|
4834
|
+
|
|
4835
|
+
|
|
4836
|
+
def selection_plan_source(
|
|
4837
|
+
item: dict[str, Any], identities: dict[str, tuple[int, int, int, int, int, int, int, int]],
|
|
4838
|
+
source_cache: _SourceSnapshotCache, root_arg: str,
|
|
4839
|
+
) -> dict[str, Any]:
|
|
4840
|
+
path = str(item.get("path", ""))
|
|
4841
|
+
lines = copy.deepcopy(item.get("requested_lines", item.get("lines")))
|
|
4842
|
+
content_sha256 = frozen_source_content_sha256(path, lines, source_cache)
|
|
4843
|
+
fallback = exact_source_fallback(
|
|
4844
|
+
root_arg, path, lines, expected_content_sha256=content_sha256
|
|
4845
|
+
)
|
|
4846
|
+
if fallback.get("kind") != "exact_source_slice":
|
|
4847
|
+
raise PackError("selection plan missing exact recovery")
|
|
4848
|
+
return {
|
|
4849
|
+
"path": path,
|
|
4850
|
+
"lines": lines,
|
|
4851
|
+
"identity": frozen_source_identity(path, lines, identities, source_cache),
|
|
4852
|
+
"content_sha256": content_sha256,
|
|
4853
|
+
"exact_fallback": fallback,
|
|
4854
|
+
}
|
|
4855
|
+
|
|
4856
|
+
|
|
4857
|
+
def build_selection_plan(
|
|
4858
|
+
args: argparse.Namespace, ordinary_build: dict[str, Any], selected_build: dict[str, Any],
|
|
4859
|
+
receipt: dict[str, Any], repo_map: dict[str, Any], suggest_payload: dict[str, Any],
|
|
4860
|
+
identities: dict[str, tuple[int, int, int, int, int, int, int, int]],
|
|
4861
|
+
source_cache: _SourceSnapshotCache, *, root_arg: str,
|
|
4862
|
+
) -> dict[str, Any]:
|
|
4863
|
+
if getattr(args, "selection_plan", False) and (args.manifest_out or args.pack_out):
|
|
4864
|
+
raise PackError("selection planning is read-only; output paths are unsupported")
|
|
4865
|
+
if not args.json:
|
|
4866
|
+
raise PackError("selection planning requires --json")
|
|
4867
|
+
if getattr(args, "selection_plan", False) and getattr(args, "apply_selection_plan", None):
|
|
4868
|
+
raise PackError("selection plan and apply are mutually exclusive")
|
|
4869
|
+
if args.delta_from_pack_id:
|
|
4870
|
+
raise PackError("selection plan does not cross private receipt boundaries")
|
|
4871
|
+
explicit_paths = split_suggest_files(args.files) + list(args.output or []) + list(args.test_output or [])
|
|
4872
|
+
if any(repo_map_path_has_sensitive_evidence(path) or re.search(r"(?i)(?:^|[-_/.])(scorer|private)(?:[-_/.]|$)", path) for path in explicit_paths):
|
|
4873
|
+
raise PackError("selection plan refuses scorer/private data")
|
|
4874
|
+
if any(
|
|
4875
|
+
isinstance(item, dict) and str(item.get("path", "")).startswith("redacted-path#")
|
|
4876
|
+
for item in repo_map.get("token_tree", [])
|
|
4877
|
+
):
|
|
4878
|
+
raise PackError("selection plan refuses scorer/private data")
|
|
4879
|
+
caps = repo_map.get("caps", {}) if isinstance(repo_map.get("caps"), dict) else {}
|
|
4880
|
+
summary = repo_map.get("summary", {}) if isinstance(repo_map.get("summary"), dict) else {}
|
|
4881
|
+
graph = repo_map.get("graph", {}) if isinstance(repo_map.get("graph"), dict) else {}
|
|
4882
|
+
if SECRET_CONTENT_RE.search(str(args.query)):
|
|
4883
|
+
raise PackError("selection plan refuses secret-risk input")
|
|
4884
|
+
if (
|
|
4885
|
+
any(bool(caps.get(key)) for key in ("files_capped", "candidate_files_capped", "scan_files_capped"))
|
|
4886
|
+
or int(summary.get("bytes_per_file_capped_count", 0) or 0) != 0
|
|
4887
|
+
or bool(repo_map.get("omitted_files"))
|
|
4888
|
+
or int(graph.get("edges_omitted_by_cap", 0) or 0) != 0
|
|
4889
|
+
or bool(ordinary_build.get("input", {}).get("capped"))
|
|
4890
|
+
or bool(selected_build.get("input", {}).get("capped"))
|
|
4891
|
+
or any(
|
|
4892
|
+
isinstance(item, dict) and item.get("reason") == "query_scan_truncated"
|
|
4893
|
+
for item in suggest_payload.get("omitted_sources", [])
|
|
4894
|
+
)
|
|
4895
|
+
):
|
|
4896
|
+
raise PackError("selection plan requires a complete scan")
|
|
4897
|
+
secret_scan = repo_map.get("secret_scan", {}) if isinstance(repo_map.get("secret_scan"), dict) else {}
|
|
4898
|
+
if (
|
|
4899
|
+
int(ordinary_build.get("redaction", {}).get("redacted_lines", 0) or 0) != 0
|
|
4900
|
+
or int(selected_build.get("redaction", {}).get("redacted_lines", 0) or 0) != 0
|
|
4901
|
+
or bool(secret_scan.get("files_with_risks"))
|
|
4902
|
+
or int(secret_scan.get("files_omitted_by_cap", 0) or 0) != 0
|
|
4903
|
+
):
|
|
4904
|
+
raise PackError("selection plan refuses secret-risk input")
|
|
4905
|
+
|
|
4906
|
+
ordinary = [
|
|
4907
|
+
selection_plan_source(item, identities, source_cache, root_arg)
|
|
4908
|
+
for item in ordinary_build.get("included_sources", []) if isinstance(item, dict)
|
|
4909
|
+
]
|
|
4910
|
+
decisions = [copy.deepcopy(item) for item in receipt.get("decisions", []) if isinstance(item, dict)]
|
|
4911
|
+
for decision in decisions:
|
|
4912
|
+
fallback = decision.get("exact_fallback", {})
|
|
4913
|
+
if fallback.get("kind") != "exact_source_slice":
|
|
4914
|
+
raise PackError("selection plan missing exact recovery")
|
|
4915
|
+
recovered_removed = []
|
|
4916
|
+
for removed in decision.get("removed_sources", []):
|
|
4917
|
+
if not isinstance(removed, dict):
|
|
4918
|
+
raise PackError("selection plan missing exact recovery")
|
|
4919
|
+
recovered = copy.deepcopy(removed)
|
|
4920
|
+
if recovered.get("exact_fallback", {}).get("kind") != "exact_source_slice":
|
|
4921
|
+
source = selection_plan_source(recovered, identities, source_cache, root_arg)
|
|
4922
|
+
recovered["frozen_identity"] = source["identity"]
|
|
4923
|
+
recovered["exact_fallback"] = exact_source_fallback(
|
|
4924
|
+
root_arg, source["path"], source["lines"],
|
|
4925
|
+
expected_content_sha256=source["content_sha256"],
|
|
4926
|
+
)
|
|
4927
|
+
recovered_removed.append(recovered)
|
|
4928
|
+
decision["removed_sources"] = recovered_removed
|
|
4929
|
+
selected = [item for item in decisions if item.get("status") == "selected"]
|
|
4930
|
+
omitted = [item for item in decisions if item.get("status") != "selected"]
|
|
4931
|
+
replacement = [
|
|
4932
|
+
{"candidate_identity": item["frozen_identity"], "removed": copy.deepcopy(item.get("removed_sources", []))}
|
|
4933
|
+
for item in selected if item.get("removed_sources")
|
|
4934
|
+
]
|
|
4935
|
+
fallback = [
|
|
4936
|
+
{"identity": item["identity"], "exact_fallback": copy.deepcopy(item["exact_fallback"])}
|
|
4937
|
+
for item in ordinary
|
|
4938
|
+
] + [
|
|
4939
|
+
{"identity": item["frozen_identity"], "exact_fallback": copy.deepcopy(item["exact_fallback"])}
|
|
4940
|
+
for item in decisions
|
|
4941
|
+
]
|
|
4942
|
+
material: dict[str, Any] = {
|
|
4943
|
+
"schema_version": SELECTION_PLAN_SCHEMA_VERSION,
|
|
4944
|
+
"ordinary": ordinary,
|
|
4945
|
+
"candidate": decisions,
|
|
4946
|
+
"selected": selected,
|
|
4947
|
+
"omitted": omitted,
|
|
4948
|
+
"replacement": replacement,
|
|
4949
|
+
"ceiling": {
|
|
4950
|
+
"unit": "rendered_bytes",
|
|
4951
|
+
"ordinary": int(receipt.get("ordinary_pack_bytes", 0) or 0),
|
|
4952
|
+
"selected": int(receipt.get("selected_rendered_bytes", 0) or 0),
|
|
4953
|
+
},
|
|
4954
|
+
"fallback": fallback,
|
|
4955
|
+
"provenance": {
|
|
4956
|
+
"query_sha256": sha256_text(str(args.query)),
|
|
4957
|
+
"diff": cap_label(args.diff) if args.diff else None,
|
|
4958
|
+
"ordinary_pack_id": ordinary_build.get("pack_id"),
|
|
4959
|
+
"selected_pack_id": selected_build.get("pack_id"),
|
|
4960
|
+
"source_identities": sorted(item["identity"] for item in ordinary),
|
|
4961
|
+
},
|
|
4962
|
+
"safety": {
|
|
4963
|
+
"read_only": True, "provider_free": True, "complete_scan": True,
|
|
4964
|
+
"source_revalidation_required_on_apply": True,
|
|
4965
|
+
},
|
|
4966
|
+
"claim_boundary": {"provider_token_or_cost_savings_claim_allowed": False},
|
|
4967
|
+
}
|
|
4968
|
+
plan_id = "sha256:" + sha256_text(json.dumps(material, ensure_ascii=False, sort_keys=True, separators=(",", ":")))
|
|
4969
|
+
return {"schema_version": material.pop("schema_version"), "plan_id": plan_id, **material}
|
|
4970
|
+
|
|
4971
|
+
|
|
4972
|
+
def read_selection_plan(root: Path, raw_path: str) -> dict[str, Any]:
|
|
4973
|
+
rel = output_rel_for_collision_check(raw_path, "--apply-selection-plan")
|
|
4974
|
+
try:
|
|
4975
|
+
value = json.loads(
|
|
4976
|
+
read_manifest_bytes_no_follow(root / rel).decode("utf-8"),
|
|
4977
|
+
object_pairs_hook=strict_json_object,
|
|
4978
|
+
parse_constant=reject_json_constant,
|
|
4979
|
+
parse_int=parse_receipt_int,
|
|
4980
|
+
)
|
|
4981
|
+
json_depth(value)
|
|
4982
|
+
except (UnicodeDecodeError, ValueError, RecursionError) as exc:
|
|
4983
|
+
raise PackError("invalid selection plan JSON") from exc
|
|
4984
|
+
if not isinstance(value, dict) or value.get("schema_version") != SELECTION_PLAN_SCHEMA_VERSION:
|
|
4985
|
+
raise PackError("unsupported selection plan schema")
|
|
4986
|
+
return value
|
|
4987
|
+
|
|
4988
|
+
|
|
4989
|
+
def apply_symbol_memory_graph(
|
|
4990
|
+
manifest: dict[str, Any],
|
|
4991
|
+
repo_map: dict[str, Any],
|
|
4992
|
+
*,
|
|
4993
|
+
complete_secret_paths: set[str] | None = None,
|
|
4994
|
+
) -> tuple[dict[str, Any], dict[str, Any]]:
|
|
4995
|
+
"""Add bounded direct graph neighbors to an explicit auto-pack manifest."""
|
|
4996
|
+
|
|
4997
|
+
raw_sources = manifest.get("sources")
|
|
4998
|
+
if not isinstance(raw_sources, list):
|
|
4999
|
+
raise PackError("manifest sources must be a list")
|
|
5000
|
+
existing_sources = [
|
|
5001
|
+
copy.deepcopy(item) for item in raw_sources if isinstance(item, dict)
|
|
5002
|
+
]
|
|
5003
|
+
existing_paths = {
|
|
5004
|
+
str(item.get("path", "")) for item in existing_sources if item.get("path")
|
|
5005
|
+
}
|
|
5006
|
+
graph = repo_map.get("graph") if isinstance(repo_map.get("graph"), dict) else {}
|
|
5007
|
+
edges = graph.get("edges") if isinstance(graph.get("edges"), list) else []
|
|
5008
|
+
rank_items = (
|
|
5009
|
+
repo_map.get("graph_rank")
|
|
5010
|
+
if isinstance(repo_map.get("graph_rank"), list)
|
|
5011
|
+
else []
|
|
5012
|
+
)
|
|
5013
|
+
rank_by_path = {
|
|
5014
|
+
str(item.get("path", "")): item
|
|
5015
|
+
for item in rank_items
|
|
5016
|
+
if isinstance(item, dict) and item.get("path")
|
|
5017
|
+
}
|
|
5018
|
+
secret_scan = (
|
|
5019
|
+
repo_map.get("secret_scan")
|
|
5020
|
+
if isinstance(repo_map.get("secret_scan"), dict)
|
|
5021
|
+
else {}
|
|
5022
|
+
)
|
|
5023
|
+
risky_paths = (
|
|
5024
|
+
complete_secret_paths
|
|
5025
|
+
if complete_secret_paths is not None
|
|
5026
|
+
else {
|
|
5027
|
+
str(item.get("path", ""))
|
|
5028
|
+
for item in secret_scan.get("files_with_risks", [])
|
|
5029
|
+
if isinstance(item, dict) and item.get("path")
|
|
5030
|
+
}
|
|
5031
|
+
)
|
|
5032
|
+
direct_neighbors: set[str] = set()
|
|
5033
|
+
for edge in edges:
|
|
5034
|
+
if not isinstance(edge, dict):
|
|
5035
|
+
continue
|
|
5036
|
+
source = edge.get("from")
|
|
5037
|
+
target = edge.get("to")
|
|
5038
|
+
if not isinstance(source, str) or not isinstance(target, str):
|
|
5039
|
+
continue
|
|
5040
|
+
if source in existing_paths and target not in existing_paths:
|
|
5041
|
+
direct_neighbors.add(target)
|
|
5042
|
+
if target in existing_paths and source not in existing_paths:
|
|
5043
|
+
direct_neighbors.add(source)
|
|
5044
|
+
|
|
5045
|
+
eligible: list[tuple[int, str, int]] = []
|
|
5046
|
+
excluded_secret_risk_count = 0
|
|
5047
|
+
for path in direct_neighbors:
|
|
5048
|
+
if path in risky_paths:
|
|
5049
|
+
excluded_secret_risk_count += 1
|
|
5050
|
+
continue
|
|
5051
|
+
item = rank_by_path.get(path)
|
|
5052
|
+
if item is None or repo_map_path_has_sensitive_evidence(path):
|
|
5053
|
+
continue
|
|
5054
|
+
score = int(item.get("score", 0) or 0)
|
|
5055
|
+
line_count = int(item.get("line_count", 0) or 0)
|
|
5056
|
+
if score <= 0 or line_count <= 0:
|
|
5057
|
+
continue
|
|
5058
|
+
eligible.append((score, path, line_count))
|
|
5059
|
+
eligible.sort(key=lambda item: (-item[0], item[1]))
|
|
5060
|
+
|
|
5061
|
+
seed_priorities = [
|
|
5062
|
+
int(item.get("priority", 0) or 0) for item in existing_sources
|
|
5063
|
+
]
|
|
5064
|
+
maximum_graph_priority = max(1, min(seed_priorities, default=2) - 1)
|
|
5065
|
+
selected_sources: list[dict[str, Any]] = []
|
|
5066
|
+
for score, path, line_count in eligible[:MAX_GRAPH_APPLICATION_SOURCES]:
|
|
5067
|
+
source = {
|
|
5068
|
+
"path": path,
|
|
5069
|
+
"priority": max(1, min(score, maximum_graph_priority)),
|
|
5070
|
+
"label": f"graph:{path}"[:MAX_LABEL_CHARS],
|
|
5071
|
+
"lines": {"start": 1, "end": min(line_count, MAX_GRAPH_APPLICATION_LINES)},
|
|
5072
|
+
}
|
|
5073
|
+
existing_sources.append(source)
|
|
5074
|
+
selected_sources.append(
|
|
5075
|
+
{
|
|
5076
|
+
"path": source["path"],
|
|
5077
|
+
"priority": source["priority"],
|
|
5078
|
+
"lines": copy.deepcopy(source["lines"]),
|
|
5079
|
+
"reason": "direct_import_neighbor",
|
|
5080
|
+
}
|
|
5081
|
+
)
|
|
5082
|
+
result_manifest = build_suggest_manifest(existing_sources)
|
|
5083
|
+
return result_manifest, {
|
|
5084
|
+
"schema_version": GRAPH_APPLICATION_SCHEMA_VERSION,
|
|
5085
|
+
"mode": "explicit_opt_in",
|
|
5086
|
+
"selected_source_count": len(selected_sources),
|
|
5087
|
+
"selected_sources": selected_sources,
|
|
5088
|
+
"candidate_count": len(eligible),
|
|
5089
|
+
"candidate_cap": MAX_GRAPH_APPLICATION_SOURCES,
|
|
5090
|
+
"candidate_cap_reached": len(eligible) > MAX_GRAPH_APPLICATION_SOURCES,
|
|
5091
|
+
"excluded_secret_risk_count": excluded_secret_risk_count,
|
|
5092
|
+
"exact_source_fallback_retained": True,
|
|
5093
|
+
"deterministic_local_only": True,
|
|
5094
|
+
"provider_token_or_cost_savings_claim_allowed": False,
|
|
5095
|
+
}
|
|
5096
|
+
|
|
5097
|
+
|
|
5098
|
+
def bind_graph_sources_to_repo_snapshot(
|
|
5099
|
+
root: Path,
|
|
5100
|
+
specs: list[SourceSpec],
|
|
5101
|
+
required_sources: set[tuple[str, str]],
|
|
5102
|
+
source_identities: dict[str, tuple[int, int, int, int, int, int, int, int]],
|
|
5103
|
+
*,
|
|
5104
|
+
source_cache: _SourceSnapshotCache,
|
|
5105
|
+
input_budget: _SourceInputBudget,
|
|
5106
|
+
) -> dict[tuple[str, str], dict[str, Any]]:
|
|
5107
|
+
"""Warm immutable source snapshots only for repo-map-bound graph additions."""
|
|
5108
|
+
|
|
5109
|
+
rejections: dict[tuple[str, str], dict[str, Any]] = {}
|
|
5110
|
+
for spec in specs:
|
|
5111
|
+
rel, _reason = lexical_rel(spec.path)
|
|
5112
|
+
if rel is None:
|
|
5113
|
+
continue
|
|
5114
|
+
lines_identity = spec.lines.identity() if spec.lines is not None else "all"
|
|
5115
|
+
source_identity = (rel.as_posix(), lines_identity)
|
|
5116
|
+
if source_identity not in required_sources:
|
|
5117
|
+
continue
|
|
5118
|
+
expected_identity = source_identities.get(rel.as_posix())
|
|
5119
|
+
if expected_identity is None:
|
|
5120
|
+
continue
|
|
5121
|
+
_source, omitted_item = resolve_source(
|
|
5122
|
+
root,
|
|
5123
|
+
spec,
|
|
5124
|
+
source_cache=source_cache,
|
|
5125
|
+
input_budget=input_budget,
|
|
5126
|
+
expected_identity=expected_identity,
|
|
5127
|
+
)
|
|
5128
|
+
if omitted_item is not None:
|
|
5129
|
+
rejections[source_identity] = copy.deepcopy(omitted_item)
|
|
5130
|
+
return rejections
|
|
5131
|
+
|
|
5132
|
+
|
|
5133
|
+
def build_symbol_memory_payload(
|
|
5134
|
+
repo_map: dict[str, Any], *, applied: bool = False
|
|
5135
|
+
) -> dict[str, Any]:
|
|
3689
5136
|
retrieval_by_path_lines: dict[tuple[str, str], dict[str, Any]] = {}
|
|
3690
5137
|
for item in repo_map.get("retrieval", []):
|
|
3691
5138
|
if not isinstance(item, dict):
|
|
@@ -3736,7 +5183,7 @@ def build_symbol_memory_payload(repo_map: dict[str, Any]) -> dict[str, Any]:
|
|
|
3736
5183
|
retrieval = repo_map.get("retrieval", []) if isinstance(repo_map.get("retrieval"), list) else []
|
|
3737
5184
|
return {
|
|
3738
5185
|
"schema_version": SYMBOL_MEMORY_SCHEMA_VERSION,
|
|
3739
|
-
"mode": "advisory",
|
|
5186
|
+
"mode": "applied" if applied else "advisory",
|
|
3740
5187
|
"source": "contextguard.pack-repo-map.v1",
|
|
3741
5188
|
"summary": {
|
|
3742
5189
|
"symbols": len(symbols),
|
|
@@ -3756,8 +5203,9 @@ def build_symbol_memory_payload(repo_map: dict[str, Any]) -> dict[str, Any]:
|
|
|
3756
5203
|
"claim_boundary": {
|
|
3757
5204
|
"deterministic_local_only": True,
|
|
3758
5205
|
"no_network_model_embedding_lsp_or_tree_sitter_dependency": True,
|
|
3759
|
-
"advisory_does_not_change_manifest_pack_or_receipt":
|
|
3760
|
-
"
|
|
5206
|
+
"advisory_does_not_change_manifest_pack_or_receipt": not applied,
|
|
5207
|
+
"explicit_graph_application_changes_manifest_and_pack": applied,
|
|
5208
|
+
"graph_rank_is_explain_only": not applied,
|
|
3761
5209
|
"provider_token_or_cost_savings_claim_allowed": False,
|
|
3762
5210
|
},
|
|
3763
5211
|
}
|
|
@@ -3903,10 +5351,21 @@ def build_auto_explain_payload(
|
|
|
3903
5351
|
explain["repo_map"] = copy.deepcopy(repo_map_payload)
|
|
3904
5352
|
elif root is not None:
|
|
3905
5353
|
explain["repo_map"] = build_repo_map_payload(root, args, suggest_payload, build_payload, root_arg=root_arg)
|
|
5354
|
+
if isinstance(payload.get("graph_application"), dict):
|
|
5355
|
+
explain["graph_application"] = copy.deepcopy(payload["graph_application"])
|
|
5356
|
+
if isinstance(payload.get("adaptive_k_application"), dict):
|
|
5357
|
+
explain["adaptive_k_application"] = copy.deepcopy(
|
|
5358
|
+
payload["adaptive_k_application"]
|
|
5359
|
+
)
|
|
3906
5360
|
return explain
|
|
3907
5361
|
|
|
3908
5362
|
|
|
3909
5363
|
def auto_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tuple[dict[str, Any], int]:
|
|
5364
|
+
plan_only = bool(getattr(args, "selection_plan", False))
|
|
5365
|
+
apply_plan_path = getattr(args, "apply_selection_plan", None)
|
|
5366
|
+
expected_plan = read_selection_plan(root, apply_plan_path) if apply_plan_path else None
|
|
5367
|
+
source_cache = _SourceSnapshotCache()
|
|
5368
|
+
input_budget = _SourceInputBudget()
|
|
3910
5369
|
manifest_rel = output_rel_for_collision_check(args.manifest_out, "--manifest-out") if args.manifest_out else None
|
|
3911
5370
|
pack_rel = output_rel_for_collision_check(args.pack_out, "--pack-out") if args.pack_out else None
|
|
3912
5371
|
if manifest_rel is not None and pack_rel is not None:
|
|
@@ -3923,8 +5382,52 @@ def auto_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tuple[d
|
|
|
3923
5382
|
validate_output_path_under_root(root, args.pack_out, "--pack-out")
|
|
3924
5383
|
suggest_args = copy.copy(args)
|
|
3925
5384
|
suggest_args.manifest_out = None
|
|
3926
|
-
|
|
5385
|
+
self_financing = bool(getattr(args, "self_financing_selection", False) or plan_only or apply_plan_path)
|
|
5386
|
+
apply_adaptive_k = bool(getattr(args, "apply_adaptive_k", False) or self_financing)
|
|
5387
|
+
if apply_adaptive_k:
|
|
5388
|
+
suggest_args.adaptive_k = True
|
|
5389
|
+
suggest_payload, rc = suggest_pack(
|
|
5390
|
+
root,
|
|
5391
|
+
suggest_args,
|
|
5392
|
+
root_arg=root_arg,
|
|
5393
|
+
_source_cache=source_cache,
|
|
5394
|
+
_input_budget=input_budget,
|
|
5395
|
+
)
|
|
3927
5396
|
manifest = suggest_payload["manifest"]
|
|
5397
|
+
ordinary_build_payload: dict[str, Any] | None = None
|
|
5398
|
+
if self_financing:
|
|
5399
|
+
ordinary_build_payload = build_pack(
|
|
5400
|
+
root,
|
|
5401
|
+
manifest_to_source_specs(manifest),
|
|
5402
|
+
budget_bytes=bounded_int(args.budget_bytes, DEFAULT_BUDGET_BYTES, MIN_BUDGET_BYTES, MAX_BUDGET_BYTES),
|
|
5403
|
+
root_arg=root_arg,
|
|
5404
|
+
store_artifact=False,
|
|
5405
|
+
sketch_duplicate_veto=getattr(args, "sketch_duplicate_veto", False),
|
|
5406
|
+
_source_cache=source_cache,
|
|
5407
|
+
_input_budget=input_budget,
|
|
5408
|
+
)
|
|
5409
|
+
adaptive_k_application: dict[str, Any] | None = None
|
|
5410
|
+
if apply_adaptive_k and isinstance(suggest_payload.get("adaptive_k"), dict):
|
|
5411
|
+
manifest, adaptive_k_application = apply_adaptive_k_manifest(
|
|
5412
|
+
manifest,
|
|
5413
|
+
suggest_payload["adaptive_k"],
|
|
5414
|
+
)
|
|
5415
|
+
suggest_payload["manifest"] = manifest
|
|
5416
|
+
retained_identities = {
|
|
5417
|
+
(str(item.get("path", "")), line_range_identity(item.get("lines")))
|
|
5418
|
+
for item in manifest.get("sources", [])
|
|
5419
|
+
if isinstance(item, dict)
|
|
5420
|
+
}
|
|
5421
|
+
suggest_payload["sources"] = [
|
|
5422
|
+
item
|
|
5423
|
+
for item in suggest_payload.get("sources", [])
|
|
5424
|
+
if isinstance(item, dict)
|
|
5425
|
+
and (
|
|
5426
|
+
str(item.get("path", "")),
|
|
5427
|
+
line_range_identity(item.get("lines")),
|
|
5428
|
+
)
|
|
5429
|
+
in retained_identities
|
|
5430
|
+
]
|
|
3928
5431
|
specs = manifest_to_source_specs(manifest)
|
|
3929
5432
|
budget = bounded_int(args.budget_bytes, DEFAULT_BUDGET_BYTES, MIN_BUDGET_BYTES, MAX_BUDGET_BYTES)
|
|
3930
5433
|
build_payload = build_pack(
|
|
@@ -3935,7 +5438,387 @@ def auto_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tuple[d
|
|
|
3935
5438
|
store_artifact=False,
|
|
3936
5439
|
delta_from_pack_id=args.delta_from_pack_id,
|
|
3937
5440
|
sketch_duplicate_veto=getattr(args, "sketch_duplicate_veto", False),
|
|
5441
|
+
_source_cache=source_cache,
|
|
5442
|
+
_input_budget=input_budget,
|
|
3938
5443
|
)
|
|
5444
|
+
if apply_adaptive_k:
|
|
5445
|
+
suggest_payload["estimated_pack_bytes"] = build_payload.get("pack_bytes", 0)
|
|
5446
|
+
suggest_payload["token_proxy"] = copy.deepcopy(
|
|
5447
|
+
build_payload.get("token_proxy", {})
|
|
5448
|
+
)
|
|
5449
|
+
repo_map_payload: dict[str, Any] | None = None
|
|
5450
|
+
graph_application: dict[str, Any] | None = None
|
|
5451
|
+
apply_symbol_memory = bool(getattr(args, "apply_symbol_memory", False) or self_financing)
|
|
5452
|
+
complete_secret_paths: set[str] | None = set() if apply_symbol_memory else None
|
|
5453
|
+
repo_map_source_identities: dict[
|
|
5454
|
+
str,
|
|
5455
|
+
tuple[int, int, int, int, int, int, int, int],
|
|
5456
|
+
] = {}
|
|
5457
|
+
if getattr(args, "symbol_memory", False) or apply_symbol_memory or args.explain:
|
|
5458
|
+
repo_map_payload = build_repo_map_payload(
|
|
5459
|
+
root,
|
|
5460
|
+
args,
|
|
5461
|
+
suggest_payload,
|
|
5462
|
+
build_payload,
|
|
5463
|
+
root_arg=root_arg,
|
|
5464
|
+
complete_secret_paths_out=complete_secret_paths,
|
|
5465
|
+
source_identities_out=(
|
|
5466
|
+
repo_map_source_identities if apply_symbol_memory else None
|
|
5467
|
+
),
|
|
5468
|
+
)
|
|
5469
|
+
self_financing_receipt: dict[str, Any] | None = None
|
|
5470
|
+
if self_financing and isinstance(repo_map_payload, dict) and ordinary_build_payload is not None:
|
|
5471
|
+
ordinary_ceiling = int(ordinary_build_payload.get("pack_bytes", 0) or 0)
|
|
5472
|
+
decisions: list[dict[str, Any]] = []
|
|
5473
|
+
recorded_secret_decisions: set[tuple[str, str]] = set()
|
|
5474
|
+
ordinary_sources = ordinary_build_payload.get("included_sources", [])
|
|
5475
|
+
retained_keys = {
|
|
5476
|
+
(str(item.get("path", "")), line_range_identity(item.get("lines")))
|
|
5477
|
+
for item in manifest.get("sources", []) if isinstance(item, dict)
|
|
5478
|
+
}
|
|
5479
|
+
for item in ordinary_sources if isinstance(ordinary_sources, list) else []:
|
|
5480
|
+
if not isinstance(item, dict):
|
|
5481
|
+
continue
|
|
5482
|
+
key = (str(item.get("path", "")), line_range_identity(item.get("requested_lines")))
|
|
5483
|
+
if key not in retained_keys:
|
|
5484
|
+
source = {"path": key[0], "lines": copy.deepcopy(item.get("requested_lines"))}
|
|
5485
|
+
decisions.append(self_financing_candidate_receipt(
|
|
5486
|
+
phase="adaptive", source=source, reason="adaptive_headroom_removal",
|
|
5487
|
+
status="selected", secret_decision="allow", identities=repo_map_source_identities,
|
|
5488
|
+
root_arg=root_arg, source_cache=source_cache,
|
|
5489
|
+
byte_delta=-int(item.get("bytes", 0) or 0),
|
|
5490
|
+
removed_sources=[source],
|
|
5491
|
+
))
|
|
5492
|
+
|
|
5493
|
+
existing_paths = {str(item.get("path", "")) for item in manifest.get("sources", []) if isinstance(item, dict)}
|
|
5494
|
+
query_terms = suggest_tokens(str(suggest_payload.get("query", "")))
|
|
5495
|
+
candidates: list[tuple[str, dict[str, Any], str]] = []
|
|
5496
|
+
for signature in repo_map_payload.get("signature_index", []):
|
|
5497
|
+
if not isinstance(signature, dict):
|
|
5498
|
+
continue
|
|
5499
|
+
path = str(signature.get("path", ""))
|
|
5500
|
+
searchable = suggest_tokens(f"{signature.get('name', '')} {signature.get('signature', '')}")
|
|
5501
|
+
if not query_terms.intersection(searchable):
|
|
5502
|
+
continue
|
|
5503
|
+
source = {
|
|
5504
|
+
"path": path, "priority": 2, "label": f"symbol:{path}"[:MAX_LABEL_CHARS],
|
|
5505
|
+
"lines": copy.deepcopy(signature.get("lines")),
|
|
5506
|
+
}
|
|
5507
|
+
if path in (complete_secret_paths or set()):
|
|
5508
|
+
decisions.append(self_financing_candidate_receipt(
|
|
5509
|
+
phase="symbol", source=source, reason="secret_risk", status="no_op",
|
|
5510
|
+
secret_decision="reject", identities=repo_map_source_identities,
|
|
5511
|
+
root_arg=root_arg, source_cache=source_cache,
|
|
5512
|
+
))
|
|
5513
|
+
recorded_secret_decisions.add(("symbol", path))
|
|
5514
|
+
continue
|
|
5515
|
+
if path in existing_paths:
|
|
5516
|
+
existing_source = next(
|
|
5517
|
+
(
|
|
5518
|
+
item for item in build_payload.get("included_sources", [])
|
|
5519
|
+
if isinstance(item, dict)
|
|
5520
|
+
and item.get("path") == path
|
|
5521
|
+
and item.get("status") == "included"
|
|
5522
|
+
and item.get("requested_lines") == item.get("included_lines")
|
|
5523
|
+
),
|
|
5524
|
+
None,
|
|
5525
|
+
)
|
|
5526
|
+
if existing_source is not None:
|
|
5527
|
+
source["lines"] = copy.deepcopy(existing_source["requested_lines"])
|
|
5528
|
+
decisions.append(self_financing_candidate_receipt(
|
|
5529
|
+
phase="symbol", source=source, reason="duplicate_source", status="no_op",
|
|
5530
|
+
secret_decision="allow", identities=repo_map_source_identities,
|
|
5531
|
+
root_arg=root_arg, source_cache=source_cache,
|
|
5532
|
+
))
|
|
5533
|
+
continue
|
|
5534
|
+
candidates.append(("symbol", source, "task_matching_symbol"))
|
|
5535
|
+
if len([item for item in candidates if item[0] == "symbol"]) >= MAX_GRAPH_APPLICATION_SOURCES:
|
|
5536
|
+
break
|
|
5537
|
+
graph_manifest, graph_preview = apply_symbol_memory_graph(
|
|
5538
|
+
manifest, repo_map_payload, complete_secret_paths=complete_secret_paths,
|
|
5539
|
+
)
|
|
5540
|
+
graph_sources = graph_manifest.get("sources", [])
|
|
5541
|
+
for source in graph_sources[len(manifest.get("sources", [])):]:
|
|
5542
|
+
if isinstance(source, dict):
|
|
5543
|
+
candidates.append(("graph", source, "direct_import_neighbor"))
|
|
5544
|
+
|
|
5545
|
+
frozen_candidate_sources = {
|
|
5546
|
+
(str(source.get("path", "")), line_range_identity(source.get("lines")))
|
|
5547
|
+
for _phase, source, _reason in candidates
|
|
5548
|
+
}
|
|
5549
|
+
candidate_specs = manifest_to_source_specs(build_suggest_manifest(
|
|
5550
|
+
[source for _phase, source, _reason in candidates]
|
|
5551
|
+
))
|
|
5552
|
+
candidate_snapshot_rejections = bind_graph_sources_to_repo_snapshot(
|
|
5553
|
+
root, candidate_specs, frozen_candidate_sources, repo_map_source_identities,
|
|
5554
|
+
source_cache=source_cache, input_budget=input_budget,
|
|
5555
|
+
)
|
|
5556
|
+
|
|
5557
|
+
# Record secret-risk direct neighbors as explicit no-ops without exposing their contents.
|
|
5558
|
+
graph = repo_map_payload.get("graph", {})
|
|
5559
|
+
for edge in graph.get("edges", []) if isinstance(graph, dict) else []:
|
|
5560
|
+
if not isinstance(edge, dict):
|
|
5561
|
+
continue
|
|
5562
|
+
for path in (edge.get("from"), edge.get("to")):
|
|
5563
|
+
if isinstance(path, str) and path in (complete_secret_paths or set()) and path not in existing_paths:
|
|
5564
|
+
if ("graph", path) in recorded_secret_decisions:
|
|
5565
|
+
continue
|
|
5566
|
+
source = {"path": path, "lines": None}
|
|
5567
|
+
decisions.append(self_financing_candidate_receipt(
|
|
5568
|
+
phase="graph", source=source, reason="secret_risk", status="no_op",
|
|
5569
|
+
secret_decision="reject", identities=repo_map_source_identities,
|
|
5570
|
+
root_arg=root_arg, source_cache=source_cache,
|
|
5571
|
+
))
|
|
5572
|
+
recorded_secret_decisions.add(("graph", path))
|
|
5573
|
+
|
|
5574
|
+
current_manifest = copy.deepcopy(manifest)
|
|
5575
|
+
current_build = build_payload
|
|
5576
|
+
protected_sources = [
|
|
5577
|
+
(
|
|
5578
|
+
str(item.get("path", "")),
|
|
5579
|
+
copy.deepcopy(item.get("lines")) if isinstance(item.get("lines"), dict) else None,
|
|
5580
|
+
)
|
|
5581
|
+
for item in current_manifest.get("sources", [])
|
|
5582
|
+
if isinstance(item, dict)
|
|
5583
|
+
and str(item.get("label", "")).startswith(
|
|
5584
|
+
("file:", "output:", "test-output:", "diff:", "critical:")
|
|
5585
|
+
)
|
|
5586
|
+
]
|
|
5587
|
+
|
|
5588
|
+
def protected_sources_are_exact(build: dict[str, Any]) -> bool:
|
|
5589
|
+
included_sources = [
|
|
5590
|
+
item for item in build.get("included_sources", [])
|
|
5591
|
+
if isinstance(item, dict)
|
|
5592
|
+
]
|
|
5593
|
+
for protected_path, protected_lines in protected_sources:
|
|
5594
|
+
matched = False
|
|
5595
|
+
for item in included_sources:
|
|
5596
|
+
if (
|
|
5597
|
+
item.get("path") != protected_path
|
|
5598
|
+
or item.get("status") != "included"
|
|
5599
|
+
or item.get("requested_lines") != item.get("included_lines")
|
|
5600
|
+
):
|
|
5601
|
+
continue
|
|
5602
|
+
if protected_lines is not None and item.get("requested_lines") != protected_lines:
|
|
5603
|
+
continue
|
|
5604
|
+
matched = True
|
|
5605
|
+
break
|
|
5606
|
+
if not matched:
|
|
5607
|
+
return False
|
|
5608
|
+
return True
|
|
5609
|
+
|
|
5610
|
+
seen_candidates: set[tuple[str, str]] = set()
|
|
5611
|
+
for phase, candidate, reason in candidates:
|
|
5612
|
+
key = (str(candidate.get("path", "")), line_range_identity(candidate.get("lines")))
|
|
5613
|
+
if key in seen_candidates or key[0] in {str(item.get("path", "")) for item in current_manifest.get("sources", []) if isinstance(item, dict)}:
|
|
5614
|
+
decisions.append(self_financing_candidate_receipt(
|
|
5615
|
+
phase=phase, source=candidate, reason="duplicate_source", status="no_op",
|
|
5616
|
+
secret_decision="allow", identities=repo_map_source_identities,
|
|
5617
|
+
root_arg=root_arg, source_cache=source_cache,
|
|
5618
|
+
))
|
|
5619
|
+
continue
|
|
5620
|
+
seen_candidates.add(key)
|
|
5621
|
+
trial_sources = [copy.deepcopy(item) for item in current_manifest.get("sources", []) if isinstance(item, dict)] + [copy.deepcopy(candidate)]
|
|
5622
|
+
candidate_priority = int(candidate.get("priority", 0) or 0)
|
|
5623
|
+
removable = sorted(
|
|
5624
|
+
[
|
|
5625
|
+
item for item in trial_sources[:-1]
|
|
5626
|
+
if not str(item.get("label", "")).startswith(
|
|
5627
|
+
("file:", "output:", "test-output:", "diff:", "critical:")
|
|
5628
|
+
)
|
|
5629
|
+
and int(item.get("priority", 0) or 0) < candidate_priority
|
|
5630
|
+
],
|
|
5631
|
+
key=lambda item: (int(item.get("priority", 0) or 0), str(item.get("path", ""))),
|
|
5632
|
+
)
|
|
5633
|
+
removed: list[dict[str, Any]] = []
|
|
5634
|
+
accepted_build: dict[str, Any] | None = None
|
|
5635
|
+
while True:
|
|
5636
|
+
trial_manifest = build_suggest_manifest(trial_sources)
|
|
5637
|
+
trial_build = build_pack(
|
|
5638
|
+
root, manifest_to_source_specs(trial_manifest), budget_bytes=max(MIN_BUDGET_BYTES, ordinary_ceiling),
|
|
5639
|
+
root_arg=root_arg, store_artifact=False,
|
|
5640
|
+
sketch_duplicate_veto=getattr(args, "sketch_duplicate_veto", False),
|
|
5641
|
+
_source_cache=source_cache, _input_budget=input_budget,
|
|
5642
|
+
_required_snapshot_sources=frozen_candidate_sources,
|
|
5643
|
+
_expected_source_identities=repo_map_source_identities,
|
|
5644
|
+
_snapshot_rejections=candidate_snapshot_rejections,
|
|
5645
|
+
)
|
|
5646
|
+
candidate_included_exactly = any(
|
|
5647
|
+
isinstance(item, dict)
|
|
5648
|
+
and str(item.get("path", "")) == key[0]
|
|
5649
|
+
and line_range_identity(item.get("requested_lines")) == key[1]
|
|
5650
|
+
and line_range_identity(item.get("included_lines")) == key[1]
|
|
5651
|
+
and item.get("status") == "included"
|
|
5652
|
+
for item in trial_build.get("included_sources", [])
|
|
5653
|
+
)
|
|
5654
|
+
if (
|
|
5655
|
+
int(trial_build.get("pack_bytes", 0) or 0) <= ordinary_ceiling
|
|
5656
|
+
and candidate_included_exactly
|
|
5657
|
+
and protected_sources_are_exact(trial_build)
|
|
5658
|
+
):
|
|
5659
|
+
accepted_build = trial_build
|
|
5660
|
+
break
|
|
5661
|
+
if not removable:
|
|
5662
|
+
break
|
|
5663
|
+
victim = removable.pop(0)
|
|
5664
|
+
trial_sources.remove(victim)
|
|
5665
|
+
victim_source = {
|
|
5666
|
+
"path": victim.get("path"),
|
|
5667
|
+
"lines": copy.deepcopy(victim.get("lines")),
|
|
5668
|
+
}
|
|
5669
|
+
if not isinstance(victim_source["lines"], dict):
|
|
5670
|
+
prior = next(
|
|
5671
|
+
(
|
|
5672
|
+
item for item in current_build.get("included_sources", [])
|
|
5673
|
+
if isinstance(item, dict)
|
|
5674
|
+
and item.get("path") == victim_source["path"]
|
|
5675
|
+
and item.get("status") == "included"
|
|
5676
|
+
and item.get("requested_lines") == item.get("included_lines")
|
|
5677
|
+
),
|
|
5678
|
+
None,
|
|
5679
|
+
)
|
|
5680
|
+
if prior is not None:
|
|
5681
|
+
victim_source["lines"] = copy.deepcopy(prior["requested_lines"])
|
|
5682
|
+
removed.append({
|
|
5683
|
+
"path": victim_source["path"], "lines": victim_source["lines"],
|
|
5684
|
+
"reason": "lower_value_replacement",
|
|
5685
|
+
"frozen_identity": frozen_source_identity(
|
|
5686
|
+
str(victim_source["path"] or ""), victim_source["lines"],
|
|
5687
|
+
repo_map_source_identities, source_cache,
|
|
5688
|
+
),
|
|
5689
|
+
"exact_fallback": exact_source_fallback(
|
|
5690
|
+
root_arg, str(victim_source["path"] or ""), victim_source["lines"],
|
|
5691
|
+
expected_content_sha256=frozen_source_content_sha256(
|
|
5692
|
+
str(victim_source["path"] or ""), victim_source["lines"], source_cache
|
|
5693
|
+
),
|
|
5694
|
+
),
|
|
5695
|
+
})
|
|
5696
|
+
if accepted_build is None:
|
|
5697
|
+
decisions.append(self_financing_candidate_receipt(
|
|
5698
|
+
phase=phase, source=candidate, reason="ordinary_ceiling_no_safe_replacement", status="no_op",
|
|
5699
|
+
secret_decision="allow", identities=repo_map_source_identities,
|
|
5700
|
+
root_arg=root_arg, source_cache=source_cache,
|
|
5701
|
+
))
|
|
5702
|
+
continue
|
|
5703
|
+
previous_bytes = int(current_build.get("pack_bytes", 0) or 0)
|
|
5704
|
+
current_manifest = build_suggest_manifest(trial_sources)
|
|
5705
|
+
current_build = accepted_build
|
|
5706
|
+
decisions.append(self_financing_candidate_receipt(
|
|
5707
|
+
phase=phase, source=candidate, reason=reason, status="selected",
|
|
5708
|
+
secret_decision="allow", identities=repo_map_source_identities,
|
|
5709
|
+
root_arg=root_arg, source_cache=source_cache,
|
|
5710
|
+
byte_delta=int(current_build.get("pack_bytes", 0) or 0) - previous_bytes,
|
|
5711
|
+
removed_sources=removed,
|
|
5712
|
+
))
|
|
5713
|
+
manifest = current_manifest
|
|
5714
|
+
build_payload = current_build
|
|
5715
|
+
selected_rendered_bytes = int(build_payload.get("pack_bytes", 0) or 0)
|
|
5716
|
+
if (
|
|
5717
|
+
selected_rendered_bytes > ordinary_ceiling
|
|
5718
|
+
or not protected_sources_are_exact(build_payload)
|
|
5719
|
+
):
|
|
5720
|
+
raise PackError("self-financing selection invariant failed")
|
|
5721
|
+
suggest_payload["manifest"] = manifest
|
|
5722
|
+
suggest_payload["estimated_pack_bytes"] = build_payload.get("pack_bytes", 0)
|
|
5723
|
+
suggest_payload["token_proxy"] = copy.deepcopy(build_payload.get("token_proxy", {}))
|
|
5724
|
+
selected_graph_decisions = [
|
|
5725
|
+
item for item in decisions
|
|
5726
|
+
if item.get("phase") == "graph" and item.get("status") == "selected"
|
|
5727
|
+
]
|
|
5728
|
+
graph_application = {
|
|
5729
|
+
**graph_preview,
|
|
5730
|
+
"selected_source_count": len(selected_graph_decisions),
|
|
5731
|
+
"selected_sources": [
|
|
5732
|
+
{
|
|
5733
|
+
"path": item.get("path"), "lines": copy.deepcopy(item.get("lines")),
|
|
5734
|
+
"reason": item.get("reason"),
|
|
5735
|
+
}
|
|
5736
|
+
for item in selected_graph_decisions
|
|
5737
|
+
],
|
|
5738
|
+
}
|
|
5739
|
+
phase_results = {}
|
|
5740
|
+
for phase in ("adaptive", "symbol", "graph"):
|
|
5741
|
+
phase_decisions = [item for item in decisions if item.get("phase") == phase]
|
|
5742
|
+
phase_results[phase] = {
|
|
5743
|
+
"status": "applied" if any(item.get("status") == "selected" for item in phase_decisions) else "no_op",
|
|
5744
|
+
"selected_count": sum(item.get("status") == "selected" for item in phase_decisions),
|
|
5745
|
+
"no_op_count": sum(item.get("status") == "no_op" for item in phase_decisions),
|
|
5746
|
+
}
|
|
5747
|
+
self_financing_receipt = {
|
|
5748
|
+
"schema_version": SELF_FINANCING_SELECTION_SCHEMA_VERSION,
|
|
5749
|
+
"mode": "explicit_opt_in", "phase_order": ["adaptive", "symbol", "graph"],
|
|
5750
|
+
"ordinary_pack_bytes": ordinary_ceiling,
|
|
5751
|
+
"selected_rendered_bytes": selected_rendered_bytes,
|
|
5752
|
+
"ceiling_respected": selected_rendered_bytes <= ordinary_ceiling,
|
|
5753
|
+
"phase_results": phase_results,
|
|
5754
|
+
"decisions": decisions,
|
|
5755
|
+
"claim_boundary": {"provider_token_or_cost_savings_claim_allowed": False},
|
|
5756
|
+
}
|
|
5757
|
+
elif apply_symbol_memory and isinstance(repo_map_payload, dict):
|
|
5758
|
+
repo_map_payload["safety"]["explain_only"] = False
|
|
5759
|
+
repo_map_payload["safety"]["caveats"] = [
|
|
5760
|
+
"Repo-map bytes are local sampled UTF-8 bytes and estimated chars_div_4 token proxies, not provider-token or savings claims.",
|
|
5761
|
+
"Graph ranking is applied only to the bounded direct-neighbor expansion recorded in graph_application; exact source retrieval remains available.",
|
|
5762
|
+
]
|
|
5763
|
+
pre_graph_sources = {
|
|
5764
|
+
(str(item.get("path", "")), line_range_identity(item.get("lines")))
|
|
5765
|
+
for item in manifest.get("sources", [])
|
|
5766
|
+
if isinstance(item, dict) and item.get("path")
|
|
5767
|
+
}
|
|
5768
|
+
manifest, graph_application = apply_symbol_memory_graph(
|
|
5769
|
+
manifest,
|
|
5770
|
+
repo_map_payload,
|
|
5771
|
+
complete_secret_paths=complete_secret_paths,
|
|
5772
|
+
)
|
|
5773
|
+
suggest_payload["manifest"] = manifest
|
|
5774
|
+
specs = manifest_to_source_specs(manifest)
|
|
5775
|
+
graph_snapshot_sources = {
|
|
5776
|
+
(str(item.get("path", "")), line_range_identity(item.get("lines")))
|
|
5777
|
+
for item in manifest.get("sources", [])
|
|
5778
|
+
if isinstance(item, dict) and item.get("path")
|
|
5779
|
+
} - pre_graph_sources
|
|
5780
|
+
graph_snapshot_rejections = bind_graph_sources_to_repo_snapshot(
|
|
5781
|
+
root,
|
|
5782
|
+
specs,
|
|
5783
|
+
graph_snapshot_sources,
|
|
5784
|
+
repo_map_source_identities,
|
|
5785
|
+
source_cache=source_cache,
|
|
5786
|
+
input_budget=input_budget,
|
|
5787
|
+
)
|
|
5788
|
+
build_payload = build_pack(
|
|
5789
|
+
root,
|
|
5790
|
+
specs,
|
|
5791
|
+
budget_bytes=budget,
|
|
5792
|
+
root_arg=root_arg,
|
|
5793
|
+
store_artifact=False,
|
|
5794
|
+
delta_from_pack_id=args.delta_from_pack_id,
|
|
5795
|
+
sketch_duplicate_veto=getattr(args, "sketch_duplicate_veto", False),
|
|
5796
|
+
_source_cache=source_cache,
|
|
5797
|
+
_input_budget=input_budget,
|
|
5798
|
+
_required_snapshot_sources=graph_snapshot_sources,
|
|
5799
|
+
_expected_source_identities=repo_map_source_identities,
|
|
5800
|
+
_snapshot_rejections=graph_snapshot_rejections,
|
|
5801
|
+
)
|
|
5802
|
+
suggest_payload["estimated_pack_bytes"] = build_payload.get("pack_bytes", 0)
|
|
5803
|
+
suggest_payload["token_proxy"] = copy.deepcopy(
|
|
5804
|
+
build_payload.get("token_proxy", {})
|
|
5805
|
+
)
|
|
5806
|
+
selection_plan_payload: dict[str, Any] | None = None
|
|
5807
|
+
if plan_only or apply_plan_path:
|
|
5808
|
+
if not isinstance(self_financing_receipt, dict) or not isinstance(repo_map_payload, dict) or ordinary_build_payload is None:
|
|
5809
|
+
raise PackError("selection plan unavailable")
|
|
5810
|
+
selection_plan_payload = build_selection_plan(
|
|
5811
|
+
args, ordinary_build_payload, build_payload, self_financing_receipt,
|
|
5812
|
+
repo_map_payload, suggest_payload, repo_map_source_identities, source_cache, root_arg=root_arg,
|
|
5813
|
+
)
|
|
5814
|
+
if plan_only:
|
|
5815
|
+
return {
|
|
5816
|
+
"tool": TOOL_NAME, "schema_version": AUTO_SCHEMA_VERSION,
|
|
5817
|
+
"version": VERSION, "mode": "selection_plan",
|
|
5818
|
+
"selection_plan": selection_plan_payload,
|
|
5819
|
+
}, rc
|
|
5820
|
+
if expected_plan != selection_plan_payload:
|
|
5821
|
+
raise PackError("selection plan drift; regenerate the plan")
|
|
3939
5822
|
if not args.no_artifact:
|
|
3940
5823
|
receipt_rel = Path(PACK_DIR) / f"{build_payload['pack_id']}.json"
|
|
3941
5824
|
if manifest_rel is not None:
|
|
@@ -3982,7 +5865,7 @@ def auto_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tuple[d
|
|
|
3982
5865
|
"suggest": suggest_payload,
|
|
3983
5866
|
"build": build_payload,
|
|
3984
5867
|
"sources": {
|
|
3985
|
-
"suggested": len(
|
|
5868
|
+
"suggested": len(manifest.get("sources", [])),
|
|
3986
5869
|
"included": build_payload.get("sources", {}).get("included", 0),
|
|
3987
5870
|
"partial": build_payload.get("sources", {}).get("partial", 0),
|
|
3988
5871
|
"omitted": build_payload.get("sources", {}).get("omitted", 0),
|
|
@@ -3996,13 +5879,26 @@ def auto_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tuple[d
|
|
|
3996
5879
|
}
|
|
3997
5880
|
if build_hint_omitted_reason:
|
|
3998
5881
|
payload["build_hint_omitted_reason"] = build_hint_omitted_reason
|
|
3999
|
-
if getattr(args, "adaptive_k", False) and isinstance(
|
|
5882
|
+
if (getattr(args, "adaptive_k", False) or apply_adaptive_k) and isinstance(
|
|
5883
|
+
suggest_payload.get("adaptive_k"), dict
|
|
5884
|
+
):
|
|
4000
5885
|
payload["adaptive_k"] = copy.deepcopy(suggest_payload["adaptive_k"])
|
|
4001
|
-
|
|
4002
|
-
|
|
4003
|
-
|
|
4004
|
-
|
|
4005
|
-
|
|
5886
|
+
if adaptive_k_application is not None:
|
|
5887
|
+
payload["adaptive_k_application"] = adaptive_k_application
|
|
5888
|
+
if graph_application is not None:
|
|
5889
|
+
payload["graph_application"] = graph_application
|
|
5890
|
+
if self_financing_receipt is not None:
|
|
5891
|
+
payload["self_financing_selection"] = self_financing_receipt
|
|
5892
|
+
if selection_plan_payload is not None:
|
|
5893
|
+
payload["selection_plan"] = selection_plan_payload
|
|
5894
|
+
payload["selection_plan_application"] = {
|
|
5895
|
+
"status": "applied", "explicit": True,
|
|
5896
|
+
"revalidated_plan_id": selection_plan_payload["plan_id"],
|
|
5897
|
+
}
|
|
5898
|
+
if (getattr(args, "symbol_memory", False) or apply_symbol_memory) and isinstance(repo_map_payload, dict):
|
|
5899
|
+
payload["symbol_memory"] = build_symbol_memory_payload(
|
|
5900
|
+
repo_map_payload, applied=apply_symbol_memory
|
|
5901
|
+
)
|
|
4006
5902
|
if args.explain:
|
|
4007
5903
|
payload["explain"] = build_auto_explain_payload(
|
|
4008
5904
|
args,
|
|
@@ -4038,6 +5934,8 @@ def print_adaptive_k_text(payload: dict[str, Any]) -> None:
|
|
|
4038
5934
|
reason_text = ",".join(str(item) for item in reason_codes[:5])
|
|
4039
5935
|
else:
|
|
4040
5936
|
reason_text = str(reason_codes)
|
|
5937
|
+
application = payload.get("adaptive_k_application")
|
|
5938
|
+
applied = isinstance(application, dict) and application.get("status") == "applied"
|
|
4041
5939
|
print(
|
|
4042
5940
|
"adaptive-k: "
|
|
4043
5941
|
f"recommended={adaptive.get('recommended_k', 0)}/{adaptive.get('requested_top', 0)} "
|
|
@@ -4045,7 +5943,7 @@ def print_adaptive_k_text(payload: dict[str, Any]) -> None:
|
|
|
4045
5943
|
f"gates={regression_gates.get('status', 'pass')} "
|
|
4046
5944
|
f"candidates={score_distribution.get('candidate_count', 0)} "
|
|
4047
5945
|
f"budget_limited={budget_fit.get('budget_limited', False)} "
|
|
4048
|
-
f"apply=
|
|
5946
|
+
f"apply={str(applied).lower()} reasons={reason_text or 'none'}"
|
|
4049
5947
|
)
|
|
4050
5948
|
|
|
4051
5949
|
|
|
@@ -4132,6 +6030,19 @@ def print_auto_text(payload: dict[str, Any]) -> None:
|
|
|
4132
6030
|
print(f"omitted reasons: {reason_text}")
|
|
4133
6031
|
print_adaptive_k_text(payload)
|
|
4134
6032
|
print_symbol_memory_text(payload)
|
|
6033
|
+
self_financing = payload.get("self_financing_selection")
|
|
6034
|
+
if isinstance(self_financing, dict):
|
|
6035
|
+
phases = self_financing.get("phase_results", {})
|
|
6036
|
+
phase_text = ",".join(
|
|
6037
|
+
f"{name}={phases.get(name, {}).get('status', 'no_op')}"
|
|
6038
|
+
for name in ("adaptive", "symbol", "graph")
|
|
6039
|
+
)
|
|
6040
|
+
print(
|
|
6041
|
+
"self-financing: "
|
|
6042
|
+
f"ceiling={self_financing.get('ordinary_pack_bytes', 0)} "
|
|
6043
|
+
f"selected={self_financing.get('selected_rendered_bytes', 0)} "
|
|
6044
|
+
f"{phase_text} provider_savings_claim=false"
|
|
6045
|
+
)
|
|
4135
6046
|
if payload.get("manifest_path"):
|
|
4136
6047
|
print(f"manifest: {payload['manifest_path']}")
|
|
4137
6048
|
if payload.get("pack_path"):
|
|
@@ -4224,10 +6135,43 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
4224
6135
|
)
|
|
4225
6136
|
auto.add_argument("--explain", action="store_true", help="include deterministic local selection/build explanation metadata")
|
|
4226
6137
|
auto.add_argument("--adaptive-k", action="store_true", help="include local score/budget top-k advisory metadata without changing the manifest or pack")
|
|
6138
|
+
auto.add_argument(
|
|
6139
|
+
"--apply-adaptive-k",
|
|
6140
|
+
action="store_true",
|
|
6141
|
+
help=(
|
|
6142
|
+
"explicitly prune heuristic-selected sources to the locally recommended top-k "
|
|
6143
|
+
"after regression gates pass while always retaining explicit file/output/diff sources; "
|
|
6144
|
+
"implies --adaptive-k"
|
|
6145
|
+
),
|
|
6146
|
+
)
|
|
4227
6147
|
auto.add_argument("--adaptive-k-policy", choices=ADAPTIVE_K_POLICIES, default="balanced", help="local adaptive-k recommendation policy used when --adaptive-k is set")
|
|
4228
6148
|
auto.add_argument("--adaptive-k-min-recall-proxy", type=adaptive_k_threshold, default=0.0, help="metadata-only minimum recall proxy gate for --adaptive-k")
|
|
4229
6149
|
auto.add_argument("--adaptive-k-min-precision-proxy", type=adaptive_k_threshold, default=0.0, help="metadata-only minimum precision proxy gate for --adaptive-k")
|
|
4230
6150
|
auto.add_argument("--symbol-memory", action="store_true", help="include repo-map derived symbol/graph advisory metadata with exact source verification hints")
|
|
6151
|
+
auto.add_argument(
|
|
6152
|
+
"--apply-symbol-memory",
|
|
6153
|
+
action="store_true",
|
|
6154
|
+
help=(
|
|
6155
|
+
"explicitly add up to four direct import-neighbor slices from the local "
|
|
6156
|
+
"repo map to the manifest and pack; implies --symbol-memory"
|
|
6157
|
+
),
|
|
6158
|
+
)
|
|
6159
|
+
auto.add_argument(
|
|
6160
|
+
"--self-financing-selection",
|
|
6161
|
+
action="store_true",
|
|
6162
|
+
help=(
|
|
6163
|
+
"explicitly apply Adaptive, then Symbol, then one-hop Graph selection while replacing "
|
|
6164
|
+
"only lower-value non-caller sources and never exceeding the ordinary pack bytes"
|
|
6165
|
+
),
|
|
6166
|
+
)
|
|
6167
|
+
auto.add_argument(
|
|
6168
|
+
"--selection-plan", action="store_true",
|
|
6169
|
+
help="emit a read-only provider-free self-financing selection plan; requires --json",
|
|
6170
|
+
)
|
|
6171
|
+
auto.add_argument(
|
|
6172
|
+
"--apply-selection-plan", metavar="PLAN_JSON",
|
|
6173
|
+
help="explicitly apply a previously emitted selection plan after identity revalidation",
|
|
6174
|
+
)
|
|
4231
6175
|
return parser
|
|
4232
6176
|
|
|
4233
6177
|
|