@ictechgy/context-guard 0.4.16 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +72 -0
- package/README.ko.md +82 -1
- package/README.md +86 -1
- package/docs/distribution.md +100 -0
- package/package.json +4 -1
- package/plugins/context-guard/.claude-plugin/plugin.json +1 -1
- package/plugins/context-guard/README.ko.md +25 -1
- package/plugins/context-guard/README.md +26 -1
- package/plugins/context-guard/bin/bash_reference_policy.py +967 -0
- package/plugins/context-guard/bin/context-guard-bench +4216 -103
- package/plugins/context-guard/bin/context-guard-failed-nudge +95 -23
- package/plugins/context-guard/bin/context-guard-guard-read +6 -2
- package/plugins/context-guard/bin/context-guard-mcp +2 -1
- package/plugins/context-guard/bin/context-guard-pack +1463 -142
- package/plugins/context-guard/bin/context-guard-rewrite-bash +497 -45
- package/plugins/context-guard/bin/context-guard-sanitize-output +178 -22
- package/plugins/context-guard/bin/context-guard-setup +812 -16
- package/plugins/context-guard/bin/context-guard-statusline +33 -2
- package/plugins/context-guard/bin/context-guard-statusline-merged +71 -20
- package/plugins/context-guard/bin/context-guard-trim-output +706 -35
- package/plugins/context-guard/lib/context_guard_commands.py +4 -0
- package/plugins/context-guard/lib/credential_policy.py +10 -2
|
@@ -23,6 +23,7 @@ import posixpath
|
|
|
23
23
|
from pathlib import Path
|
|
24
24
|
import re
|
|
25
25
|
import shlex
|
|
26
|
+
import signal
|
|
26
27
|
import stat
|
|
27
28
|
import subprocess
|
|
28
29
|
import sys
|
|
@@ -48,7 +49,9 @@ AUTO_SCHEMA_VERSION = "contextguard.pack-auto.v1"
|
|
|
48
49
|
AUTO_EXPLAIN_SCHEMA_VERSION = "contextguard.pack-auto-explain.v1"
|
|
49
50
|
REPO_MAP_SCHEMA_VERSION = "contextguard.pack-repo-map.v1"
|
|
50
51
|
ADAPTIVE_K_SCHEMA_VERSION = "contextguard.pack-adaptive-k.v1"
|
|
52
|
+
ADAPTIVE_K_APPLICATION_SCHEMA_VERSION = "contextguard.pack-adaptive-k-application.v1"
|
|
51
53
|
SYMBOL_MEMORY_SCHEMA_VERSION = "contextguard.pack-symbol-memory.v1"
|
|
54
|
+
GRAPH_APPLICATION_SCHEMA_VERSION = "contextguard.pack-graph-application.v1"
|
|
52
55
|
CONTENT_ADDRESS_SCHEMA_VERSION = "contextguard.pack-content-address.v1"
|
|
53
56
|
ROLLING_DELTA_SCHEMA_VERSION = "contextguard.pack-rolling-delta.v1"
|
|
54
57
|
SKETCH_DUPLICATE_SHINGLE_WIDTH = 5
|
|
@@ -65,10 +68,24 @@ DEFAULT_SUGGEST_CONTEXT_LINES = 20
|
|
|
65
68
|
MAX_SUGGEST_CONTEXT_LINES = 120
|
|
66
69
|
SUGGEST_WHOLE_FILE_MAX_LINES = 120
|
|
67
70
|
MAX_SUGGEST_INPUT_BYTES = 256_000
|
|
71
|
+
MAX_GIT_DIFF_STDERR_BYTES = 16_000
|
|
72
|
+
GIT_DIFF_TIMEOUT_SECONDS = 10.0
|
|
68
73
|
MAX_QUERY_SCAN_FILES = 2_000
|
|
69
74
|
MAX_QUERY_SCAN_BYTES_PER_FILE = 200_000
|
|
70
75
|
MAX_GIT_LS_FILES_OUTPUT_BYTES = MAX_QUERY_SCAN_FILES * 512
|
|
71
76
|
GIT_LS_FILES_READ_CHUNK_BYTES = 64 * 1024
|
|
77
|
+
MAX_GIT_ATTR_INPUT_BYTES = MAX_GIT_LS_FILES_OUTPUT_BYTES
|
|
78
|
+
MAX_GIT_ATTR_OUTPUT_BYTES = MAX_GIT_ATTR_INPUT_BYTES * 2
|
|
79
|
+
GIT_ATTR_TIMEOUT_SECONDS = 10.0
|
|
80
|
+
MAX_QUERY_WALK_DIRS = 2_000
|
|
81
|
+
MAX_QUERY_WALK_ENTRIES = 10_000
|
|
82
|
+
MAX_QUERY_WALK_DEPTH = 32
|
|
83
|
+
MAX_QUERY_WALK_SECONDS = 2.0
|
|
84
|
+
MAX_SOURCE_INPUT_BYTES = 4_000_000
|
|
85
|
+
MAX_SOURCE_INPUT_LINES = 100_000
|
|
86
|
+
MAX_SOURCE_LINE_BYTES = 256_000
|
|
87
|
+
MAX_TOTAL_SOURCE_INPUT_BYTES = 16_000_000
|
|
88
|
+
MAX_TOTAL_SOURCE_INPUT_LINES = 400_000
|
|
72
89
|
MAX_REPO_MAP_FILES = 1_000
|
|
73
90
|
MAX_REPO_MAP_SCAN_FILES = 160
|
|
74
91
|
MAX_REPO_MAP_BYTES_PER_FILE = 120_000
|
|
@@ -85,6 +102,8 @@ MAX_ADAPTIVE_K_VERIFICATION_HINTS = 12
|
|
|
85
102
|
ADAPTIVE_K_POLICIES = ("balanced", "recall", "precision")
|
|
86
103
|
MAX_SYMBOL_MEMORY_ITEMS = 12
|
|
87
104
|
MAX_SYMBOL_MEMORY_GRAPH_ITEMS = 12
|
|
105
|
+
MAX_GRAPH_APPLICATION_SOURCES = 4
|
|
106
|
+
MAX_GRAPH_APPLICATION_LINES = 80
|
|
88
107
|
PACK_DIR = ".context-guard/packs"
|
|
89
108
|
REDACTED_PATH_COMPONENT = "[REDACTED-PATH-COMPONENT]"
|
|
90
109
|
ALLOWED_FIRST_ABSOLUTE_SYMLINKS = {
|
|
@@ -178,6 +197,97 @@ class ResolvedSource:
|
|
|
178
197
|
selected_lines: list[str]
|
|
179
198
|
total_lines: int
|
|
180
199
|
redacted_lines: int
|
|
200
|
+
total_lines_exact: bool = True
|
|
201
|
+
input_bytes_read: int = 0
|
|
202
|
+
input_lines_read: int = 0
|
|
203
|
+
sanitized_through_line: int = 0
|
|
204
|
+
input_limit_reason: str | None = None
|
|
205
|
+
redacted_lines_exact: bool = True
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
@dataclass(frozen=True)
|
|
209
|
+
class _SourceScanResult:
|
|
210
|
+
selected_lines: tuple[str, ...]
|
|
211
|
+
total_lines: int
|
|
212
|
+
redacted_lines: int
|
|
213
|
+
total_lines_exact: bool
|
|
214
|
+
input_bytes_read: int
|
|
215
|
+
input_lines_read: int
|
|
216
|
+
sanitized_through_line: int
|
|
217
|
+
limit_reason: str | None
|
|
218
|
+
selection_complete: bool
|
|
219
|
+
redacted_lines_exact: bool
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
@dataclass(frozen=True)
|
|
223
|
+
class _SourceSnapshot:
|
|
224
|
+
identity: tuple[int, int, int, int, int, int, int, int]
|
|
225
|
+
display_path: str
|
|
226
|
+
redacted_path: bool
|
|
227
|
+
requested_lines: LineRange
|
|
228
|
+
selected_lines: tuple[str, ...]
|
|
229
|
+
total_lines: int
|
|
230
|
+
redacted_lines: int
|
|
231
|
+
total_lines_exact: bool
|
|
232
|
+
input_bytes_read: int
|
|
233
|
+
input_lines_read: int
|
|
234
|
+
sanitized_through_line: int
|
|
235
|
+
input_limit_reason: str | None
|
|
236
|
+
redacted_lines_exact: bool
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
class _SourceInputBudget:
|
|
240
|
+
def __init__(self) -> None:
|
|
241
|
+
self.bytes_read = 0
|
|
242
|
+
self.lines_read = 0
|
|
243
|
+
self.bytes_attempted = 0
|
|
244
|
+
self.lines_attempted = 0
|
|
245
|
+
self.bytes_charged = 0
|
|
246
|
+
self.lines_charged = 0
|
|
247
|
+
self.capped = (
|
|
248
|
+
MAX_TOTAL_SOURCE_INPUT_BYTES <= 0
|
|
249
|
+
or MAX_TOTAL_SOURCE_INPUT_LINES <= 0
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
def remaining_bytes(self) -> int:
|
|
253
|
+
return max(0, MAX_TOTAL_SOURCE_INPUT_BYTES - self.bytes_charged)
|
|
254
|
+
|
|
255
|
+
def remaining_lines(self) -> int:
|
|
256
|
+
return max(0, MAX_TOTAL_SOURCE_INPUT_LINES - self.lines_charged)
|
|
257
|
+
|
|
258
|
+
def record_read(self, bytes_count: int) -> str | None:
|
|
259
|
+
bytes_remaining = self.remaining_bytes()
|
|
260
|
+
lines_remaining = self.remaining_lines()
|
|
261
|
+
self.bytes_read += bytes_count
|
|
262
|
+
self.lines_read += 1
|
|
263
|
+
self.bytes_attempted += bytes_count
|
|
264
|
+
self.lines_attempted += 1
|
|
265
|
+
self.bytes_charged = min(
|
|
266
|
+
MAX_TOTAL_SOURCE_INPUT_BYTES,
|
|
267
|
+
self.bytes_charged + bytes_count,
|
|
268
|
+
)
|
|
269
|
+
self.lines_charged = min(
|
|
270
|
+
MAX_TOTAL_SOURCE_INPUT_LINES,
|
|
271
|
+
self.lines_charged + 1,
|
|
272
|
+
)
|
|
273
|
+
self.capped = (
|
|
274
|
+
self.bytes_charged >= MAX_TOTAL_SOURCE_INPUT_BYTES
|
|
275
|
+
or self.lines_charged >= MAX_TOTAL_SOURCE_INPUT_LINES
|
|
276
|
+
)
|
|
277
|
+
if lines_remaining <= 0:
|
|
278
|
+
return "cumulative_input_lines_exceeded"
|
|
279
|
+
if bytes_count > bytes_remaining:
|
|
280
|
+
return "cumulative_input_bytes_exceeded"
|
|
281
|
+
return None
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
class _SourceSnapshotCache:
|
|
285
|
+
def __init__(self) -> None:
|
|
286
|
+
self.entries: dict[tuple[str, str, str], _SourceSnapshot] = {}
|
|
287
|
+
|
|
288
|
+
@staticmethod
|
|
289
|
+
def key(rel: Path, requested: LineRange | None, context: str) -> tuple[str, str, str]:
|
|
290
|
+
return (rel.as_posix(), requested.identity() if requested is not None else "all", context)
|
|
181
291
|
|
|
182
292
|
|
|
183
293
|
@dataclass
|
|
@@ -354,12 +464,33 @@ def sanitize_source_lines(
|
|
|
354
464
|
context: str = "source_code",
|
|
355
465
|
private_roots: tuple[str, ...] = (),
|
|
356
466
|
) -> tuple[list[str], int, int]:
|
|
357
|
-
"""
|
|
467
|
+
"""Compatibility wrapper for the bounded source scanner."""
|
|
468
|
+
scan = _scan_source_lines(
|
|
469
|
+
handle,
|
|
470
|
+
requested,
|
|
471
|
+
context=context,
|
|
472
|
+
private_roots=private_roots,
|
|
473
|
+
input_budget=_SourceInputBudget(),
|
|
474
|
+
)
|
|
475
|
+
return list(scan.selected_lines), scan.total_lines, scan.redacted_lines
|
|
476
|
+
|
|
358
477
|
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
478
|
+
def _scan_source_lines(
|
|
479
|
+
handle: Any,
|
|
480
|
+
requested: LineRange | None,
|
|
481
|
+
*,
|
|
482
|
+
context: str,
|
|
483
|
+
private_roots: tuple[str, ...],
|
|
484
|
+
input_budget: _SourceInputBudget,
|
|
485
|
+
expected_size_bytes: int | None = None,
|
|
486
|
+
) -> _SourceScanResult:
|
|
487
|
+
"""Read with byte/line caps and sanitize only the required prefix.
|
|
488
|
+
|
|
489
|
+
Stateful sanitizers still see every line through ``requested.end``. The
|
|
490
|
+
remaining tail is counted without invoking the sanitizer so range requests
|
|
491
|
+
do not pay sanitizer cost for irrelevant content. If counting reaches a
|
|
492
|
+
cap, the selected range remains usable and the total is explicitly marked
|
|
493
|
+
as a lower bound.
|
|
363
494
|
"""
|
|
364
495
|
sanitizer = load_line_sanitizer(
|
|
365
496
|
context=context,
|
|
@@ -368,16 +499,95 @@ def sanitize_source_lines(
|
|
|
368
499
|
selected: list[str] = []
|
|
369
500
|
redacted = 0
|
|
370
501
|
total_lines = 0
|
|
502
|
+
input_bytes = 0
|
|
503
|
+
input_lines = 0
|
|
371
504
|
collect_all = requested is None
|
|
372
505
|
start = requested.start if requested is not None else 1
|
|
373
506
|
end = requested.end if requested is not None else 0
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
507
|
+
total_lines_exact = False
|
|
508
|
+
limit_reason: str | None = None
|
|
509
|
+
redacted_lines_exact = True
|
|
510
|
+
iterator = None if callable(getattr(handle, "readline", None)) else iter(handle)
|
|
511
|
+
|
|
512
|
+
while True:
|
|
513
|
+
boundary_reason: str | None = None
|
|
514
|
+
if total_lines >= MAX_SOURCE_INPUT_LINES:
|
|
515
|
+
boundary_reason = "source_input_lines_exceeded"
|
|
516
|
+
elif input_budget.remaining_lines() <= 0:
|
|
517
|
+
boundary_reason = "cumulative_input_lines_exceeded"
|
|
518
|
+
source_remaining = MAX_SOURCE_INPUT_BYTES - input_bytes
|
|
519
|
+
cumulative_remaining = input_budget.remaining_bytes()
|
|
520
|
+
if boundary_reason is None and source_remaining <= 0:
|
|
521
|
+
boundary_reason = "source_input_bytes_exceeded"
|
|
522
|
+
elif boundary_reason is None and cumulative_remaining <= 0:
|
|
523
|
+
boundary_reason = "cumulative_input_bytes_exceeded"
|
|
524
|
+
if boundary_reason is not None:
|
|
525
|
+
if expected_size_bytes is not None:
|
|
526
|
+
try:
|
|
527
|
+
if handle.tell() == expected_size_bytes:
|
|
528
|
+
total_lines_exact = True
|
|
529
|
+
break
|
|
530
|
+
except (AttributeError, OSError):
|
|
531
|
+
pass
|
|
532
|
+
limit_reason = boundary_reason
|
|
533
|
+
break
|
|
534
|
+
read_char_cap = min(MAX_SOURCE_LINE_BYTES, source_remaining, cumulative_remaining)
|
|
535
|
+
try:
|
|
536
|
+
if iterator is None:
|
|
537
|
+
raw_line = handle.readline(read_char_cap + 1)
|
|
538
|
+
else:
|
|
539
|
+
raw_line = next(iterator, "")
|
|
540
|
+
except (OSError, UnicodeError):
|
|
541
|
+
limit_reason = "unsafe_path"
|
|
542
|
+
break
|
|
543
|
+
if raw_line == "":
|
|
544
|
+
total_lines_exact = True
|
|
545
|
+
break
|
|
546
|
+
raw_bytes = byte_len(raw_line)
|
|
547
|
+
cumulative_reason = input_budget.record_read(raw_bytes)
|
|
548
|
+
input_bytes += raw_bytes
|
|
549
|
+
input_lines += 1
|
|
550
|
+
if len(raw_line) > MAX_SOURCE_LINE_BYTES or raw_bytes > MAX_SOURCE_LINE_BYTES:
|
|
551
|
+
limit_reason = "source_line_bytes_exceeded"
|
|
552
|
+
break
|
|
553
|
+
if raw_bytes > source_remaining:
|
|
554
|
+
limit_reason = "source_input_bytes_exceeded"
|
|
555
|
+
break
|
|
556
|
+
if cumulative_reason is not None:
|
|
557
|
+
limit_reason = cumulative_reason
|
|
558
|
+
break
|
|
559
|
+
total_lines += 1
|
|
560
|
+
|
|
561
|
+
must_sanitize = collect_all or total_lines <= end
|
|
562
|
+
if must_sanitize:
|
|
563
|
+
sanitized, did_redact = sanitizer.sanitize(raw_line) # type: ignore[attr-defined]
|
|
564
|
+
if did_redact:
|
|
565
|
+
redacted += 1
|
|
566
|
+
if collect_all or start <= total_lines <= end:
|
|
567
|
+
selected.append(sanitized)
|
|
568
|
+
else:
|
|
569
|
+
redacted_lines_exact = False
|
|
570
|
+
if SECRET_CONTENT_RE.search(raw_line):
|
|
571
|
+
redacted += 1
|
|
572
|
+
|
|
573
|
+
selection_complete = (
|
|
574
|
+
total_lines_exact
|
|
575
|
+
if collect_all
|
|
576
|
+
else total_lines >= end or (total_lines_exact and total_lines >= start)
|
|
577
|
+
)
|
|
578
|
+
sanitized_through = total_lines if collect_all else min(total_lines, end)
|
|
579
|
+
return _SourceScanResult(
|
|
580
|
+
selected_lines=tuple(selected),
|
|
581
|
+
total_lines=total_lines,
|
|
582
|
+
redacted_lines=redacted,
|
|
583
|
+
total_lines_exact=total_lines_exact,
|
|
584
|
+
input_bytes_read=input_bytes,
|
|
585
|
+
input_lines_read=input_lines,
|
|
586
|
+
sanitized_through_line=sanitized_through,
|
|
587
|
+
limit_reason=limit_reason,
|
|
588
|
+
selection_complete=selection_complete,
|
|
589
|
+
redacted_lines_exact=redacted_lines_exact,
|
|
590
|
+
)
|
|
381
591
|
|
|
382
592
|
|
|
383
593
|
def byte_len(text: str) -> int:
|
|
@@ -1165,7 +1375,69 @@ def open_regular_under_root(root: Path, rel: Path) -> tuple[Any | None, str]:
|
|
|
1165
1375
|
return None, "unsafe_path"
|
|
1166
1376
|
|
|
1167
1377
|
|
|
1168
|
-
def
|
|
1378
|
+
def _open_source_identity(handle: Any) -> tuple[int, int, int, int, int, int, int, int] | None:
|
|
1379
|
+
try:
|
|
1380
|
+
st = os.fstat(handle.fileno())
|
|
1381
|
+
except (AttributeError, OSError):
|
|
1382
|
+
return None
|
|
1383
|
+
return (
|
|
1384
|
+
st.st_dev,
|
|
1385
|
+
st.st_ino,
|
|
1386
|
+
st.st_mode,
|
|
1387
|
+
st.st_uid,
|
|
1388
|
+
st.st_nlink,
|
|
1389
|
+
st.st_size,
|
|
1390
|
+
int(getattr(st, "st_mtime_ns", int(st.st_mtime * 1_000_000_000))),
|
|
1391
|
+
int(getattr(st, "st_ctime_ns", int(st.st_ctime * 1_000_000_000))),
|
|
1392
|
+
)
|
|
1393
|
+
|
|
1394
|
+
|
|
1395
|
+
def _input_limit_metadata(reason: str) -> dict[str, Any]:
|
|
1396
|
+
caps = {
|
|
1397
|
+
"source_line_bytes_exceeded": ("source_line_bytes", MAX_SOURCE_LINE_BYTES),
|
|
1398
|
+
"source_input_bytes_exceeded": ("source_bytes", MAX_SOURCE_INPUT_BYTES),
|
|
1399
|
+
"source_input_lines_exceeded": ("source_lines", MAX_SOURCE_INPUT_LINES),
|
|
1400
|
+
"cumulative_input_bytes_exceeded": ("cumulative_bytes", MAX_TOTAL_SOURCE_INPUT_BYTES),
|
|
1401
|
+
"cumulative_input_lines_exceeded": ("cumulative_lines", MAX_TOTAL_SOURCE_INPUT_LINES),
|
|
1402
|
+
}
|
|
1403
|
+
kind, cap = caps.get(reason, ("unknown", 0))
|
|
1404
|
+
return {"kind": kind, "cap_bytes" if "bytes" in kind else "cap_lines": cap}
|
|
1405
|
+
|
|
1406
|
+
|
|
1407
|
+
def _snapshot_to_source(
|
|
1408
|
+
snapshot: _SourceSnapshot,
|
|
1409
|
+
*,
|
|
1410
|
+
root: Path,
|
|
1411
|
+
rel: Path,
|
|
1412
|
+
spec: SourceSpec,
|
|
1413
|
+
) -> ResolvedSource:
|
|
1414
|
+
return ResolvedSource(
|
|
1415
|
+
spec=spec,
|
|
1416
|
+
abs_path=root / rel,
|
|
1417
|
+
display_path=snapshot.display_path,
|
|
1418
|
+
redacted_path=snapshot.redacted_path,
|
|
1419
|
+
requested_lines=spec.lines or snapshot.requested_lines,
|
|
1420
|
+
selected_lines=list(snapshot.selected_lines),
|
|
1421
|
+
total_lines=snapshot.total_lines,
|
|
1422
|
+
redacted_lines=snapshot.redacted_lines,
|
|
1423
|
+
total_lines_exact=snapshot.total_lines_exact,
|
|
1424
|
+
input_bytes_read=snapshot.input_bytes_read,
|
|
1425
|
+
input_lines_read=snapshot.input_lines_read,
|
|
1426
|
+
sanitized_through_line=snapshot.sanitized_through_line,
|
|
1427
|
+
input_limit_reason=snapshot.input_limit_reason,
|
|
1428
|
+
redacted_lines_exact=snapshot.redacted_lines_exact,
|
|
1429
|
+
)
|
|
1430
|
+
|
|
1431
|
+
|
|
1432
|
+
def resolve_source(
|
|
1433
|
+
root: Path,
|
|
1434
|
+
spec: SourceSpec,
|
|
1435
|
+
*,
|
|
1436
|
+
source_cache: _SourceSnapshotCache | None = None,
|
|
1437
|
+
input_budget: _SourceInputBudget | None = None,
|
|
1438
|
+
expected_identity: tuple[int, int, int, int, int, int, int, int] | None = None,
|
|
1439
|
+
require_cached: bool = False,
|
|
1440
|
+
) -> tuple[ResolvedSource | None, dict[str, Any] | None]:
|
|
1169
1441
|
if spec.lines is not None and spec.lines.start < 1:
|
|
1170
1442
|
return None, omission(spec, "invalid_lines")
|
|
1171
1443
|
rel, reason = lexical_rel(spec.path)
|
|
@@ -1175,10 +1447,45 @@ def resolve_source(root: Path, spec: SourceSpec) -> tuple[ResolvedSource | None,
|
|
|
1175
1447
|
handle, reason = open_regular_under_root(root, rel)
|
|
1176
1448
|
if handle is None:
|
|
1177
1449
|
return None, omission(spec, reason, path=display, redacted_path=redacted_path)
|
|
1450
|
+
scan_budget = input_budget if input_budget is not None else _SourceInputBudget()
|
|
1451
|
+
requested = spec.lines
|
|
1452
|
+
cache_key = _SourceSnapshotCache.key(rel, requested, spec.sanitization_context)
|
|
1178
1453
|
try:
|
|
1179
1454
|
with handle:
|
|
1180
|
-
|
|
1181
|
-
|
|
1455
|
+
before_identity = _open_source_identity(handle)
|
|
1456
|
+
if expected_identity is not None and before_identity != expected_identity:
|
|
1457
|
+
return None, omission(
|
|
1458
|
+
spec,
|
|
1459
|
+
"graph_source_changed_since_repo_map_snapshot",
|
|
1460
|
+
path=display,
|
|
1461
|
+
redacted_path=redacted_path,
|
|
1462
|
+
)
|
|
1463
|
+
cached = source_cache.entries.get(cache_key) if source_cache is not None else None
|
|
1464
|
+
if require_cached and cached is None:
|
|
1465
|
+
return None, omission(
|
|
1466
|
+
spec,
|
|
1467
|
+
"graph_source_snapshot_unavailable",
|
|
1468
|
+
path=display,
|
|
1469
|
+
redacted_path=redacted_path,
|
|
1470
|
+
)
|
|
1471
|
+
if cached is not None:
|
|
1472
|
+
if before_identity != cached.identity:
|
|
1473
|
+
return None, omission(
|
|
1474
|
+
spec,
|
|
1475
|
+
"source_changed_during_auto",
|
|
1476
|
+
path=display,
|
|
1477
|
+
redacted_path=redacted_path,
|
|
1478
|
+
)
|
|
1479
|
+
source = _snapshot_to_source(cached, root=root, rel=rel, spec=spec)
|
|
1480
|
+
if _open_source_identity(handle) != before_identity:
|
|
1481
|
+
return None, omission(
|
|
1482
|
+
spec,
|
|
1483
|
+
"source_changed_during_auto",
|
|
1484
|
+
path=display,
|
|
1485
|
+
redacted_path=redacted_path,
|
|
1486
|
+
)
|
|
1487
|
+
return source, None
|
|
1488
|
+
scan = _scan_source_lines(
|
|
1182
1489
|
handle,
|
|
1183
1490
|
requested,
|
|
1184
1491
|
context=spec.sanitization_context,
|
|
@@ -1187,17 +1494,38 @@ def resolve_source(root: Path, spec: SourceSpec) -> tuple[ResolvedSource | None,
|
|
|
1187
1494
|
if spec.sanitization_context == "filesystem_listing"
|
|
1188
1495
|
else ()
|
|
1189
1496
|
),
|
|
1497
|
+
input_budget=scan_budget,
|
|
1498
|
+
expected_size_bytes=before_identity[5] if before_identity is not None else None,
|
|
1190
1499
|
)
|
|
1500
|
+
after_identity = _open_source_identity(handle)
|
|
1191
1501
|
except OSError:
|
|
1192
1502
|
return None, omission(spec, "unsafe_path", path=display, redacted_path=redacted_path)
|
|
1503
|
+
if before_identity is not None and after_identity != before_identity:
|
|
1504
|
+
return None, omission(spec, "source_changed_during_read", path=display, redacted_path=redacted_path)
|
|
1505
|
+
if scan.limit_reason == "unsafe_path":
|
|
1506
|
+
return None, omission(spec, "unsafe_path", path=display, redacted_path=redacted_path)
|
|
1507
|
+
if scan.limit_reason is not None and not scan.selection_complete:
|
|
1508
|
+
item = omission(spec, scan.limit_reason, path=display, redacted_path=redacted_path)
|
|
1509
|
+
item["input_limit"] = _input_limit_metadata(scan.limit_reason)
|
|
1510
|
+
item["input_observed"] = {
|
|
1511
|
+
"bytes": scan.input_bytes_read,
|
|
1512
|
+
"lines": scan.input_lines_read,
|
|
1513
|
+
"bytes_attempted": scan.input_bytes_read,
|
|
1514
|
+
"lines_attempted": scan.input_lines_read,
|
|
1515
|
+
"capped": True,
|
|
1516
|
+
}
|
|
1517
|
+
return None, item
|
|
1518
|
+
selected = list(scan.selected_lines)
|
|
1519
|
+
total_lines = scan.total_lines
|
|
1520
|
+
redacted_lines = scan.redacted_lines
|
|
1193
1521
|
if total_lines <= 0:
|
|
1194
1522
|
return None, omission(spec, "empty_source", path=display, redacted_path=redacted_path)
|
|
1195
1523
|
requested = requested or LineRange(1, total_lines)
|
|
1196
|
-
if requested.start > total_lines:
|
|
1524
|
+
if scan.total_lines_exact and requested.start > total_lines:
|
|
1197
1525
|
return None, omission(spec, "empty_source", path=display, redacted_path=redacted_path)
|
|
1198
1526
|
if not selected:
|
|
1199
1527
|
return None, omission(spec, "empty_source", path=display, redacted_path=redacted_path)
|
|
1200
|
-
|
|
1528
|
+
source = ResolvedSource(
|
|
1201
1529
|
spec=spec,
|
|
1202
1530
|
abs_path=root / rel,
|
|
1203
1531
|
display_path=display,
|
|
@@ -1206,7 +1534,33 @@ def resolve_source(root: Path, spec: SourceSpec) -> tuple[ResolvedSource | None,
|
|
|
1206
1534
|
selected_lines=selected,
|
|
1207
1535
|
total_lines=total_lines,
|
|
1208
1536
|
redacted_lines=redacted_lines,
|
|
1209
|
-
|
|
1537
|
+
total_lines_exact=scan.total_lines_exact,
|
|
1538
|
+
input_bytes_read=scan.input_bytes_read,
|
|
1539
|
+
input_lines_read=scan.input_lines_read,
|
|
1540
|
+
sanitized_through_line=scan.sanitized_through_line,
|
|
1541
|
+
input_limit_reason=scan.limit_reason,
|
|
1542
|
+
redacted_lines_exact=scan.redacted_lines_exact,
|
|
1543
|
+
)
|
|
1544
|
+
if source_cache is not None and before_identity is not None:
|
|
1545
|
+
snapshot = _SourceSnapshot(
|
|
1546
|
+
identity=before_identity,
|
|
1547
|
+
display_path=display,
|
|
1548
|
+
redacted_path=redacted_path,
|
|
1549
|
+
requested_lines=requested,
|
|
1550
|
+
selected_lines=tuple(selected),
|
|
1551
|
+
total_lines=total_lines,
|
|
1552
|
+
redacted_lines=redacted_lines,
|
|
1553
|
+
total_lines_exact=scan.total_lines_exact,
|
|
1554
|
+
input_bytes_read=scan.input_bytes_read,
|
|
1555
|
+
input_lines_read=scan.input_lines_read,
|
|
1556
|
+
sanitized_through_line=scan.sanitized_through_line,
|
|
1557
|
+
input_limit_reason=scan.limit_reason,
|
|
1558
|
+
redacted_lines_exact=scan.redacted_lines_exact,
|
|
1559
|
+
)
|
|
1560
|
+
source_cache.entries[cache_key] = snapshot
|
|
1561
|
+
canonical_key = _SourceSnapshotCache.key(rel, source_selected_range(source), spec.sanitization_context)
|
|
1562
|
+
source_cache.entries[canonical_key] = snapshot
|
|
1563
|
+
return source, None
|
|
1210
1564
|
|
|
1211
1565
|
|
|
1212
1566
|
def retrieval_cli(root_arg: str, display_path: str, lines: LineRange) -> str:
|
|
@@ -1245,31 +1599,99 @@ def retrieval_for(root_arg: str, display_path: str, lines: LineRange, *, redacte
|
|
|
1245
1599
|
return retrieval_cli(safe_root, display_path, lines), None
|
|
1246
1600
|
|
|
1247
1601
|
|
|
1248
|
-
|
|
1249
|
-
|
|
1602
|
+
def markdown_metadata_text(value: object) -> str:
|
|
1603
|
+
out: list[str] = []
|
|
1604
|
+
for char in str(value):
|
|
1605
|
+
code = ord(char)
|
|
1606
|
+
if not char.isprintable():
|
|
1607
|
+
out.append(f"\\u{code:04X}" if code <= 0xFFFF else f"\\U{code:08X}")
|
|
1608
|
+
elif char == "&":
|
|
1609
|
+
out.append("&")
|
|
1610
|
+
elif char == "<":
|
|
1611
|
+
out.append("<")
|
|
1612
|
+
elif char == ">":
|
|
1613
|
+
out.append(">")
|
|
1614
|
+
elif char in {"[", "]", "(", ")", "!"}:
|
|
1615
|
+
out.append("\\" + char)
|
|
1616
|
+
elif char in {"`", "\\"}:
|
|
1617
|
+
out.append("\\" + char)
|
|
1618
|
+
else:
|
|
1619
|
+
out.append(char)
|
|
1620
|
+
return "".join(out)
|
|
1621
|
+
|
|
1622
|
+
|
|
1623
|
+
def markdown_inline_code(value: object) -> str:
|
|
1624
|
+
text = "".join(
|
|
1625
|
+
(f"\\u{ord(char):04X}" if ord(char) <= 0xFFFF else f"\\U{ord(char):08X}")
|
|
1626
|
+
if not char.isprintable()
|
|
1627
|
+
else char
|
|
1628
|
+
for char in str(value)
|
|
1629
|
+
)
|
|
1630
|
+
max_run = max((len(match.group(0)) for match in re.finditer(r"`+", text)), default=0)
|
|
1631
|
+
delimiter = "`" * max(1, max_run + 1)
|
|
1632
|
+
padding = " " if text.startswith("`") or text.endswith("`") else ""
|
|
1633
|
+
return f"{delimiter}{padding}{text}{padding}{delimiter}"
|
|
1634
|
+
|
|
1635
|
+
|
|
1636
|
+
def markdown_block_delimiters(lines: list[str]) -> tuple[str, str]:
|
|
1637
|
+
max_run = 0
|
|
1638
|
+
for line in lines:
|
|
1639
|
+
line_max = max((len(match.group(0)) for match in re.finditer(r"`+", line)), default=0)
|
|
1640
|
+
max_run = max(max_run, line_max)
|
|
1641
|
+
fence = "`" * max(3, max_run + 1)
|
|
1642
|
+
return f"\n\n{fence}text\n", f"{fence}\n\n"
|
|
1250
1643
|
|
|
1251
1644
|
|
|
1252
1645
|
def render_block_header(source: ResolvedSource, *, root_arg: str, status: str, included: LineRange) -> str:
|
|
1253
|
-
title = source.spec.label or source.display_path
|
|
1646
|
+
title = markdown_metadata_text(source.spec.label or source.display_path)
|
|
1254
1647
|
requested = source.requested_lines or LineRange(1, source.total_lines)
|
|
1255
1648
|
retrieval, retrieval_omitted_reason = retrieval_for(root_arg, source.display_path, included, redacted_path=source.redacted_path)
|
|
1256
1649
|
header = [
|
|
1257
1650
|
f"## {title}",
|
|
1258
|
-
f"Source:
|
|
1651
|
+
f"Source: {markdown_inline_code(source.display_path)}",
|
|
1259
1652
|
f"Priority: {source.spec.priority}",
|
|
1260
1653
|
f"Status: {status}",
|
|
1261
1654
|
f"Included lines: {included.start}:{included.end}",
|
|
1262
1655
|
f"Requested lines: {requested.start}:{requested.end}",
|
|
1263
1656
|
]
|
|
1264
1657
|
if retrieval:
|
|
1265
|
-
header.append(f"Retrieval:
|
|
1658
|
+
header.append(f"Retrieval: {markdown_inline_code(retrieval)}")
|
|
1266
1659
|
elif retrieval_omitted_reason:
|
|
1267
1660
|
header.append(f"Retrieval omitted: {retrieval_omitted_reason}")
|
|
1268
1661
|
return "\n".join(header)
|
|
1269
1662
|
|
|
1270
1663
|
|
|
1271
1664
|
def render_block(source: ResolvedSource, lines: list[str], *, root_arg: str, status: str, included: LineRange) -> str:
|
|
1272
|
-
|
|
1665
|
+
block_open, block_close = markdown_block_delimiters(lines)
|
|
1666
|
+
return render_block_header(source, root_arg=root_arg, status=status, included=included) + block_open + "".join(lines) + ("" if not lines or lines[-1].endswith("\n") else "\n") + block_close
|
|
1667
|
+
|
|
1668
|
+
|
|
1669
|
+
def source_input_metadata(source: ResolvedSource) -> dict[str, Any]:
|
|
1670
|
+
item: dict[str, Any] = {
|
|
1671
|
+
"bytes_read": source.input_bytes_read,
|
|
1672
|
+
"lines_read": source.input_lines_read,
|
|
1673
|
+
"bytes_attempted": source.input_bytes_read,
|
|
1674
|
+
"lines_attempted": source.input_lines_read,
|
|
1675
|
+
"capped": source.input_limit_reason is not None,
|
|
1676
|
+
"total_lines_exact": source.total_lines_exact,
|
|
1677
|
+
"truncated": not source.total_lines_exact,
|
|
1678
|
+
"sanitized_through_line": source.sanitized_through_line,
|
|
1679
|
+
"redacted_lines_exact": source.redacted_lines_exact,
|
|
1680
|
+
"limits": {
|
|
1681
|
+
"source_bytes": MAX_SOURCE_INPUT_BYTES,
|
|
1682
|
+
"source_lines": MAX_SOURCE_INPUT_LINES,
|
|
1683
|
+
"source_line_bytes": MAX_SOURCE_LINE_BYTES,
|
|
1684
|
+
"cumulative_bytes": MAX_TOTAL_SOURCE_INPUT_BYTES,
|
|
1685
|
+
"cumulative_lines": MAX_TOTAL_SOURCE_INPUT_LINES,
|
|
1686
|
+
},
|
|
1687
|
+
}
|
|
1688
|
+
if source.total_lines_exact:
|
|
1689
|
+
item["total_lines"] = source.total_lines
|
|
1690
|
+
else:
|
|
1691
|
+
item["total_lines_lower_bound"] = source.total_lines
|
|
1692
|
+
if source.input_limit_reason is not None:
|
|
1693
|
+
item["limit_reason"] = source.input_limit_reason
|
|
1694
|
+
return item
|
|
1273
1695
|
|
|
1274
1696
|
|
|
1275
1697
|
def source_metadata(source: ResolvedSource, *, status: str, lines: list[str], included: LineRange, root_arg: str) -> dict[str, Any]:
|
|
@@ -1285,6 +1707,7 @@ def source_metadata(source: ResolvedSource, *, status: str, lines: list[str], in
|
|
|
1285
1707
|
}
|
|
1286
1708
|
if source.spec.label:
|
|
1287
1709
|
item["label"] = source.spec.label
|
|
1710
|
+
item["input"] = source_input_metadata(source)
|
|
1288
1711
|
retrieval, retrieval_omitted_reason = retrieval_for(root_arg, source.display_path, included, redacted_path=source.redacted_path)
|
|
1289
1712
|
if retrieval:
|
|
1290
1713
|
item["retrieval_cli"] = retrieval
|
|
@@ -1299,7 +1722,11 @@ def budget_omission(source: ResolvedSource, *, root_arg: str) -> dict[str, Any]:
|
|
|
1299
1722
|
requested = source.requested_lines or LineRange(1, source.total_lines)
|
|
1300
1723
|
item = omission(source.spec, "budget_exhausted", path=source.display_path, redacted_path=source.redacted_path)
|
|
1301
1724
|
item["requested_lines"] = requested.as_dict()
|
|
1302
|
-
|
|
1725
|
+
if source.total_lines_exact:
|
|
1726
|
+
item["total_lines"] = source.total_lines
|
|
1727
|
+
else:
|
|
1728
|
+
item["total_lines_lower_bound"] = source.total_lines
|
|
1729
|
+
item["input"] = source_input_metadata(source)
|
|
1303
1730
|
retrieval, retrieval_omitted_reason = retrieval_for(root_arg, source.display_path, requested, redacted_path=source.redacted_path)
|
|
1304
1731
|
if retrieval:
|
|
1305
1732
|
item["retrieval_cli"] = retrieval
|
|
@@ -1335,7 +1762,8 @@ def render_block_byte_len(
|
|
|
1335
1762
|
body_bytes = line_prefixes[line_count]
|
|
1336
1763
|
if line_count > 0 and not source.selected_lines[line_count - 1].endswith("\n"):
|
|
1337
1764
|
body_bytes += 1
|
|
1338
|
-
|
|
1765
|
+
block_open, block_close = markdown_block_delimiters(source.selected_lines[:line_count])
|
|
1766
|
+
return byte_len(render_block_header(source, root_arg=root_arg, status=status, included=included)) + byte_len(block_open) + body_bytes + byte_len(block_close)
|
|
1339
1767
|
|
|
1340
1768
|
|
|
1341
1769
|
def fit_partial_lines(
|
|
@@ -1834,7 +2262,13 @@ def build_pack(
|
|
|
1834
2262
|
store_artifact: bool,
|
|
1835
2263
|
delta_from_pack_id: str | None = None,
|
|
1836
2264
|
sketch_duplicate_veto: bool = False,
|
|
2265
|
+
_source_cache: _SourceSnapshotCache | None = None,
|
|
2266
|
+
_input_budget: _SourceInputBudget | None = None,
|
|
2267
|
+
_required_snapshot_sources: set[tuple[str, str]] | None = None,
|
|
2268
|
+
_expected_source_identities: dict[str, tuple[int, int, int, int, int, int, int, int]] | None = None,
|
|
2269
|
+
_snapshot_rejections: dict[tuple[str, str], dict[str, Any]] | None = None,
|
|
1837
2270
|
) -> dict[str, Any]:
|
|
2271
|
+
input_budget = _input_budget if _input_budget is not None else _SourceInputBudget()
|
|
1838
2272
|
seen: set[tuple[str, str]] = set()
|
|
1839
2273
|
resolved: list[ResolvedSource] = []
|
|
1840
2274
|
paired_candidates: list[_PairedCandidate] = []
|
|
@@ -1861,7 +2295,55 @@ def build_pack(
|
|
|
1861
2295
|
continue
|
|
1862
2296
|
if rel is not None:
|
|
1863
2297
|
seen.add(identity)
|
|
1864
|
-
|
|
2298
|
+
rel_path = rel.as_posix() if rel is not None else ""
|
|
2299
|
+
require_cached = bool(
|
|
2300
|
+
_required_snapshot_sources
|
|
2301
|
+
and (rel_path, identity_lines) in _required_snapshot_sources
|
|
2302
|
+
)
|
|
2303
|
+
expected_identity = (
|
|
2304
|
+
_expected_source_identities.get(rel_path)
|
|
2305
|
+
if require_cached and _expected_source_identities is not None
|
|
2306
|
+
else None
|
|
2307
|
+
)
|
|
2308
|
+
if require_cached and expected_identity is None:
|
|
2309
|
+
display, redacted = display_rel_path(rel_path)
|
|
2310
|
+
omitted_item = omission(
|
|
2311
|
+
spec,
|
|
2312
|
+
"graph_source_not_in_repo_map_snapshot",
|
|
2313
|
+
path=display,
|
|
2314
|
+
redacted_path=redacted,
|
|
2315
|
+
)
|
|
2316
|
+
omitted.append(omitted_item)
|
|
2317
|
+
canonical_specs.append({
|
|
2318
|
+
"path": display,
|
|
2319
|
+
"priority": spec.priority,
|
|
2320
|
+
"lines": identity_lines,
|
|
2321
|
+
"status": omitted_item.get("reason"),
|
|
2322
|
+
})
|
|
2323
|
+
continue
|
|
2324
|
+
cached_rejection = (
|
|
2325
|
+
_snapshot_rejections.get((rel_path, identity_lines))
|
|
2326
|
+
if require_cached and _snapshot_rejections is not None
|
|
2327
|
+
else None
|
|
2328
|
+
)
|
|
2329
|
+
if cached_rejection is not None:
|
|
2330
|
+
omitted_item = copy.deepcopy(cached_rejection)
|
|
2331
|
+
omitted.append(omitted_item)
|
|
2332
|
+
canonical_specs.append({
|
|
2333
|
+
"path": omitted_item.get("path"),
|
|
2334
|
+
"priority": spec.priority,
|
|
2335
|
+
"lines": identity_lines,
|
|
2336
|
+
"status": omitted_item.get("reason"),
|
|
2337
|
+
})
|
|
2338
|
+
continue
|
|
2339
|
+
source, omitted_item = resolve_source(
|
|
2340
|
+
root,
|
|
2341
|
+
spec,
|
|
2342
|
+
source_cache=_source_cache,
|
|
2343
|
+
input_budget=input_budget,
|
|
2344
|
+
expected_identity=expected_identity,
|
|
2345
|
+
require_cached=require_cached,
|
|
2346
|
+
)
|
|
1865
2347
|
if omitted_item is not None:
|
|
1866
2348
|
omitted.append(omitted_item)
|
|
1867
2349
|
canonical_specs.append({"path": omitted_item.get("path"), "priority": spec.priority, "lines": identity_lines, "status": omitted_item.get("reason")})
|
|
@@ -1936,7 +2418,27 @@ def build_pack(
|
|
|
1936
2418
|
"sources": {"total": len(specs), "included": len(included) - partial_count, "partial": partial_count, "omitted": len(omitted_sorted)},
|
|
1937
2419
|
"included_sources": included,
|
|
1938
2420
|
"omitted_sources": omitted_sorted,
|
|
1939
|
-
"redaction": {
|
|
2421
|
+
"redaction": {
|
|
2422
|
+
"redacted_lines": redacted_lines,
|
|
2423
|
+
"redacted_lines_exact": all(source.redacted_lines_exact for source in all_resolved),
|
|
2424
|
+
"redacted_before_pack": True,
|
|
2425
|
+
},
|
|
2426
|
+
"input": {
|
|
2427
|
+
"bytes_read": input_budget.bytes_read,
|
|
2428
|
+
"lines_read": input_budget.lines_read,
|
|
2429
|
+
"bytes_attempted": input_budget.bytes_attempted,
|
|
2430
|
+
"lines_attempted": input_budget.lines_attempted,
|
|
2431
|
+
"bytes_charged": input_budget.bytes_charged,
|
|
2432
|
+
"lines_charged": input_budget.lines_charged,
|
|
2433
|
+
"capped": input_budget.capped,
|
|
2434
|
+
"limits": {
|
|
2435
|
+
"source_bytes": MAX_SOURCE_INPUT_BYTES,
|
|
2436
|
+
"source_lines": MAX_SOURCE_INPUT_LINES,
|
|
2437
|
+
"source_line_bytes": MAX_SOURCE_LINE_BYTES,
|
|
2438
|
+
"cumulative_bytes": MAX_TOTAL_SOURCE_INPUT_BYTES,
|
|
2439
|
+
"cumulative_lines": MAX_TOTAL_SOURCE_INPUT_LINES,
|
|
2440
|
+
},
|
|
2441
|
+
},
|
|
1940
2442
|
"artifact": {"stored": False, "path": None, "bytes": 0, "capped": False, "cap_bytes": MAX_RECEIPT_BYTES},
|
|
1941
2443
|
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
1942
2444
|
}
|
|
@@ -1982,7 +2484,12 @@ def slice_source(root: Path, *, raw_path: str, lines: LineRange) -> tuple[dict[s
|
|
|
1982
2484
|
"query": {"type": "lines", "start": lines.start, "end": min(lines.end, source.total_lines), "returned_lines": len(source.selected_lines)},
|
|
1983
2485
|
"content": content,
|
|
1984
2486
|
"bytes": byte_len(content),
|
|
1985
|
-
"redaction": {
|
|
2487
|
+
"redaction": {
|
|
2488
|
+
"redacted_lines": source.redacted_lines,
|
|
2489
|
+
"redacted_lines_exact": source.redacted_lines_exact,
|
|
2490
|
+
"redacted_before_pack": True,
|
|
2491
|
+
},
|
|
2492
|
+
"input": source_input_metadata(source),
|
|
1986
2493
|
}
|
|
1987
2494
|
return payload, 0
|
|
1988
2495
|
|
|
@@ -2052,32 +2559,245 @@ def add_suggest_candidate(
|
|
|
2052
2559
|
)
|
|
2053
2560
|
|
|
2054
2561
|
|
|
2562
|
+
def trusted_git_executable() -> str:
|
|
2563
|
+
if os.name != "posix":
|
|
2564
|
+
raise OSError("Git execution is unavailable on this platform")
|
|
2565
|
+
executable_names = ("git",)
|
|
2566
|
+
for directory in os.defpath.split(os.pathsep):
|
|
2567
|
+
if not directory:
|
|
2568
|
+
continue
|
|
2569
|
+
for name in executable_names:
|
|
2570
|
+
candidate = Path(directory) / name
|
|
2571
|
+
try:
|
|
2572
|
+
if candidate.is_file() and os.access(candidate, os.X_OK):
|
|
2573
|
+
return str(candidate)
|
|
2574
|
+
except OSError:
|
|
2575
|
+
continue
|
|
2576
|
+
raise OSError("trusted system git executable unavailable")
|
|
2577
|
+
|
|
2578
|
+
|
|
2579
|
+
def guarded_git_environment() -> dict[str, str]:
|
|
2580
|
+
return {
|
|
2581
|
+
"PATH": os.defpath,
|
|
2582
|
+
"LANG": "C",
|
|
2583
|
+
"LC_ALL": "C",
|
|
2584
|
+
"GIT_CONFIG_GLOBAL": os.devnull,
|
|
2585
|
+
"GIT_CONFIG_SYSTEM": os.devnull,
|
|
2586
|
+
"GIT_CONFIG_NOSYSTEM": "1",
|
|
2587
|
+
"GIT_ATTR_NOSYSTEM": "1",
|
|
2588
|
+
"GIT_TERMINAL_PROMPT": "0",
|
|
2589
|
+
"GIT_ASKPASS": os.devnull,
|
|
2590
|
+
"SSH_ASKPASS": os.devnull,
|
|
2591
|
+
"GCM_INTERACTIVE": "Never",
|
|
2592
|
+
"GIT_NO_LAZY_FETCH": "1",
|
|
2593
|
+
"GIT_OPTIONAL_LOCKS": "0",
|
|
2594
|
+
"GIT_PAGER": "cat",
|
|
2595
|
+
"PAGER": "cat",
|
|
2596
|
+
}
|
|
2597
|
+
|
|
2598
|
+
|
|
2599
|
+
def guarded_git_command(root: Path, *args: str) -> list[str]:
|
|
2600
|
+
return [
|
|
2601
|
+
trusted_git_executable(),
|
|
2602
|
+
"-c",
|
|
2603
|
+
"core.fsmonitor=false",
|
|
2604
|
+
"-c",
|
|
2605
|
+
f"core.hooksPath={os.devnull}",
|
|
2606
|
+
"-c",
|
|
2607
|
+
f"core.attributesFile={os.devnull}",
|
|
2608
|
+
"-c",
|
|
2609
|
+
"credential.helper=",
|
|
2610
|
+
"-c",
|
|
2611
|
+
"core.askPass=",
|
|
2612
|
+
"-c",
|
|
2613
|
+
"credential.interactive=never",
|
|
2614
|
+
"-c",
|
|
2615
|
+
"filter.unset.clean=",
|
|
2616
|
+
"-c",
|
|
2617
|
+
"filter.unset.process=",
|
|
2618
|
+
"-c",
|
|
2619
|
+
"filter.unset.required=false",
|
|
2620
|
+
"-c",
|
|
2621
|
+
"filter.unspecified.clean=",
|
|
2622
|
+
"-c",
|
|
2623
|
+
"filter.unspecified.process=",
|
|
2624
|
+
"-c",
|
|
2625
|
+
"filter.unspecified.required=false",
|
|
2626
|
+
"-C",
|
|
2627
|
+
str(root),
|
|
2628
|
+
*args,
|
|
2629
|
+
]
|
|
2630
|
+
|
|
2631
|
+
|
|
2632
|
+
def _signal_process_group(proc: subprocess.Popen[Any], *, force: bool) -> None:
|
|
2633
|
+
requested_signal = getattr(signal, "SIGKILL", signal.SIGTERM) if force else signal.SIGTERM
|
|
2634
|
+
if os.name == "posix" and hasattr(os, "killpg"):
|
|
2635
|
+
try:
|
|
2636
|
+
os.killpg(proc.pid, requested_signal)
|
|
2637
|
+
return
|
|
2638
|
+
except ProcessLookupError:
|
|
2639
|
+
return
|
|
2640
|
+
except OSError:
|
|
2641
|
+
pass
|
|
2642
|
+
if proc.poll() is not None:
|
|
2643
|
+
return
|
|
2644
|
+
try:
|
|
2645
|
+
if force:
|
|
2646
|
+
proc.kill()
|
|
2647
|
+
else:
|
|
2648
|
+
proc.terminate()
|
|
2649
|
+
except OSError:
|
|
2650
|
+
pass
|
|
2651
|
+
|
|
2652
|
+
|
|
2653
|
+
def _run_process_capped(
|
|
2654
|
+
command: list[str],
|
|
2655
|
+
*,
|
|
2656
|
+
stdout_cap: int,
|
|
2657
|
+
stderr_cap: int,
|
|
2658
|
+
timeout_seconds: float,
|
|
2659
|
+
environment: dict[str, str] | None = None,
|
|
2660
|
+
stdin_data: bytes | None = None,
|
|
2661
|
+
stdin_cap_bytes: int | None = None,
|
|
2662
|
+
) -> tuple[int, bytes, bytes, bool, bool]:
|
|
2663
|
+
if stdin_data is not None and (
|
|
2664
|
+
stdin_cap_bytes is None or len(stdin_data) > stdin_cap_bytes
|
|
2665
|
+
):
|
|
2666
|
+
raise PackError("process stdin exceeds cap")
|
|
2667
|
+
proc = subprocess.Popen(
|
|
2668
|
+
command,
|
|
2669
|
+
stdin=subprocess.PIPE if stdin_data is not None else subprocess.DEVNULL,
|
|
2670
|
+
stdout=subprocess.PIPE,
|
|
2671
|
+
stderr=subprocess.PIPE,
|
|
2672
|
+
text=False,
|
|
2673
|
+
start_new_session=os.name == "posix",
|
|
2674
|
+
env=environment,
|
|
2675
|
+
)
|
|
2676
|
+
buffers: dict[str, list[bytes]] = {"stdout": [], "stderr": []}
|
|
2677
|
+
capped = {"stdout": False, "stderr": False}
|
|
2678
|
+
stop = threading.Event()
|
|
2679
|
+
|
|
2680
|
+
def drain(name: str, stream: Any, cap: int) -> None:
|
|
2681
|
+
total = 0
|
|
2682
|
+
try:
|
|
2683
|
+
while not stop.is_set() and total <= cap:
|
|
2684
|
+
chunk = stream.read(min(64 * 1024, cap + 1 - total))
|
|
2685
|
+
if not chunk:
|
|
2686
|
+
break
|
|
2687
|
+
buffers[name].append(chunk)
|
|
2688
|
+
total += len(chunk)
|
|
2689
|
+
if total > cap:
|
|
2690
|
+
capped[name] = True
|
|
2691
|
+
stop.set()
|
|
2692
|
+
_signal_process_group(proc, force=False)
|
|
2693
|
+
break
|
|
2694
|
+
finally:
|
|
2695
|
+
try:
|
|
2696
|
+
stream.close()
|
|
2697
|
+
except OSError:
|
|
2698
|
+
pass
|
|
2699
|
+
|
|
2700
|
+
threads = [
|
|
2701
|
+
threading.Thread(target=drain, args=("stdout", proc.stdout, stdout_cap), daemon=True),
|
|
2702
|
+
threading.Thread(target=drain, args=("stderr", proc.stderr, stderr_cap), daemon=True),
|
|
2703
|
+
]
|
|
2704
|
+
if stdin_data is not None:
|
|
2705
|
+
def write_stdin() -> None:
|
|
2706
|
+
try:
|
|
2707
|
+
assert proc.stdin is not None
|
|
2708
|
+
view = memoryview(stdin_data)
|
|
2709
|
+
for offset in range(0, len(view), 64 * 1024):
|
|
2710
|
+
if stop.is_set():
|
|
2711
|
+
break
|
|
2712
|
+
proc.stdin.write(view[offset : offset + 64 * 1024])
|
|
2713
|
+
proc.stdin.flush()
|
|
2714
|
+
except (BrokenPipeError, OSError, ValueError):
|
|
2715
|
+
pass
|
|
2716
|
+
finally:
|
|
2717
|
+
if proc.stdin is not None:
|
|
2718
|
+
try:
|
|
2719
|
+
proc.stdin.close()
|
|
2720
|
+
except OSError:
|
|
2721
|
+
pass
|
|
2722
|
+
|
|
2723
|
+
threads.append(threading.Thread(target=write_stdin, daemon=True))
|
|
2724
|
+
for thread in threads:
|
|
2725
|
+
thread.start()
|
|
2726
|
+
timed_out = False
|
|
2727
|
+
try:
|
|
2728
|
+
proc.wait(timeout=timeout_seconds)
|
|
2729
|
+
except subprocess.TimeoutExpired:
|
|
2730
|
+
timed_out = True
|
|
2731
|
+
stop.set()
|
|
2732
|
+
_signal_process_group(proc, force=False)
|
|
2733
|
+
try:
|
|
2734
|
+
proc.wait(timeout=0.2)
|
|
2735
|
+
except subprocess.TimeoutExpired:
|
|
2736
|
+
_signal_process_group(proc, force=True)
|
|
2737
|
+
try:
|
|
2738
|
+
proc.wait(timeout=2)
|
|
2739
|
+
except subprocess.TimeoutExpired:
|
|
2740
|
+
pass
|
|
2741
|
+
if capped["stdout"] or capped["stderr"] or timed_out:
|
|
2742
|
+
_signal_process_group(proc, force=True)
|
|
2743
|
+
for thread in threads:
|
|
2744
|
+
thread.join(0.5)
|
|
2745
|
+
if any(thread.is_alive() for thread in threads):
|
|
2746
|
+
stop.set()
|
|
2747
|
+
_signal_process_group(proc, force=True)
|
|
2748
|
+
for thread in threads:
|
|
2749
|
+
thread.join(0.2)
|
|
2750
|
+
stdout = b"".join(buffers["stdout"])[:stdout_cap]
|
|
2751
|
+
stderr = b"".join(buffers["stderr"])[:stderr_cap]
|
|
2752
|
+
return proc.returncode if proc.returncode is not None else -1, stdout, stderr, capped["stdout"], capped["stderr"] or timed_out
|
|
2753
|
+
|
|
2754
|
+
|
|
2055
2755
|
def run_git_diff(root: Path, diff_ref: str) -> str:
|
|
2056
2756
|
ref = diff_ref.strip()
|
|
2057
2757
|
if not ref:
|
|
2058
2758
|
raise PackError("empty --diff")
|
|
2059
|
-
|
|
2759
|
+
git_args = [
|
|
2760
|
+
"diff",
|
|
2761
|
+
"--no-ext-diff",
|
|
2762
|
+
"--no-textconv",
|
|
2763
|
+
"--ignore-submodules=all",
|
|
2764
|
+
"--unified=3",
|
|
2765
|
+
]
|
|
2060
2766
|
if ref in {"staged", "--staged", "cached", "--cached"}:
|
|
2061
|
-
|
|
2767
|
+
git_args.append("--cached")
|
|
2062
2768
|
elif ref in {"worktree", "unstaged", "working-tree"}:
|
|
2063
2769
|
pass
|
|
2064
2770
|
elif ref.startswith("-"):
|
|
2065
2771
|
raise PackError("invalid --diff: revision must not start with '-'")
|
|
2066
2772
|
else:
|
|
2067
|
-
|
|
2773
|
+
git_args.append(ref)
|
|
2068
2774
|
try:
|
|
2069
|
-
|
|
2775
|
+
reject_configured_git_filters(root)
|
|
2776
|
+
command = guarded_git_command(root, *git_args)
|
|
2777
|
+
returncode, stdout, stderr, stdout_capped, stderr_capped_or_timeout = _run_process_capped(
|
|
2778
|
+
command,
|
|
2779
|
+
stdout_cap=MAX_SUGGEST_INPUT_BYTES,
|
|
2780
|
+
stderr_cap=MAX_GIT_DIFF_STDERR_BYTES,
|
|
2781
|
+
timeout_seconds=GIT_DIFF_TIMEOUT_SECONDS,
|
|
2782
|
+
environment=guarded_git_environment(),
|
|
2783
|
+
)
|
|
2070
2784
|
except (OSError, UnicodeError, subprocess.TimeoutExpired) as exc:
|
|
2071
2785
|
raise PackError(f"could not read diff: {exc.__class__.__name__}") from exc
|
|
2072
|
-
if
|
|
2786
|
+
if stdout_capped:
|
|
2787
|
+
raise PackError(f"could not read diff: diff output exceeds cap ({MAX_SUGGEST_INPUT_BYTES} bytes)")
|
|
2788
|
+
if stderr_capped_or_timeout:
|
|
2789
|
+
raise PackError("could not read diff: stderr cap or timeout exceeded")
|
|
2790
|
+
stdout_text = stdout.decode("utf-8", "replace")
|
|
2791
|
+
stderr_text = stderr.decode("utf-8", "replace")
|
|
2792
|
+
if returncode != 0:
|
|
2073
2793
|
detail = sanitize_text(
|
|
2074
|
-
|
|
2794
|
+
stderr_text or stdout_text or "git diff failed",
|
|
2075
2795
|
context="command_search_diff",
|
|
2076
2796
|
)[0].strip().splitlines()
|
|
2077
2797
|
message = detail[0] if detail else "git diff failed"
|
|
2078
2798
|
raise PackError(f"could not read diff: {cap_label(message, default='git diff failed', limit=160)}")
|
|
2079
2799
|
return sanitize_text(
|
|
2080
|
-
|
|
2800
|
+
stdout_text,
|
|
2081
2801
|
context="command_search_diff",
|
|
2082
2802
|
)[0]
|
|
2083
2803
|
|
|
@@ -2190,100 +2910,269 @@ def collect_output_candidates(
|
|
|
2190
2910
|
return candidates, omitted
|
|
2191
2911
|
|
|
2192
2912
|
|
|
2193
|
-
def
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
try:
|
|
2205
|
-
while total <= limit:
|
|
2206
|
-
chunk = proc.stdout.read(min(GIT_LS_FILES_READ_CHUNK_BYTES, limit + 1 - total))
|
|
2207
|
-
if not chunk:
|
|
2208
|
-
break
|
|
2209
|
-
chunks.append(chunk)
|
|
2210
|
-
total += len(chunk)
|
|
2211
|
-
if total > limit:
|
|
2212
|
-
capped = True
|
|
2213
|
-
break
|
|
2214
|
-
finally:
|
|
2215
|
-
if capped and proc.poll() is None:
|
|
2216
|
-
try:
|
|
2217
|
-
proc.terminate()
|
|
2218
|
-
except OSError:
|
|
2219
|
-
pass
|
|
2220
|
-
try:
|
|
2221
|
-
proc.stdout.close()
|
|
2222
|
-
except OSError:
|
|
2223
|
-
pass
|
|
2913
|
+
def _read_git_stdout_capped(
|
|
2914
|
+
proc: subprocess.Popen[bytes],
|
|
2915
|
+
limit: int,
|
|
2916
|
+
timeout_seconds: float,
|
|
2917
|
+
) -> tuple[bytes, bool]:
|
|
2918
|
+
if proc.stdout is None:
|
|
2919
|
+
return b"", False
|
|
2920
|
+
chunks: list[bytes] = []
|
|
2921
|
+
total = 0
|
|
2922
|
+
capped = False
|
|
2923
|
+
timed_out = False
|
|
2224
2924
|
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2925
|
+
def reader() -> None:
|
|
2926
|
+
nonlocal total, capped
|
|
2927
|
+
try:
|
|
2928
|
+
while total <= limit:
|
|
2929
|
+
chunk = proc.stdout.read(min(GIT_LS_FILES_READ_CHUNK_BYTES, limit + 1 - total))
|
|
2930
|
+
if not chunk:
|
|
2931
|
+
break
|
|
2932
|
+
chunks.append(chunk)
|
|
2933
|
+
total += len(chunk)
|
|
2934
|
+
if total > limit:
|
|
2935
|
+
capped = True
|
|
2936
|
+
break
|
|
2937
|
+
finally:
|
|
2938
|
+
if capped:
|
|
2939
|
+
_signal_process_group(proc, force=False)
|
|
2230
2940
|
try:
|
|
2231
|
-
proc.
|
|
2941
|
+
proc.stdout.close()
|
|
2232
2942
|
except OSError:
|
|
2233
2943
|
pass
|
|
2944
|
+
|
|
2945
|
+
thread = threading.Thread(target=reader, daemon=True)
|
|
2946
|
+
thread.start()
|
|
2947
|
+
thread.join(timeout_seconds)
|
|
2948
|
+
if thread.is_alive():
|
|
2949
|
+
timed_out = True
|
|
2950
|
+
_signal_process_group(proc, force=False)
|
|
2951
|
+
try:
|
|
2952
|
+
proc.wait(timeout=0.2 if timed_out else 2)
|
|
2953
|
+
except subprocess.TimeoutExpired:
|
|
2954
|
+
_signal_process_group(proc, force=True)
|
|
2234
2955
|
try:
|
|
2235
2956
|
proc.wait(timeout=2)
|
|
2236
2957
|
except subprocess.TimeoutExpired:
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
|
|
2246
|
-
raw_output
|
|
2247
|
-
|
|
2248
|
-
|
|
2958
|
+
pass
|
|
2959
|
+
if capped or timed_out:
|
|
2960
|
+
_signal_process_group(proc, force=True)
|
|
2961
|
+
thread.join(0.5)
|
|
2962
|
+
raw_output = b"".join(chunks)[:limit]
|
|
2963
|
+
complete = (
|
|
2964
|
+
proc.returncode == 0
|
|
2965
|
+
and not capped
|
|
2966
|
+
and not timed_out
|
|
2967
|
+
and (not raw_output or raw_output.endswith(b"\0"))
|
|
2968
|
+
)
|
|
2969
|
+
return raw_output, complete
|
|
2970
|
+
|
|
2249
2971
|
|
|
2250
|
-
|
|
2251
|
-
git_returncode: int | None = None
|
|
2972
|
+
def _git_ls_files_raw(root: Path) -> tuple[bytes, bool, int | None]:
|
|
2252
2973
|
try:
|
|
2253
2974
|
proc = subprocess.Popen(
|
|
2254
|
-
|
|
2975
|
+
guarded_git_command(root, "ls-files", "-z"),
|
|
2976
|
+
stdin=subprocess.DEVNULL,
|
|
2255
2977
|
stdout=subprocess.PIPE,
|
|
2256
2978
|
stderr=subprocess.DEVNULL,
|
|
2257
2979
|
text=False,
|
|
2980
|
+
start_new_session=os.name == "posix",
|
|
2981
|
+
env=guarded_git_environment(),
|
|
2982
|
+
)
|
|
2983
|
+
raw, complete = _read_git_stdout_capped(
|
|
2984
|
+
proc,
|
|
2985
|
+
MAX_GIT_LS_FILES_OUTPUT_BYTES,
|
|
2986
|
+
10,
|
|
2258
2987
|
)
|
|
2259
|
-
raw,
|
|
2260
|
-
git_returncode = proc.returncode
|
|
2988
|
+
return raw, complete, proc.returncode
|
|
2261
2989
|
except (OSError, subprocess.TimeoutExpired):
|
|
2262
|
-
|
|
2990
|
+
return b"", False, None
|
|
2991
|
+
|
|
2992
|
+
|
|
2993
|
+
def _iter_nul_fields(raw: bytes):
|
|
2994
|
+
view = memoryview(raw)
|
|
2995
|
+
start = 0
|
|
2996
|
+
while start < len(raw):
|
|
2997
|
+
end = raw.find(b"\0", start)
|
|
2998
|
+
if end < 0:
|
|
2999
|
+
return
|
|
3000
|
+
yield view[start:end]
|
|
3001
|
+
start = end + 1
|
|
3002
|
+
|
|
3003
|
+
|
|
3004
|
+
def git_ls_files(root: Path, diagnostics: dict[str, Any] | None = None) -> list[str]:
|
|
3005
|
+
raw, git_complete, git_returncode = _git_ls_files_raw(root)
|
|
2263
3006
|
if raw:
|
|
2264
3007
|
if not raw.endswith(b"\0"):
|
|
2265
3008
|
raw = raw.rsplit(b"\0", 1)[0] if b"\0" in raw else b""
|
|
2266
|
-
|
|
2267
|
-
|
|
3009
|
+
retained_parts: list[bytes] = []
|
|
3010
|
+
file_cap_reached = False
|
|
3011
|
+
for part_view in _iter_nul_fields(raw):
|
|
3012
|
+
if not part_view:
|
|
3013
|
+
continue
|
|
3014
|
+
if len(retained_parts) >= MAX_QUERY_SCAN_FILES:
|
|
3015
|
+
file_cap_reached = True
|
|
3016
|
+
break
|
|
3017
|
+
retained_parts.append(bytes(part_view))
|
|
3018
|
+
if diagnostics is not None:
|
|
3019
|
+
diagnostics.update({
|
|
3020
|
+
"mode": "git",
|
|
3021
|
+
"truncated": not git_complete or file_cap_reached,
|
|
3022
|
+
"truncation_reason": (
|
|
3023
|
+
"git_output_cap_or_timeout"
|
|
3024
|
+
if not git_complete
|
|
3025
|
+
else "file_cap" if file_cap_reached else None
|
|
3026
|
+
),
|
|
3027
|
+
})
|
|
3028
|
+
return [part.decode("utf-8", "replace") for part in retained_parts]
|
|
3029
|
+
if git_returncode == 0:
|
|
3030
|
+
if diagnostics is not None:
|
|
3031
|
+
diagnostics.update({"mode": "git", "truncated": False, "truncation_reason": None})
|
|
3032
|
+
return []
|
|
3033
|
+
if git_returncode is not None and git_returncode < 0:
|
|
3034
|
+
if diagnostics is not None:
|
|
3035
|
+
diagnostics.update({
|
|
3036
|
+
"mode": "git",
|
|
3037
|
+
"truncated": True,
|
|
3038
|
+
"truncation_reason": "git_output_cap_or_timeout",
|
|
3039
|
+
})
|
|
2268
3040
|
return []
|
|
2269
3041
|
out: list[str] = []
|
|
2270
3042
|
skip_dirs = {".git", ".omx", ".context-guard", "node_modules", "dist", "build", "__pycache__"}
|
|
2271
|
-
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
|
|
3043
|
+
started = time.monotonic()
|
|
3044
|
+
visited_dirs = 0
|
|
3045
|
+
visited_entries = 0
|
|
3046
|
+
truncation_reason: str | None = None
|
|
3047
|
+
pending: deque[tuple[Path, int]] = deque([(root, 0)])
|
|
3048
|
+
while pending:
|
|
3049
|
+
if time.monotonic() - started > MAX_QUERY_WALK_SECONDS:
|
|
3050
|
+
truncation_reason = "time_cap"
|
|
3051
|
+
break
|
|
3052
|
+
if visited_dirs >= MAX_QUERY_WALK_DIRS:
|
|
3053
|
+
truncation_reason = "directory_cap"
|
|
3054
|
+
break
|
|
3055
|
+
current_path, depth = pending.popleft()
|
|
3056
|
+
visited_dirs += 1
|
|
3057
|
+
try:
|
|
3058
|
+
iterator = os.scandir(current_path)
|
|
3059
|
+
except OSError:
|
|
3060
|
+
truncation_reason = "unsafe_path"
|
|
3061
|
+
break
|
|
3062
|
+
child_dirs: list[Path] = []
|
|
3063
|
+
try:
|
|
3064
|
+
with iterator:
|
|
3065
|
+
for entry in iterator:
|
|
3066
|
+
if time.monotonic() - started > MAX_QUERY_WALK_SECONDS:
|
|
3067
|
+
truncation_reason = "time_cap"
|
|
3068
|
+
break
|
|
3069
|
+
if visited_entries >= MAX_QUERY_WALK_ENTRIES:
|
|
3070
|
+
truncation_reason = "entry_cap"
|
|
3071
|
+
break
|
|
3072
|
+
visited_entries += 1
|
|
3073
|
+
name = entry.name
|
|
3074
|
+
try:
|
|
3075
|
+
is_dir = entry.is_dir(follow_symlinks=False)
|
|
3076
|
+
is_file = entry.is_file(follow_symlinks=False)
|
|
3077
|
+
except OSError:
|
|
3078
|
+
continue
|
|
3079
|
+
if is_dir:
|
|
3080
|
+
if name in skip_dirs or name.startswith(".pytest"):
|
|
3081
|
+
continue
|
|
3082
|
+
if depth >= MAX_QUERY_WALK_DEPTH:
|
|
3083
|
+
truncation_reason = truncation_reason or "depth_cap"
|
|
3084
|
+
continue
|
|
3085
|
+
child_dirs.append(current_path / name)
|
|
3086
|
+
elif is_file:
|
|
3087
|
+
try:
|
|
3088
|
+
rel = (current_path / name).relative_to(root).as_posix()
|
|
3089
|
+
except ValueError:
|
|
3090
|
+
truncation_reason = "unsafe_path"
|
|
3091
|
+
break
|
|
3092
|
+
out.append(rel)
|
|
3093
|
+
if len(out) >= MAX_QUERY_SCAN_FILES:
|
|
3094
|
+
truncation_reason = "file_cap"
|
|
3095
|
+
break
|
|
3096
|
+
except OSError:
|
|
3097
|
+
truncation_reason = "unsafe_path"
|
|
3098
|
+
break
|
|
3099
|
+
if truncation_reason in {"time_cap", "entry_cap", "file_cap", "unsafe_path"}:
|
|
3100
|
+
break
|
|
3101
|
+
for child in reversed(sorted(child_dirs, key=lambda path: path.name)):
|
|
3102
|
+
pending.appendleft((child, depth + 1))
|
|
3103
|
+
if diagnostics is not None:
|
|
3104
|
+
diagnostics.update({
|
|
3105
|
+
"mode": "walk",
|
|
3106
|
+
"truncated": truncation_reason is not None,
|
|
3107
|
+
"truncation_reason": truncation_reason,
|
|
3108
|
+
"visited_dirs": min(visited_dirs, MAX_QUERY_WALK_DIRS),
|
|
3109
|
+
"visited_entries": min(visited_entries, MAX_QUERY_WALK_ENTRIES),
|
|
3110
|
+
})
|
|
2279
3111
|
return out
|
|
2280
3112
|
|
|
2281
3113
|
|
|
2282
|
-
def
|
|
3114
|
+
def reject_configured_git_filters(root: Path) -> None:
|
|
3115
|
+
raw_paths, complete, returncode = _git_ls_files_raw(root)
|
|
3116
|
+
if returncode != 0 or not complete:
|
|
3117
|
+
raise PackError("could not verify git filters: tracked path scan failed or truncated")
|
|
3118
|
+
if not raw_paths:
|
|
3119
|
+
return
|
|
3120
|
+
if len(raw_paths) > MAX_GIT_ATTR_INPUT_BYTES:
|
|
3121
|
+
raise PackError("could not verify git filters: tracked path input exceeds cap")
|
|
3122
|
+
|
|
3123
|
+
try:
|
|
3124
|
+
command = guarded_git_command(
|
|
3125
|
+
root,
|
|
3126
|
+
"check-attr",
|
|
3127
|
+
"-z",
|
|
3128
|
+
"--stdin",
|
|
3129
|
+
"filter",
|
|
3130
|
+
)
|
|
3131
|
+
returncode, stdout, _stderr, stdout_capped, failed_or_timed_out = _run_process_capped(
|
|
3132
|
+
command,
|
|
3133
|
+
stdout_cap=MAX_GIT_ATTR_OUTPUT_BYTES,
|
|
3134
|
+
stderr_cap=MAX_GIT_DIFF_STDERR_BYTES,
|
|
3135
|
+
timeout_seconds=GIT_ATTR_TIMEOUT_SECONDS,
|
|
3136
|
+
environment=guarded_git_environment(),
|
|
3137
|
+
stdin_data=raw_paths,
|
|
3138
|
+
stdin_cap_bytes=MAX_GIT_ATTR_INPUT_BYTES,
|
|
3139
|
+
)
|
|
3140
|
+
except (OSError, UnicodeError, subprocess.TimeoutExpired) as exc:
|
|
3141
|
+
raise PackError(f"could not verify git filters: {exc.__class__.__name__}") from exc
|
|
3142
|
+
if stdout_capped or failed_or_timed_out or returncode != 0:
|
|
3143
|
+
raise PackError("could not verify git filters: check-attr failed or exceeded cap")
|
|
3144
|
+
if not stdout.endswith(b"\0"):
|
|
3145
|
+
raise PackError("could not verify git filters: malformed check-attr output")
|
|
3146
|
+
output_fields = iter(_iter_nul_fields(stdout))
|
|
3147
|
+
for expected_path in _iter_nul_fields(raw_paths):
|
|
3148
|
+
try:
|
|
3149
|
+
path = next(output_fields)
|
|
3150
|
+
attribute = next(output_fields)
|
|
3151
|
+
value = next(output_fields)
|
|
3152
|
+
except StopIteration as exc:
|
|
3153
|
+
raise PackError("could not verify git filters: incomplete check-attr output") from exc
|
|
3154
|
+
if path != expected_path or bytes(attribute) != b"filter":
|
|
3155
|
+
raise PackError("could not verify git filters: mismatched check-attr output")
|
|
3156
|
+
if bytes(value) not in {b"unspecified", b"unset"}:
|
|
3157
|
+
raise PackError("git diff blocked: configured filter attribute")
|
|
3158
|
+
try:
|
|
3159
|
+
next(output_fields)
|
|
3160
|
+
except StopIteration:
|
|
3161
|
+
return
|
|
3162
|
+
raise PackError("could not verify git filters: excess check-attr output")
|
|
3163
|
+
|
|
3164
|
+
|
|
3165
|
+
def collect_query_candidates(
|
|
3166
|
+
root: Path,
|
|
3167
|
+
query_terms: set[str],
|
|
3168
|
+
context_lines: int,
|
|
3169
|
+
*,
|
|
3170
|
+
diagnostics: dict[str, Any] | None = None,
|
|
3171
|
+
) -> list[SuggestCandidate]:
|
|
2283
3172
|
if not query_terms:
|
|
2284
3173
|
return []
|
|
2285
3174
|
candidates: list[SuggestCandidate] = []
|
|
2286
|
-
for rel_path in git_ls_files(root):
|
|
3175
|
+
for rel_path in git_ls_files(root, diagnostics):
|
|
2287
3176
|
rel, reason = lexical_rel(rel_path)
|
|
2288
3177
|
if rel is None or reason:
|
|
2289
3178
|
continue
|
|
@@ -2373,16 +3262,28 @@ def suggested_source_payload(source: ResolvedSource, candidate: SuggestCandidate
|
|
|
2373
3262
|
return payload
|
|
2374
3263
|
|
|
2375
3264
|
|
|
2376
|
-
def normalize_suggest_source(
|
|
3265
|
+
def normalize_suggest_source(
|
|
3266
|
+
root: Path,
|
|
3267
|
+
candidate: SuggestCandidate,
|
|
3268
|
+
*,
|
|
3269
|
+
source_cache: _SourceSnapshotCache | None = None,
|
|
3270
|
+
input_budget: _SourceInputBudget | None = None,
|
|
3271
|
+
) -> tuple[ResolvedSource | None, dict[str, Any] | None]:
|
|
3272
|
+
effective_lines = candidate.lines or LineRange(1, SUGGEST_WHOLE_FILE_MAX_LINES)
|
|
2377
3273
|
spec = SourceSpec(
|
|
2378
3274
|
path=candidate.path,
|
|
2379
3275
|
priority=candidate.score,
|
|
2380
|
-
lines=
|
|
3276
|
+
lines=effective_lines,
|
|
2381
3277
|
label=candidate.label,
|
|
2382
3278
|
input_index=candidate.input_index,
|
|
2383
3279
|
origin="suggest",
|
|
2384
3280
|
)
|
|
2385
|
-
source, omitted_item = resolve_source(
|
|
3281
|
+
source, omitted_item = resolve_source(
|
|
3282
|
+
root,
|
|
3283
|
+
spec,
|
|
3284
|
+
source_cache=source_cache,
|
|
3285
|
+
input_budget=input_budget,
|
|
3286
|
+
)
|
|
2386
3287
|
if omitted_item is not None:
|
|
2387
3288
|
omitted_item["reason"] = omitted_item.get("reason") or candidate.reason
|
|
2388
3289
|
omitted_item["suggest_reason"] = candidate.reason
|
|
@@ -2390,20 +3291,6 @@ def normalize_suggest_source(root: Path, candidate: SuggestCandidate) -> tuple[R
|
|
|
2390
3291
|
assert source is not None
|
|
2391
3292
|
if source.redacted_path:
|
|
2392
3293
|
return None, omission(spec, "redacted_path", path=source.display_path, redacted_path=True)
|
|
2393
|
-
if spec.lines is None and source.total_lines > SUGGEST_WHOLE_FILE_MAX_LINES:
|
|
2394
|
-
capped = SourceSpec(
|
|
2395
|
-
path=candidate.path,
|
|
2396
|
-
priority=candidate.score,
|
|
2397
|
-
lines=LineRange(1, min(SUGGEST_WHOLE_FILE_MAX_LINES, source.total_lines)),
|
|
2398
|
-
label=candidate.label,
|
|
2399
|
-
input_index=candidate.input_index,
|
|
2400
|
-
origin="suggest",
|
|
2401
|
-
)
|
|
2402
|
-
source, omitted_item = resolve_source(root, capped)
|
|
2403
|
-
if omitted_item is not None:
|
|
2404
|
-
omitted_item["suggest_reason"] = candidate.reason
|
|
2405
|
-
return None, omitted_item
|
|
2406
|
-
assert source is not None
|
|
2407
3294
|
return source, None
|
|
2408
3295
|
|
|
2409
3296
|
|
|
@@ -2943,7 +3830,15 @@ def build_adaptive_k_advisory(
|
|
|
2943
3830
|
}
|
|
2944
3831
|
|
|
2945
3832
|
|
|
2946
|
-
def suggest_pack(
|
|
3833
|
+
def suggest_pack(
|
|
3834
|
+
root: Path,
|
|
3835
|
+
args: argparse.Namespace,
|
|
3836
|
+
*,
|
|
3837
|
+
root_arg: str,
|
|
3838
|
+
_source_cache: _SourceSnapshotCache | None = None,
|
|
3839
|
+
_input_budget: _SourceInputBudget | None = None,
|
|
3840
|
+
) -> tuple[dict[str, Any], int]:
|
|
3841
|
+
input_budget = _input_budget if _input_budget is not None else _SourceInputBudget()
|
|
2947
3842
|
query_text, _query_redactions = sanitize_text(args.query or "")
|
|
2948
3843
|
query = " ".join(query_text.split())
|
|
2949
3844
|
query_terms = suggest_tokens(query)
|
|
@@ -2973,7 +3868,23 @@ def suggest_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tupl
|
|
|
2973
3868
|
candidates.extend(test_candidates)
|
|
2974
3869
|
omitted.extend(output_omitted)
|
|
2975
3870
|
omitted.extend(test_omitted)
|
|
2976
|
-
|
|
3871
|
+
query_scan: dict[str, Any] = {}
|
|
3872
|
+
candidates.extend(
|
|
3873
|
+
collect_query_candidates(
|
|
3874
|
+
root,
|
|
3875
|
+
query_terms,
|
|
3876
|
+
context_lines,
|
|
3877
|
+
diagnostics=query_scan,
|
|
3878
|
+
)
|
|
3879
|
+
)
|
|
3880
|
+
if query_scan.get("truncated"):
|
|
3881
|
+
omitted.append({
|
|
3882
|
+
"path": "repository",
|
|
3883
|
+
"status": "omitted",
|
|
3884
|
+
"reason": "query_scan_truncated",
|
|
3885
|
+
"scan_truncation_reason": query_scan.get("truncation_reason"),
|
|
3886
|
+
"priority": 0,
|
|
3887
|
+
})
|
|
2977
3888
|
|
|
2978
3889
|
candidates.sort(key=lambda item: (-item.score, item.input_index, item.path, item.lines.identity() if item.lines else "0:0"))
|
|
2979
3890
|
seen: set[tuple[str, str]] = set()
|
|
@@ -3000,7 +3911,12 @@ def suggest_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tupl
|
|
|
3000
3911
|
continue
|
|
3001
3912
|
if rel is not None:
|
|
3002
3913
|
seen.add(identity)
|
|
3003
|
-
source, omitted_item = normalize_suggest_source(
|
|
3914
|
+
source, omitted_item = normalize_suggest_source(
|
|
3915
|
+
root,
|
|
3916
|
+
candidate,
|
|
3917
|
+
source_cache=_source_cache,
|
|
3918
|
+
input_budget=input_budget,
|
|
3919
|
+
)
|
|
3004
3920
|
if omitted_item is not None:
|
|
3005
3921
|
omitted_item["priority"] = candidate.score
|
|
3006
3922
|
omitted_item["suggest_reason"] = candidate.reason
|
|
@@ -3040,7 +3956,12 @@ def suggest_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tupl
|
|
|
3040
3956
|
input_index=candidate.input_index,
|
|
3041
3957
|
origin="suggest",
|
|
3042
3958
|
)
|
|
3043
|
-
source, omitted_item = resolve_source(
|
|
3959
|
+
source, omitted_item = resolve_source(
|
|
3960
|
+
root,
|
|
3961
|
+
partial_spec,
|
|
3962
|
+
source_cache=_source_cache,
|
|
3963
|
+
input_budget=input_budget,
|
|
3964
|
+
)
|
|
3044
3965
|
if omitted_item is not None:
|
|
3045
3966
|
omitted_item["priority"] = candidate.score
|
|
3046
3967
|
omitted_item["suggest_reason"] = candidate.reason
|
|
@@ -3099,6 +4020,17 @@ def suggest_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tupl
|
|
|
3099
4020
|
"Byte and token values are pack-size proxies, not billing claims.",
|
|
3100
4021
|
],
|
|
3101
4022
|
}
|
|
4023
|
+
if query_scan:
|
|
4024
|
+
payload["query_scan"] = {
|
|
4025
|
+
**query_scan,
|
|
4026
|
+
"limits": {
|
|
4027
|
+
"files": MAX_QUERY_SCAN_FILES,
|
|
4028
|
+
"directories": MAX_QUERY_WALK_DIRS,
|
|
4029
|
+
"entries": MAX_QUERY_WALK_ENTRIES,
|
|
4030
|
+
"depth": MAX_QUERY_WALK_DEPTH,
|
|
4031
|
+
"seconds": MAX_QUERY_WALK_SECONDS,
|
|
4032
|
+
},
|
|
4033
|
+
}
|
|
3102
4034
|
if build_hint_omitted_reason:
|
|
3103
4035
|
payload["build_hint_omitted_reason"] = build_hint_omitted_reason
|
|
3104
4036
|
if getattr(args, "adaptive_k", False):
|
|
@@ -3124,6 +4056,57 @@ def line_range_identity(value: object) -> str:
|
|
|
3124
4056
|
return str(value)
|
|
3125
4057
|
|
|
3126
4058
|
|
|
4059
|
+
def apply_adaptive_k_manifest(
|
|
4060
|
+
manifest: dict[str, Any],
|
|
4061
|
+
advisory: dict[str, Any],
|
|
4062
|
+
) -> tuple[dict[str, Any], dict[str, Any]]:
|
|
4063
|
+
raw_sources = manifest.get("sources", [])
|
|
4064
|
+
sources = copy.deepcopy(raw_sources) if isinstance(raw_sources, list) else []
|
|
4065
|
+
gates = advisory.get("regression_gates", {})
|
|
4066
|
+
gates_passed = isinstance(gates, dict) and gates.get("status") == "pass"
|
|
4067
|
+
recommended_k = max(0, int(advisory.get("recommended_k", 0) or 0))
|
|
4068
|
+
protected_prefixes = ("file:", "output:", "test-output:", "diff:")
|
|
4069
|
+
protected_indexes = {
|
|
4070
|
+
index
|
|
4071
|
+
for index, item in enumerate(sources)
|
|
4072
|
+
if isinstance(item, dict)
|
|
4073
|
+
and str(item.get("label", "")).startswith(protected_prefixes)
|
|
4074
|
+
}
|
|
4075
|
+
retained: list[dict[str, Any]] = []
|
|
4076
|
+
if gates_passed:
|
|
4077
|
+
target_count = max(recommended_k, len(protected_indexes))
|
|
4078
|
+
for index, item in enumerate(sources):
|
|
4079
|
+
if not isinstance(item, dict):
|
|
4080
|
+
continue
|
|
4081
|
+
if index in protected_indexes or len(retained) < target_count:
|
|
4082
|
+
retained.append(item)
|
|
4083
|
+
else:
|
|
4084
|
+
retained = [item for item in sources if isinstance(item, dict)]
|
|
4085
|
+
omitted_count = len(sources) - len(retained)
|
|
4086
|
+
status = "applied" if gates_passed and omitted_count else "no_change"
|
|
4087
|
+
if not gates_passed:
|
|
4088
|
+
status = "gate_failed"
|
|
4089
|
+
applied_manifest = {"version": 1, "sources": retained}
|
|
4090
|
+
return applied_manifest, {
|
|
4091
|
+
"schema_version": ADAPTIVE_K_APPLICATION_SCHEMA_VERSION,
|
|
4092
|
+
"mode": "explicit_opt_in",
|
|
4093
|
+
"status": status,
|
|
4094
|
+
"recommended_k": recommended_k,
|
|
4095
|
+
"input_source_count": len(sources),
|
|
4096
|
+
"applied_source_count": len(retained),
|
|
4097
|
+
"omitted_source_count": omitted_count,
|
|
4098
|
+
"regression_gates_passed": gates_passed,
|
|
4099
|
+
"explicit_sources_retained": all(
|
|
4100
|
+
sources[index] in retained for index in protected_indexes
|
|
4101
|
+
),
|
|
4102
|
+
"claim_boundary": {
|
|
4103
|
+
"deterministic_local_only": True,
|
|
4104
|
+
"exact_source_fallback_retained": True,
|
|
4105
|
+
"provider_token_or_cost_savings_claim_allowed": False,
|
|
4106
|
+
},
|
|
4107
|
+
}
|
|
4108
|
+
|
|
4109
|
+
|
|
3127
4110
|
def copy_explain_fields(item: dict[str, Any], fields: tuple[str, ...]) -> dict[str, Any]:
|
|
3128
4111
|
out: dict[str, Any] = {}
|
|
3129
4112
|
for field in fields:
|
|
@@ -3193,7 +4176,12 @@ def is_repo_map_text_path(path: str) -> bool:
|
|
|
3193
4176
|
return Path(path).suffix.lower() in REPO_MAP_TEXT_EXTENSIONS
|
|
3194
4177
|
|
|
3195
4178
|
|
|
3196
|
-
def read_repo_map_text(
|
|
4179
|
+
def read_repo_map_text(
|
|
4180
|
+
root: Path,
|
|
4181
|
+
rel_path: str,
|
|
4182
|
+
*,
|
|
4183
|
+
source_identities_out: dict[str, tuple[int, int, int, int, int, int, int, int]] | None = None,
|
|
4184
|
+
) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
|
|
3197
4185
|
rel, reason = lexical_rel(rel_path)
|
|
3198
4186
|
if rel is None:
|
|
3199
4187
|
return None, {"path": repo_map_safe_raw_path_label(rel_path), "reason": reason}
|
|
@@ -3205,14 +4193,24 @@ def read_repo_map_text(root: Path, rel_path: str) -> tuple[dict[str, Any] | None
|
|
|
3205
4193
|
return None, {"path": display, "reason": open_reason, "retrieval_omitted_reason": "redacted_path" if redacted_path else None}
|
|
3206
4194
|
try:
|
|
3207
4195
|
with handle:
|
|
4196
|
+
before_identity = _open_source_identity(handle)
|
|
3208
4197
|
text = handle.read(MAX_REPO_MAP_BYTES_PER_FILE + 1)
|
|
4198
|
+
after_identity = _open_source_identity(handle)
|
|
3209
4199
|
except (OSError, UnicodeError):
|
|
3210
4200
|
return None, {"path": display, "reason": "unsafe_path", "retrieval_omitted_reason": "redacted_path" if redacted_path else None}
|
|
4201
|
+
if before_identity is None or after_identity != before_identity:
|
|
4202
|
+
return None, {
|
|
4203
|
+
"path": display,
|
|
4204
|
+
"reason": "source_changed_during_repo_map",
|
|
4205
|
+
"retrieval_omitted_reason": "redacted_path" if redacted_path else None,
|
|
4206
|
+
}
|
|
3211
4207
|
capped = byte_len(text) > MAX_REPO_MAP_BYTES_PER_FILE
|
|
3212
4208
|
if capped:
|
|
3213
4209
|
text = text.encode("utf-8", errors="replace")[:MAX_REPO_MAP_BYTES_PER_FILE].decode("utf-8", errors="ignore")
|
|
3214
4210
|
risk_counts = secret_risk_counts(text)
|
|
3215
4211
|
sanitized_text, redacted_lines = sanitize_text(text)
|
|
4212
|
+
if source_identities_out is not None:
|
|
4213
|
+
source_identities_out[rel.as_posix()] = before_identity
|
|
3216
4214
|
return {
|
|
3217
4215
|
"path": display,
|
|
3218
4216
|
"raw_path": rel.as_posix(),
|
|
@@ -3251,7 +4249,13 @@ def repo_map_scan_paths(paths: list[str], *, seed_paths: set[str], query_terms:
|
|
|
3251
4249
|
return [path for _index, path in ranked[:MAX_REPO_MAP_SCAN_FILES]]
|
|
3252
4250
|
|
|
3253
4251
|
|
|
3254
|
-
def repo_map_records(
|
|
4252
|
+
def repo_map_records(
|
|
4253
|
+
root: Path,
|
|
4254
|
+
*,
|
|
4255
|
+
seed_paths: set[str],
|
|
4256
|
+
query_terms: set[str],
|
|
4257
|
+
source_identities_out: dict[str, tuple[int, int, int, int, int, int, int, int]] | None = None,
|
|
4258
|
+
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], dict[str, Any]]:
|
|
3255
4259
|
paths = git_ls_files(root)
|
|
3256
4260
|
candidate_paths = paths[:MAX_REPO_MAP_FILES]
|
|
3257
4261
|
path_cap_reached = len(paths) > MAX_REPO_MAP_FILES
|
|
@@ -3260,7 +4264,14 @@ def repo_map_records(root: Path, *, seed_paths: set[str], query_terms: set[str])
|
|
|
3260
4264
|
records: list[dict[str, Any]] = []
|
|
3261
4265
|
omitted: list[dict[str, Any]] = []
|
|
3262
4266
|
for rel_path in scan_paths:
|
|
3263
|
-
|
|
4267
|
+
if source_identities_out is None:
|
|
4268
|
+
record, omission_item = read_repo_map_text(root, rel_path)
|
|
4269
|
+
else:
|
|
4270
|
+
record, omission_item = read_repo_map_text(
|
|
4271
|
+
root,
|
|
4272
|
+
rel_path,
|
|
4273
|
+
source_identities_out=source_identities_out,
|
|
4274
|
+
)
|
|
3264
4275
|
if record is not None:
|
|
3265
4276
|
records.append(record)
|
|
3266
4277
|
elif omission_item is not None and omission_item.get("reason") != "unsupported_file_type":
|
|
@@ -3522,9 +4533,18 @@ def build_graph_rank(
|
|
|
3522
4533
|
query_terms: set[str],
|
|
3523
4534
|
seed_paths: set[str],
|
|
3524
4535
|
secret_scan: dict[str, Any],
|
|
4536
|
+
complete_secret_paths: set[str] | None = None,
|
|
3525
4537
|
) -> list[dict[str, Any]]:
|
|
3526
4538
|
signature_paths = {str(item.get("path", "")) for item in signatures}
|
|
3527
|
-
secret_paths =
|
|
4539
|
+
secret_paths = (
|
|
4540
|
+
complete_secret_paths
|
|
4541
|
+
if complete_secret_paths is not None
|
|
4542
|
+
else {
|
|
4543
|
+
str(item.get("path", ""))
|
|
4544
|
+
for item in secret_scan.get("files_with_risks", [])
|
|
4545
|
+
if isinstance(item, dict)
|
|
4546
|
+
}
|
|
4547
|
+
)
|
|
3528
4548
|
degree: dict[str, int] = {}
|
|
3529
4549
|
for edge in edges:
|
|
3530
4550
|
degree[edge["from"]] = degree.get(edge["from"], 0) + 1
|
|
@@ -3621,13 +4641,27 @@ def build_repo_map_payload(
|
|
|
3621
4641
|
build_payload: dict[str, Any],
|
|
3622
4642
|
*,
|
|
3623
4643
|
root_arg: str,
|
|
4644
|
+
complete_secret_paths_out: set[str] | None = None,
|
|
4645
|
+
source_identities_out: dict[str, tuple[int, int, int, int, int, int, int, int]] | None = None,
|
|
3624
4646
|
) -> dict[str, Any]:
|
|
3625
4647
|
query_terms = suggest_tokens(str(suggest_payload.get("query", "")))
|
|
3626
4648
|
seed_paths = repo_map_seed_paths(args, suggest_payload, build_payload)
|
|
3627
|
-
records, omitted, caps = repo_map_records(
|
|
4649
|
+
records, omitted, caps = repo_map_records(
|
|
4650
|
+
root,
|
|
4651
|
+
seed_paths=seed_paths,
|
|
4652
|
+
query_terms=query_terms,
|
|
4653
|
+
source_identities_out=source_identities_out,
|
|
4654
|
+
)
|
|
3628
4655
|
record_by_path = {str(record["path"]): record for record in records}
|
|
3629
4656
|
signatures = extract_signatures(records)
|
|
3630
4657
|
secret_scan = build_secret_scan(records)
|
|
4658
|
+
complete_secret_paths = {
|
|
4659
|
+
str(record.get("path", ""))
|
|
4660
|
+
for record in records
|
|
4661
|
+
if record.get("secret_risk_counts")
|
|
4662
|
+
}
|
|
4663
|
+
if complete_secret_paths_out is not None:
|
|
4664
|
+
complete_secret_paths_out.update(complete_secret_paths)
|
|
3631
4665
|
edges = collect_import_edges(records)
|
|
3632
4666
|
graph_rank = build_graph_rank(
|
|
3633
4667
|
records,
|
|
@@ -3636,6 +4670,7 @@ def build_repo_map_payload(
|
|
|
3636
4670
|
query_terms=query_terms,
|
|
3637
4671
|
seed_paths=seed_paths,
|
|
3638
4672
|
secret_scan=secret_scan,
|
|
4673
|
+
complete_secret_paths=complete_secret_paths,
|
|
3639
4674
|
)
|
|
3640
4675
|
retrieval = repo_map_retrieval(record_by_path, signatures, graph_rank, root_arg=root_arg)
|
|
3641
4676
|
tree = build_token_tree(records)
|
|
@@ -3685,7 +4720,153 @@ def line_identity_from_dict(value: object) -> str:
|
|
|
3685
4720
|
return f"{value.get('start')}:{value.get('end')}"
|
|
3686
4721
|
|
|
3687
4722
|
|
|
3688
|
-
def
|
|
4723
|
+
def apply_symbol_memory_graph(
|
|
4724
|
+
manifest: dict[str, Any],
|
|
4725
|
+
repo_map: dict[str, Any],
|
|
4726
|
+
*,
|
|
4727
|
+
complete_secret_paths: set[str] | None = None,
|
|
4728
|
+
) -> tuple[dict[str, Any], dict[str, Any]]:
|
|
4729
|
+
"""Add bounded direct graph neighbors to an explicit auto-pack manifest."""
|
|
4730
|
+
|
|
4731
|
+
raw_sources = manifest.get("sources")
|
|
4732
|
+
if not isinstance(raw_sources, list):
|
|
4733
|
+
raise PackError("manifest sources must be a list")
|
|
4734
|
+
existing_sources = [
|
|
4735
|
+
copy.deepcopy(item) for item in raw_sources if isinstance(item, dict)
|
|
4736
|
+
]
|
|
4737
|
+
existing_paths = {
|
|
4738
|
+
str(item.get("path", "")) for item in existing_sources if item.get("path")
|
|
4739
|
+
}
|
|
4740
|
+
graph = repo_map.get("graph") if isinstance(repo_map.get("graph"), dict) else {}
|
|
4741
|
+
edges = graph.get("edges") if isinstance(graph.get("edges"), list) else []
|
|
4742
|
+
rank_items = (
|
|
4743
|
+
repo_map.get("graph_rank")
|
|
4744
|
+
if isinstance(repo_map.get("graph_rank"), list)
|
|
4745
|
+
else []
|
|
4746
|
+
)
|
|
4747
|
+
rank_by_path = {
|
|
4748
|
+
str(item.get("path", "")): item
|
|
4749
|
+
for item in rank_items
|
|
4750
|
+
if isinstance(item, dict) and item.get("path")
|
|
4751
|
+
}
|
|
4752
|
+
secret_scan = (
|
|
4753
|
+
repo_map.get("secret_scan")
|
|
4754
|
+
if isinstance(repo_map.get("secret_scan"), dict)
|
|
4755
|
+
else {}
|
|
4756
|
+
)
|
|
4757
|
+
risky_paths = (
|
|
4758
|
+
complete_secret_paths
|
|
4759
|
+
if complete_secret_paths is not None
|
|
4760
|
+
else {
|
|
4761
|
+
str(item.get("path", ""))
|
|
4762
|
+
for item in secret_scan.get("files_with_risks", [])
|
|
4763
|
+
if isinstance(item, dict) and item.get("path")
|
|
4764
|
+
}
|
|
4765
|
+
)
|
|
4766
|
+
direct_neighbors: set[str] = set()
|
|
4767
|
+
for edge in edges:
|
|
4768
|
+
if not isinstance(edge, dict):
|
|
4769
|
+
continue
|
|
4770
|
+
source = edge.get("from")
|
|
4771
|
+
target = edge.get("to")
|
|
4772
|
+
if not isinstance(source, str) or not isinstance(target, str):
|
|
4773
|
+
continue
|
|
4774
|
+
if source in existing_paths and target not in existing_paths:
|
|
4775
|
+
direct_neighbors.add(target)
|
|
4776
|
+
if target in existing_paths and source not in existing_paths:
|
|
4777
|
+
direct_neighbors.add(source)
|
|
4778
|
+
|
|
4779
|
+
eligible: list[tuple[int, str, int]] = []
|
|
4780
|
+
excluded_secret_risk_count = 0
|
|
4781
|
+
for path in direct_neighbors:
|
|
4782
|
+
if path in risky_paths:
|
|
4783
|
+
excluded_secret_risk_count += 1
|
|
4784
|
+
continue
|
|
4785
|
+
item = rank_by_path.get(path)
|
|
4786
|
+
if item is None or repo_map_path_has_sensitive_evidence(path):
|
|
4787
|
+
continue
|
|
4788
|
+
score = int(item.get("score", 0) or 0)
|
|
4789
|
+
line_count = int(item.get("line_count", 0) or 0)
|
|
4790
|
+
if score <= 0 or line_count <= 0:
|
|
4791
|
+
continue
|
|
4792
|
+
eligible.append((score, path, line_count))
|
|
4793
|
+
eligible.sort(key=lambda item: (-item[0], item[1]))
|
|
4794
|
+
|
|
4795
|
+
seed_priorities = [
|
|
4796
|
+
int(item.get("priority", 0) or 0) for item in existing_sources
|
|
4797
|
+
]
|
|
4798
|
+
maximum_graph_priority = max(1, min(seed_priorities, default=2) - 1)
|
|
4799
|
+
selected_sources: list[dict[str, Any]] = []
|
|
4800
|
+
for score, path, line_count in eligible[:MAX_GRAPH_APPLICATION_SOURCES]:
|
|
4801
|
+
source = {
|
|
4802
|
+
"path": path,
|
|
4803
|
+
"priority": max(1, min(score, maximum_graph_priority)),
|
|
4804
|
+
"label": f"graph:{path}"[:MAX_LABEL_CHARS],
|
|
4805
|
+
"lines": {"start": 1, "end": min(line_count, MAX_GRAPH_APPLICATION_LINES)},
|
|
4806
|
+
}
|
|
4807
|
+
existing_sources.append(source)
|
|
4808
|
+
selected_sources.append(
|
|
4809
|
+
{
|
|
4810
|
+
"path": source["path"],
|
|
4811
|
+
"priority": source["priority"],
|
|
4812
|
+
"lines": copy.deepcopy(source["lines"]),
|
|
4813
|
+
"reason": "direct_import_neighbor",
|
|
4814
|
+
}
|
|
4815
|
+
)
|
|
4816
|
+
result_manifest = build_suggest_manifest(existing_sources)
|
|
4817
|
+
return result_manifest, {
|
|
4818
|
+
"schema_version": GRAPH_APPLICATION_SCHEMA_VERSION,
|
|
4819
|
+
"mode": "explicit_opt_in",
|
|
4820
|
+
"selected_source_count": len(selected_sources),
|
|
4821
|
+
"selected_sources": selected_sources,
|
|
4822
|
+
"candidate_count": len(eligible),
|
|
4823
|
+
"candidate_cap": MAX_GRAPH_APPLICATION_SOURCES,
|
|
4824
|
+
"candidate_cap_reached": len(eligible) > MAX_GRAPH_APPLICATION_SOURCES,
|
|
4825
|
+
"excluded_secret_risk_count": excluded_secret_risk_count,
|
|
4826
|
+
"exact_source_fallback_retained": True,
|
|
4827
|
+
"deterministic_local_only": True,
|
|
4828
|
+
"provider_token_or_cost_savings_claim_allowed": False,
|
|
4829
|
+
}
|
|
4830
|
+
|
|
4831
|
+
|
|
4832
|
+
def bind_graph_sources_to_repo_snapshot(
|
|
4833
|
+
root: Path,
|
|
4834
|
+
specs: list[SourceSpec],
|
|
4835
|
+
required_sources: set[tuple[str, str]],
|
|
4836
|
+
source_identities: dict[str, tuple[int, int, int, int, int, int, int, int]],
|
|
4837
|
+
*,
|
|
4838
|
+
source_cache: _SourceSnapshotCache,
|
|
4839
|
+
input_budget: _SourceInputBudget,
|
|
4840
|
+
) -> dict[tuple[str, str], dict[str, Any]]:
|
|
4841
|
+
"""Warm immutable source snapshots only for repo-map-bound graph additions."""
|
|
4842
|
+
|
|
4843
|
+
rejections: dict[tuple[str, str], dict[str, Any]] = {}
|
|
4844
|
+
for spec in specs:
|
|
4845
|
+
rel, _reason = lexical_rel(spec.path)
|
|
4846
|
+
if rel is None:
|
|
4847
|
+
continue
|
|
4848
|
+
lines_identity = spec.lines.identity() if spec.lines is not None else "all"
|
|
4849
|
+
source_identity = (rel.as_posix(), lines_identity)
|
|
4850
|
+
if source_identity not in required_sources:
|
|
4851
|
+
continue
|
|
4852
|
+
expected_identity = source_identities.get(rel.as_posix())
|
|
4853
|
+
if expected_identity is None:
|
|
4854
|
+
continue
|
|
4855
|
+
_source, omitted_item = resolve_source(
|
|
4856
|
+
root,
|
|
4857
|
+
spec,
|
|
4858
|
+
source_cache=source_cache,
|
|
4859
|
+
input_budget=input_budget,
|
|
4860
|
+
expected_identity=expected_identity,
|
|
4861
|
+
)
|
|
4862
|
+
if omitted_item is not None:
|
|
4863
|
+
rejections[source_identity] = copy.deepcopy(omitted_item)
|
|
4864
|
+
return rejections
|
|
4865
|
+
|
|
4866
|
+
|
|
4867
|
+
def build_symbol_memory_payload(
|
|
4868
|
+
repo_map: dict[str, Any], *, applied: bool = False
|
|
4869
|
+
) -> dict[str, Any]:
|
|
3689
4870
|
retrieval_by_path_lines: dict[tuple[str, str], dict[str, Any]] = {}
|
|
3690
4871
|
for item in repo_map.get("retrieval", []):
|
|
3691
4872
|
if not isinstance(item, dict):
|
|
@@ -3736,7 +4917,7 @@ def build_symbol_memory_payload(repo_map: dict[str, Any]) -> dict[str, Any]:
|
|
|
3736
4917
|
retrieval = repo_map.get("retrieval", []) if isinstance(repo_map.get("retrieval"), list) else []
|
|
3737
4918
|
return {
|
|
3738
4919
|
"schema_version": SYMBOL_MEMORY_SCHEMA_VERSION,
|
|
3739
|
-
"mode": "advisory",
|
|
4920
|
+
"mode": "applied" if applied else "advisory",
|
|
3740
4921
|
"source": "contextguard.pack-repo-map.v1",
|
|
3741
4922
|
"summary": {
|
|
3742
4923
|
"symbols": len(symbols),
|
|
@@ -3756,8 +4937,9 @@ def build_symbol_memory_payload(repo_map: dict[str, Any]) -> dict[str, Any]:
|
|
|
3756
4937
|
"claim_boundary": {
|
|
3757
4938
|
"deterministic_local_only": True,
|
|
3758
4939
|
"no_network_model_embedding_lsp_or_tree_sitter_dependency": True,
|
|
3759
|
-
"advisory_does_not_change_manifest_pack_or_receipt":
|
|
3760
|
-
"
|
|
4940
|
+
"advisory_does_not_change_manifest_pack_or_receipt": not applied,
|
|
4941
|
+
"explicit_graph_application_changes_manifest_and_pack": applied,
|
|
4942
|
+
"graph_rank_is_explain_only": not applied,
|
|
3761
4943
|
"provider_token_or_cost_savings_claim_allowed": False,
|
|
3762
4944
|
},
|
|
3763
4945
|
}
|
|
@@ -3903,10 +5085,18 @@ def build_auto_explain_payload(
|
|
|
3903
5085
|
explain["repo_map"] = copy.deepcopy(repo_map_payload)
|
|
3904
5086
|
elif root is not None:
|
|
3905
5087
|
explain["repo_map"] = build_repo_map_payload(root, args, suggest_payload, build_payload, root_arg=root_arg)
|
|
5088
|
+
if isinstance(payload.get("graph_application"), dict):
|
|
5089
|
+
explain["graph_application"] = copy.deepcopy(payload["graph_application"])
|
|
5090
|
+
if isinstance(payload.get("adaptive_k_application"), dict):
|
|
5091
|
+
explain["adaptive_k_application"] = copy.deepcopy(
|
|
5092
|
+
payload["adaptive_k_application"]
|
|
5093
|
+
)
|
|
3906
5094
|
return explain
|
|
3907
5095
|
|
|
3908
5096
|
|
|
3909
5097
|
def auto_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tuple[dict[str, Any], int]:
|
|
5098
|
+
source_cache = _SourceSnapshotCache()
|
|
5099
|
+
input_budget = _SourceInputBudget()
|
|
3910
5100
|
manifest_rel = output_rel_for_collision_check(args.manifest_out, "--manifest-out") if args.manifest_out else None
|
|
3911
5101
|
pack_rel = output_rel_for_collision_check(args.pack_out, "--pack-out") if args.pack_out else None
|
|
3912
5102
|
if manifest_rel is not None and pack_rel is not None:
|
|
@@ -3923,8 +5113,39 @@ def auto_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tuple[d
|
|
|
3923
5113
|
validate_output_path_under_root(root, args.pack_out, "--pack-out")
|
|
3924
5114
|
suggest_args = copy.copy(args)
|
|
3925
5115
|
suggest_args.manifest_out = None
|
|
3926
|
-
|
|
5116
|
+
apply_adaptive_k = bool(getattr(args, "apply_adaptive_k", False))
|
|
5117
|
+
if apply_adaptive_k:
|
|
5118
|
+
suggest_args.adaptive_k = True
|
|
5119
|
+
suggest_payload, rc = suggest_pack(
|
|
5120
|
+
root,
|
|
5121
|
+
suggest_args,
|
|
5122
|
+
root_arg=root_arg,
|
|
5123
|
+
_source_cache=source_cache,
|
|
5124
|
+
_input_budget=input_budget,
|
|
5125
|
+
)
|
|
3927
5126
|
manifest = suggest_payload["manifest"]
|
|
5127
|
+
adaptive_k_application: dict[str, Any] | None = None
|
|
5128
|
+
if apply_adaptive_k and isinstance(suggest_payload.get("adaptive_k"), dict):
|
|
5129
|
+
manifest, adaptive_k_application = apply_adaptive_k_manifest(
|
|
5130
|
+
manifest,
|
|
5131
|
+
suggest_payload["adaptive_k"],
|
|
5132
|
+
)
|
|
5133
|
+
suggest_payload["manifest"] = manifest
|
|
5134
|
+
retained_identities = {
|
|
5135
|
+
(str(item.get("path", "")), line_range_identity(item.get("lines")))
|
|
5136
|
+
for item in manifest.get("sources", [])
|
|
5137
|
+
if isinstance(item, dict)
|
|
5138
|
+
}
|
|
5139
|
+
suggest_payload["sources"] = [
|
|
5140
|
+
item
|
|
5141
|
+
for item in suggest_payload.get("sources", [])
|
|
5142
|
+
if isinstance(item, dict)
|
|
5143
|
+
and (
|
|
5144
|
+
str(item.get("path", "")),
|
|
5145
|
+
line_range_identity(item.get("lines")),
|
|
5146
|
+
)
|
|
5147
|
+
in retained_identities
|
|
5148
|
+
]
|
|
3928
5149
|
specs = manifest_to_source_specs(manifest)
|
|
3929
5150
|
budget = bounded_int(args.budget_bytes, DEFAULT_BUDGET_BYTES, MIN_BUDGET_BYTES, MAX_BUDGET_BYTES)
|
|
3930
5151
|
build_payload = build_pack(
|
|
@@ -3935,7 +5156,83 @@ def auto_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tuple[d
|
|
|
3935
5156
|
store_artifact=False,
|
|
3936
5157
|
delta_from_pack_id=args.delta_from_pack_id,
|
|
3937
5158
|
sketch_duplicate_veto=getattr(args, "sketch_duplicate_veto", False),
|
|
5159
|
+
_source_cache=source_cache,
|
|
5160
|
+
_input_budget=input_budget,
|
|
3938
5161
|
)
|
|
5162
|
+
if apply_adaptive_k:
|
|
5163
|
+
suggest_payload["estimated_pack_bytes"] = build_payload.get("pack_bytes", 0)
|
|
5164
|
+
suggest_payload["token_proxy"] = copy.deepcopy(
|
|
5165
|
+
build_payload.get("token_proxy", {})
|
|
5166
|
+
)
|
|
5167
|
+
repo_map_payload: dict[str, Any] | None = None
|
|
5168
|
+
graph_application: dict[str, Any] | None = None
|
|
5169
|
+
apply_symbol_memory = bool(getattr(args, "apply_symbol_memory", False))
|
|
5170
|
+
complete_secret_paths: set[str] | None = set() if apply_symbol_memory else None
|
|
5171
|
+
repo_map_source_identities: dict[
|
|
5172
|
+
str,
|
|
5173
|
+
tuple[int, int, int, int, int, int, int, int],
|
|
5174
|
+
] = {}
|
|
5175
|
+
if getattr(args, "symbol_memory", False) or apply_symbol_memory or args.explain:
|
|
5176
|
+
repo_map_payload = build_repo_map_payload(
|
|
5177
|
+
root,
|
|
5178
|
+
args,
|
|
5179
|
+
suggest_payload,
|
|
5180
|
+
build_payload,
|
|
5181
|
+
root_arg=root_arg,
|
|
5182
|
+
complete_secret_paths_out=complete_secret_paths,
|
|
5183
|
+
source_identities_out=(
|
|
5184
|
+
repo_map_source_identities if apply_symbol_memory else None
|
|
5185
|
+
),
|
|
5186
|
+
)
|
|
5187
|
+
if apply_symbol_memory and isinstance(repo_map_payload, dict):
|
|
5188
|
+
repo_map_payload["safety"]["explain_only"] = False
|
|
5189
|
+
repo_map_payload["safety"]["caveats"] = [
|
|
5190
|
+
"Repo-map bytes are local sampled UTF-8 bytes and estimated chars_div_4 token proxies, not provider-token or savings claims.",
|
|
5191
|
+
"Graph ranking is applied only to the bounded direct-neighbor expansion recorded in graph_application; exact source retrieval remains available.",
|
|
5192
|
+
]
|
|
5193
|
+
pre_graph_sources = {
|
|
5194
|
+
(str(item.get("path", "")), line_range_identity(item.get("lines")))
|
|
5195
|
+
for item in manifest.get("sources", [])
|
|
5196
|
+
if isinstance(item, dict) and item.get("path")
|
|
5197
|
+
}
|
|
5198
|
+
manifest, graph_application = apply_symbol_memory_graph(
|
|
5199
|
+
manifest,
|
|
5200
|
+
repo_map_payload,
|
|
5201
|
+
complete_secret_paths=complete_secret_paths,
|
|
5202
|
+
)
|
|
5203
|
+
suggest_payload["manifest"] = manifest
|
|
5204
|
+
specs = manifest_to_source_specs(manifest)
|
|
5205
|
+
graph_snapshot_sources = {
|
|
5206
|
+
(str(item.get("path", "")), line_range_identity(item.get("lines")))
|
|
5207
|
+
for item in manifest.get("sources", [])
|
|
5208
|
+
if isinstance(item, dict) and item.get("path")
|
|
5209
|
+
} - pre_graph_sources
|
|
5210
|
+
graph_snapshot_rejections = bind_graph_sources_to_repo_snapshot(
|
|
5211
|
+
root,
|
|
5212
|
+
specs,
|
|
5213
|
+
graph_snapshot_sources,
|
|
5214
|
+
repo_map_source_identities,
|
|
5215
|
+
source_cache=source_cache,
|
|
5216
|
+
input_budget=input_budget,
|
|
5217
|
+
)
|
|
5218
|
+
build_payload = build_pack(
|
|
5219
|
+
root,
|
|
5220
|
+
specs,
|
|
5221
|
+
budget_bytes=budget,
|
|
5222
|
+
root_arg=root_arg,
|
|
5223
|
+
store_artifact=False,
|
|
5224
|
+
delta_from_pack_id=args.delta_from_pack_id,
|
|
5225
|
+
sketch_duplicate_veto=getattr(args, "sketch_duplicate_veto", False),
|
|
5226
|
+
_source_cache=source_cache,
|
|
5227
|
+
_input_budget=input_budget,
|
|
5228
|
+
_required_snapshot_sources=graph_snapshot_sources,
|
|
5229
|
+
_expected_source_identities=repo_map_source_identities,
|
|
5230
|
+
_snapshot_rejections=graph_snapshot_rejections,
|
|
5231
|
+
)
|
|
5232
|
+
suggest_payload["estimated_pack_bytes"] = build_payload.get("pack_bytes", 0)
|
|
5233
|
+
suggest_payload["token_proxy"] = copy.deepcopy(
|
|
5234
|
+
build_payload.get("token_proxy", {})
|
|
5235
|
+
)
|
|
3939
5236
|
if not args.no_artifact:
|
|
3940
5237
|
receipt_rel = Path(PACK_DIR) / f"{build_payload['pack_id']}.json"
|
|
3941
5238
|
if manifest_rel is not None:
|
|
@@ -3982,7 +5279,7 @@ def auto_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tuple[d
|
|
|
3982
5279
|
"suggest": suggest_payload,
|
|
3983
5280
|
"build": build_payload,
|
|
3984
5281
|
"sources": {
|
|
3985
|
-
"suggested": len(
|
|
5282
|
+
"suggested": len(manifest.get("sources", [])),
|
|
3986
5283
|
"included": build_payload.get("sources", {}).get("included", 0),
|
|
3987
5284
|
"partial": build_payload.get("sources", {}).get("partial", 0),
|
|
3988
5285
|
"omitted": build_payload.get("sources", {}).get("omitted", 0),
|
|
@@ -3996,13 +5293,18 @@ def auto_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tuple[d
|
|
|
3996
5293
|
}
|
|
3997
5294
|
if build_hint_omitted_reason:
|
|
3998
5295
|
payload["build_hint_omitted_reason"] = build_hint_omitted_reason
|
|
3999
|
-
if getattr(args, "adaptive_k", False) and isinstance(
|
|
5296
|
+
if (getattr(args, "adaptive_k", False) or apply_adaptive_k) and isinstance(
|
|
5297
|
+
suggest_payload.get("adaptive_k"), dict
|
|
5298
|
+
):
|
|
4000
5299
|
payload["adaptive_k"] = copy.deepcopy(suggest_payload["adaptive_k"])
|
|
4001
|
-
|
|
4002
|
-
|
|
4003
|
-
|
|
4004
|
-
|
|
4005
|
-
|
|
5300
|
+
if adaptive_k_application is not None:
|
|
5301
|
+
payload["adaptive_k_application"] = adaptive_k_application
|
|
5302
|
+
if graph_application is not None:
|
|
5303
|
+
payload["graph_application"] = graph_application
|
|
5304
|
+
if (getattr(args, "symbol_memory", False) or apply_symbol_memory) and isinstance(repo_map_payload, dict):
|
|
5305
|
+
payload["symbol_memory"] = build_symbol_memory_payload(
|
|
5306
|
+
repo_map_payload, applied=apply_symbol_memory
|
|
5307
|
+
)
|
|
4006
5308
|
if args.explain:
|
|
4007
5309
|
payload["explain"] = build_auto_explain_payload(
|
|
4008
5310
|
args,
|
|
@@ -4038,6 +5340,8 @@ def print_adaptive_k_text(payload: dict[str, Any]) -> None:
|
|
|
4038
5340
|
reason_text = ",".join(str(item) for item in reason_codes[:5])
|
|
4039
5341
|
else:
|
|
4040
5342
|
reason_text = str(reason_codes)
|
|
5343
|
+
application = payload.get("adaptive_k_application")
|
|
5344
|
+
applied = isinstance(application, dict) and application.get("status") == "applied"
|
|
4041
5345
|
print(
|
|
4042
5346
|
"adaptive-k: "
|
|
4043
5347
|
f"recommended={adaptive.get('recommended_k', 0)}/{adaptive.get('requested_top', 0)} "
|
|
@@ -4045,7 +5349,7 @@ def print_adaptive_k_text(payload: dict[str, Any]) -> None:
|
|
|
4045
5349
|
f"gates={regression_gates.get('status', 'pass')} "
|
|
4046
5350
|
f"candidates={score_distribution.get('candidate_count', 0)} "
|
|
4047
5351
|
f"budget_limited={budget_fit.get('budget_limited', False)} "
|
|
4048
|
-
f"apply=
|
|
5352
|
+
f"apply={str(applied).lower()} reasons={reason_text or 'none'}"
|
|
4049
5353
|
)
|
|
4050
5354
|
|
|
4051
5355
|
|
|
@@ -4224,10 +5528,27 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
4224
5528
|
)
|
|
4225
5529
|
auto.add_argument("--explain", action="store_true", help="include deterministic local selection/build explanation metadata")
|
|
4226
5530
|
auto.add_argument("--adaptive-k", action="store_true", help="include local score/budget top-k advisory metadata without changing the manifest or pack")
|
|
5531
|
+
auto.add_argument(
|
|
5532
|
+
"--apply-adaptive-k",
|
|
5533
|
+
action="store_true",
|
|
5534
|
+
help=(
|
|
5535
|
+
"explicitly prune heuristic-selected sources to the locally recommended top-k "
|
|
5536
|
+
"after regression gates pass while always retaining explicit file/output/diff sources; "
|
|
5537
|
+
"implies --adaptive-k"
|
|
5538
|
+
),
|
|
5539
|
+
)
|
|
4227
5540
|
auto.add_argument("--adaptive-k-policy", choices=ADAPTIVE_K_POLICIES, default="balanced", help="local adaptive-k recommendation policy used when --adaptive-k is set")
|
|
4228
5541
|
auto.add_argument("--adaptive-k-min-recall-proxy", type=adaptive_k_threshold, default=0.0, help="metadata-only minimum recall proxy gate for --adaptive-k")
|
|
4229
5542
|
auto.add_argument("--adaptive-k-min-precision-proxy", type=adaptive_k_threshold, default=0.0, help="metadata-only minimum precision proxy gate for --adaptive-k")
|
|
4230
5543
|
auto.add_argument("--symbol-memory", action="store_true", help="include repo-map derived symbol/graph advisory metadata with exact source verification hints")
|
|
5544
|
+
auto.add_argument(
|
|
5545
|
+
"--apply-symbol-memory",
|
|
5546
|
+
action="store_true",
|
|
5547
|
+
help=(
|
|
5548
|
+
"explicitly add up to four direct import-neighbor slices from the local "
|
|
5549
|
+
"repo map to the manifest and pack; implies --symbol-memory"
|
|
5550
|
+
),
|
|
5551
|
+
)
|
|
4231
5552
|
return parser
|
|
4232
5553
|
|
|
4233
5554
|
|