@ictechgy/context-guard 0.4.15 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +80 -0
- package/README.ko.md +128 -2
- package/README.md +144 -3
- package/docs/distribution.md +100 -0
- package/package.json +4 -1
- package/plugins/context-guard/.claude-plugin/plugin.json +1 -1
- package/plugins/context-guard/README.ko.md +43 -1
- package/plugins/context-guard/README.md +44 -1
- package/plugins/context-guard/bin/bash_reference_policy.py +967 -0
- package/plugins/context-guard/bin/context-guard-artifact +90 -9
- package/plugins/context-guard/bin/context-guard-audit +169 -66
- package/plugins/context-guard/bin/context-guard-bench +9865 -211
- package/plugins/context-guard/bin/context-guard-compress +90 -8
- package/plugins/context-guard/bin/context-guard-diet +1 -7
- package/plugins/context-guard/bin/context-guard-experiments +5 -1
- package/plugins/context-guard/bin/context-guard-failed-nudge +777 -83
- package/plugins/context-guard/bin/context-guard-guard-read +496 -57
- package/plugins/context-guard/bin/context-guard-mcp +2 -1
- package/plugins/context-guard/bin/context-guard-pack +1570 -150
- package/plugins/context-guard/bin/context-guard-read-symbol +7 -2
- package/plugins/context-guard/bin/context-guard-rewrite-bash +2669 -236
- package/plugins/context-guard/bin/context-guard-sanitize-output +723 -92
- package/plugins/context-guard/bin/context-guard-setup +1944 -222
- package/plugins/context-guard/bin/context-guard-statusline +163 -55
- package/plugins/context-guard/bin/context-guard-statusline-merged +78 -23
- package/plugins/context-guard/bin/context-guard-tool-prune +44 -11
- package/plugins/context-guard/bin/context-guard-trim-output +795 -48
- package/plugins/context-guard/brief/README.md +19 -0
- package/plugins/context-guard/brief/narration-mode.quiet.md +21 -0
- package/plugins/context-guard/lib/context_guard_commands.py +10 -2
- package/plugins/context-guard/lib/credential_policy.py +185 -0
- package/plugins/context-guard/lib/transcript_usage_reducer.py +378 -0
|
@@ -23,6 +23,7 @@ import posixpath
|
|
|
23
23
|
from pathlib import Path
|
|
24
24
|
import re
|
|
25
25
|
import shlex
|
|
26
|
+
import signal
|
|
26
27
|
import stat
|
|
27
28
|
import subprocess
|
|
28
29
|
import sys
|
|
@@ -48,7 +49,9 @@ AUTO_SCHEMA_VERSION = "contextguard.pack-auto.v1"
|
|
|
48
49
|
AUTO_EXPLAIN_SCHEMA_VERSION = "contextguard.pack-auto-explain.v1"
|
|
49
50
|
REPO_MAP_SCHEMA_VERSION = "contextguard.pack-repo-map.v1"
|
|
50
51
|
ADAPTIVE_K_SCHEMA_VERSION = "contextguard.pack-adaptive-k.v1"
|
|
52
|
+
ADAPTIVE_K_APPLICATION_SCHEMA_VERSION = "contextguard.pack-adaptive-k-application.v1"
|
|
51
53
|
SYMBOL_MEMORY_SCHEMA_VERSION = "contextguard.pack-symbol-memory.v1"
|
|
54
|
+
GRAPH_APPLICATION_SCHEMA_VERSION = "contextguard.pack-graph-application.v1"
|
|
52
55
|
CONTENT_ADDRESS_SCHEMA_VERSION = "contextguard.pack-content-address.v1"
|
|
53
56
|
ROLLING_DELTA_SCHEMA_VERSION = "contextguard.pack-rolling-delta.v1"
|
|
54
57
|
SKETCH_DUPLICATE_SHINGLE_WIDTH = 5
|
|
@@ -65,10 +68,24 @@ DEFAULT_SUGGEST_CONTEXT_LINES = 20
|
|
|
65
68
|
MAX_SUGGEST_CONTEXT_LINES = 120
|
|
66
69
|
SUGGEST_WHOLE_FILE_MAX_LINES = 120
|
|
67
70
|
MAX_SUGGEST_INPUT_BYTES = 256_000
|
|
71
|
+
MAX_GIT_DIFF_STDERR_BYTES = 16_000
|
|
72
|
+
GIT_DIFF_TIMEOUT_SECONDS = 10.0
|
|
68
73
|
MAX_QUERY_SCAN_FILES = 2_000
|
|
69
74
|
MAX_QUERY_SCAN_BYTES_PER_FILE = 200_000
|
|
70
75
|
MAX_GIT_LS_FILES_OUTPUT_BYTES = MAX_QUERY_SCAN_FILES * 512
|
|
71
76
|
GIT_LS_FILES_READ_CHUNK_BYTES = 64 * 1024
|
|
77
|
+
MAX_GIT_ATTR_INPUT_BYTES = MAX_GIT_LS_FILES_OUTPUT_BYTES
|
|
78
|
+
MAX_GIT_ATTR_OUTPUT_BYTES = MAX_GIT_ATTR_INPUT_BYTES * 2
|
|
79
|
+
GIT_ATTR_TIMEOUT_SECONDS = 10.0
|
|
80
|
+
MAX_QUERY_WALK_DIRS = 2_000
|
|
81
|
+
MAX_QUERY_WALK_ENTRIES = 10_000
|
|
82
|
+
MAX_QUERY_WALK_DEPTH = 32
|
|
83
|
+
MAX_QUERY_WALK_SECONDS = 2.0
|
|
84
|
+
MAX_SOURCE_INPUT_BYTES = 4_000_000
|
|
85
|
+
MAX_SOURCE_INPUT_LINES = 100_000
|
|
86
|
+
MAX_SOURCE_LINE_BYTES = 256_000
|
|
87
|
+
MAX_TOTAL_SOURCE_INPUT_BYTES = 16_000_000
|
|
88
|
+
MAX_TOTAL_SOURCE_INPUT_LINES = 400_000
|
|
72
89
|
MAX_REPO_MAP_FILES = 1_000
|
|
73
90
|
MAX_REPO_MAP_SCAN_FILES = 160
|
|
74
91
|
MAX_REPO_MAP_BYTES_PER_FILE = 120_000
|
|
@@ -85,6 +102,8 @@ MAX_ADAPTIVE_K_VERIFICATION_HINTS = 12
|
|
|
85
102
|
ADAPTIVE_K_POLICIES = ("balanced", "recall", "precision")
|
|
86
103
|
MAX_SYMBOL_MEMORY_ITEMS = 12
|
|
87
104
|
MAX_SYMBOL_MEMORY_GRAPH_ITEMS = 12
|
|
105
|
+
MAX_GRAPH_APPLICATION_SOURCES = 4
|
|
106
|
+
MAX_GRAPH_APPLICATION_LINES = 80
|
|
88
107
|
PACK_DIR = ".context-guard/packs"
|
|
89
108
|
REDACTED_PATH_COMPONENT = "[REDACTED-PATH-COMPONENT]"
|
|
90
109
|
ALLOWED_FIRST_ABSOLUTE_SYMLINKS = {
|
|
@@ -165,6 +184,7 @@ class SourceSpec:
|
|
|
165
184
|
label: str | None = None
|
|
166
185
|
input_index: int = 0
|
|
167
186
|
origin: str = "cli"
|
|
187
|
+
sanitization_context: str = "source_code"
|
|
168
188
|
|
|
169
189
|
|
|
170
190
|
@dataclass
|
|
@@ -177,6 +197,97 @@ class ResolvedSource:
|
|
|
177
197
|
selected_lines: list[str]
|
|
178
198
|
total_lines: int
|
|
179
199
|
redacted_lines: int
|
|
200
|
+
total_lines_exact: bool = True
|
|
201
|
+
input_bytes_read: int = 0
|
|
202
|
+
input_lines_read: int = 0
|
|
203
|
+
sanitized_through_line: int = 0
|
|
204
|
+
input_limit_reason: str | None = None
|
|
205
|
+
redacted_lines_exact: bool = True
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
@dataclass(frozen=True)
|
|
209
|
+
class _SourceScanResult:
|
|
210
|
+
selected_lines: tuple[str, ...]
|
|
211
|
+
total_lines: int
|
|
212
|
+
redacted_lines: int
|
|
213
|
+
total_lines_exact: bool
|
|
214
|
+
input_bytes_read: int
|
|
215
|
+
input_lines_read: int
|
|
216
|
+
sanitized_through_line: int
|
|
217
|
+
limit_reason: str | None
|
|
218
|
+
selection_complete: bool
|
|
219
|
+
redacted_lines_exact: bool
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
@dataclass(frozen=True)
|
|
223
|
+
class _SourceSnapshot:
|
|
224
|
+
identity: tuple[int, int, int, int, int, int, int, int]
|
|
225
|
+
display_path: str
|
|
226
|
+
redacted_path: bool
|
|
227
|
+
requested_lines: LineRange
|
|
228
|
+
selected_lines: tuple[str, ...]
|
|
229
|
+
total_lines: int
|
|
230
|
+
redacted_lines: int
|
|
231
|
+
total_lines_exact: bool
|
|
232
|
+
input_bytes_read: int
|
|
233
|
+
input_lines_read: int
|
|
234
|
+
sanitized_through_line: int
|
|
235
|
+
input_limit_reason: str | None
|
|
236
|
+
redacted_lines_exact: bool
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
class _SourceInputBudget:
|
|
240
|
+
def __init__(self) -> None:
|
|
241
|
+
self.bytes_read = 0
|
|
242
|
+
self.lines_read = 0
|
|
243
|
+
self.bytes_attempted = 0
|
|
244
|
+
self.lines_attempted = 0
|
|
245
|
+
self.bytes_charged = 0
|
|
246
|
+
self.lines_charged = 0
|
|
247
|
+
self.capped = (
|
|
248
|
+
MAX_TOTAL_SOURCE_INPUT_BYTES <= 0
|
|
249
|
+
or MAX_TOTAL_SOURCE_INPUT_LINES <= 0
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
def remaining_bytes(self) -> int:
|
|
253
|
+
return max(0, MAX_TOTAL_SOURCE_INPUT_BYTES - self.bytes_charged)
|
|
254
|
+
|
|
255
|
+
def remaining_lines(self) -> int:
|
|
256
|
+
return max(0, MAX_TOTAL_SOURCE_INPUT_LINES - self.lines_charged)
|
|
257
|
+
|
|
258
|
+
def record_read(self, bytes_count: int) -> str | None:
|
|
259
|
+
bytes_remaining = self.remaining_bytes()
|
|
260
|
+
lines_remaining = self.remaining_lines()
|
|
261
|
+
self.bytes_read += bytes_count
|
|
262
|
+
self.lines_read += 1
|
|
263
|
+
self.bytes_attempted += bytes_count
|
|
264
|
+
self.lines_attempted += 1
|
|
265
|
+
self.bytes_charged = min(
|
|
266
|
+
MAX_TOTAL_SOURCE_INPUT_BYTES,
|
|
267
|
+
self.bytes_charged + bytes_count,
|
|
268
|
+
)
|
|
269
|
+
self.lines_charged = min(
|
|
270
|
+
MAX_TOTAL_SOURCE_INPUT_LINES,
|
|
271
|
+
self.lines_charged + 1,
|
|
272
|
+
)
|
|
273
|
+
self.capped = (
|
|
274
|
+
self.bytes_charged >= MAX_TOTAL_SOURCE_INPUT_BYTES
|
|
275
|
+
or self.lines_charged >= MAX_TOTAL_SOURCE_INPUT_LINES
|
|
276
|
+
)
|
|
277
|
+
if lines_remaining <= 0:
|
|
278
|
+
return "cumulative_input_lines_exceeded"
|
|
279
|
+
if bytes_count > bytes_remaining:
|
|
280
|
+
return "cumulative_input_bytes_exceeded"
|
|
281
|
+
return None
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
class _SourceSnapshotCache:
|
|
285
|
+
def __init__(self) -> None:
|
|
286
|
+
self.entries: dict[tuple[str, str, str], _SourceSnapshot] = {}
|
|
287
|
+
|
|
288
|
+
@staticmethod
|
|
289
|
+
def key(rel: Path, requested: LineRange | None, context: str) -> tuple[str, str, str]:
|
|
290
|
+
return (rel.as_posix(), requested.identity() if requested is not None else "all", context)
|
|
180
291
|
|
|
181
292
|
|
|
182
293
|
@dataclass
|
|
@@ -205,9 +316,32 @@ class PackError(ValueError):
|
|
|
205
316
|
pass
|
|
206
317
|
|
|
207
318
|
|
|
319
|
+
SANITIZATION_CONTEXTS = frozenset(
|
|
320
|
+
{
|
|
321
|
+
"unknown_text",
|
|
322
|
+
"command_search_diff",
|
|
323
|
+
"filesystem_listing",
|
|
324
|
+
"source_code",
|
|
325
|
+
}
|
|
326
|
+
)
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def parse_sanitization_context(value: object) -> str:
|
|
330
|
+
context = str(value or "unknown_text")
|
|
331
|
+
if context not in SANITIZATION_CONTEXTS:
|
|
332
|
+
raise PackError(f"unsupported sanitization context: {context}")
|
|
333
|
+
return context
|
|
334
|
+
|
|
335
|
+
|
|
208
336
|
class FallbackLineSanitizer:
|
|
209
|
-
def __init__(
|
|
337
|
+
def __init__(
|
|
338
|
+
self,
|
|
339
|
+
*,
|
|
340
|
+
show_paths: bool = False,
|
|
341
|
+
context: str = "unknown_text",
|
|
342
|
+
) -> None:
|
|
210
343
|
self.show_paths = show_paths
|
|
344
|
+
self.context = context
|
|
211
345
|
self.redactions = 0
|
|
212
346
|
|
|
213
347
|
def sanitize(self, raw_line: str) -> tuple[str, bool]:
|
|
@@ -252,7 +386,12 @@ def load_line_sanitizer_factory() -> Any:
|
|
|
252
386
|
if spec is None:
|
|
253
387
|
raise RuntimeError("import spec unavailable")
|
|
254
388
|
module = importlib.util.module_from_spec(spec)
|
|
255
|
-
loader.
|
|
389
|
+
sys.modules[loader.name] = module
|
|
390
|
+
try:
|
|
391
|
+
loader.exec_module(module)
|
|
392
|
+
except Exception:
|
|
393
|
+
sys.modules.pop(loader.name, None)
|
|
394
|
+
raise
|
|
256
395
|
_LINE_SANITIZER_FACTORY_CACHE = module.LineSanitizer
|
|
257
396
|
return _LINE_SANITIZER_FACTORY_CACHE
|
|
258
397
|
except Exception as exc:
|
|
@@ -261,13 +400,53 @@ def load_line_sanitizer_factory() -> Any:
|
|
|
261
400
|
return _LINE_SANITIZER_FACTORY_CACHE
|
|
262
401
|
|
|
263
402
|
|
|
264
|
-
def
|
|
403
|
+
def instantiate_line_sanitizer(
|
|
404
|
+
factory: Any,
|
|
405
|
+
*,
|
|
406
|
+
show_paths: bool,
|
|
407
|
+
context: str,
|
|
408
|
+
private_roots: tuple[str, ...] = (),
|
|
409
|
+
) -> object:
|
|
410
|
+
try:
|
|
411
|
+
return factory(
|
|
412
|
+
show_paths=show_paths,
|
|
413
|
+
context=context,
|
|
414
|
+
private_roots=private_roots,
|
|
415
|
+
)
|
|
416
|
+
except TypeError:
|
|
417
|
+
if context != "unknown_text" or private_roots:
|
|
418
|
+
raise RuntimeError(
|
|
419
|
+
"adjacent sanitizer does not support required explicit context"
|
|
420
|
+
)
|
|
421
|
+
return factory(show_paths=show_paths)
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def load_line_sanitizer(
|
|
425
|
+
show_paths: bool = False,
|
|
426
|
+
context: str = "unknown_text",
|
|
427
|
+
private_roots: tuple[str, ...] = (),
|
|
428
|
+
) -> object:
|
|
265
429
|
sanitizer_factory = load_line_sanitizer_factory()
|
|
266
|
-
return
|
|
430
|
+
return instantiate_line_sanitizer(
|
|
431
|
+
sanitizer_factory,
|
|
432
|
+
show_paths=show_paths,
|
|
433
|
+
context=context,
|
|
434
|
+
private_roots=private_roots,
|
|
435
|
+
)
|
|
267
436
|
|
|
268
437
|
|
|
269
|
-
def sanitize_text(
|
|
270
|
-
|
|
438
|
+
def sanitize_text(
|
|
439
|
+
text: str,
|
|
440
|
+
*,
|
|
441
|
+
show_paths: bool = False,
|
|
442
|
+
context: str = "unknown_text",
|
|
443
|
+
private_roots: tuple[str, ...] = (),
|
|
444
|
+
) -> tuple[str, int]:
|
|
445
|
+
sanitizer = load_line_sanitizer(
|
|
446
|
+
show_paths,
|
|
447
|
+
context=context,
|
|
448
|
+
private_roots=private_roots,
|
|
449
|
+
)
|
|
271
450
|
redacted = 0
|
|
272
451
|
out: list[str] = []
|
|
273
452
|
for line in text.splitlines(True):
|
|
@@ -278,28 +457,137 @@ def sanitize_text(text: str, *, show_paths: bool = False) -> tuple[str, int]:
|
|
|
278
457
|
return "".join(out), redacted
|
|
279
458
|
|
|
280
459
|
|
|
281
|
-
def sanitize_source_lines(
|
|
282
|
-
|
|
460
|
+
def sanitize_source_lines(
|
|
461
|
+
handle: Any,
|
|
462
|
+
requested: LineRange | None,
|
|
463
|
+
*,
|
|
464
|
+
context: str = "source_code",
|
|
465
|
+
private_roots: tuple[str, ...] = (),
|
|
466
|
+
) -> tuple[list[str], int, int]:
|
|
467
|
+
"""Compatibility wrapper for the bounded source scanner."""
|
|
468
|
+
scan = _scan_source_lines(
|
|
469
|
+
handle,
|
|
470
|
+
requested,
|
|
471
|
+
context=context,
|
|
472
|
+
private_roots=private_roots,
|
|
473
|
+
input_budget=_SourceInputBudget(),
|
|
474
|
+
)
|
|
475
|
+
return list(scan.selected_lines), scan.total_lines, scan.redacted_lines
|
|
283
476
|
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
477
|
+
|
|
478
|
+
def _scan_source_lines(
|
|
479
|
+
handle: Any,
|
|
480
|
+
requested: LineRange | None,
|
|
481
|
+
*,
|
|
482
|
+
context: str,
|
|
483
|
+
private_roots: tuple[str, ...],
|
|
484
|
+
input_budget: _SourceInputBudget,
|
|
485
|
+
expected_size_bytes: int | None = None,
|
|
486
|
+
) -> _SourceScanResult:
|
|
487
|
+
"""Read with byte/line caps and sanitize only the required prefix.
|
|
488
|
+
|
|
489
|
+
Stateful sanitizers still see every line through ``requested.end``. The
|
|
490
|
+
remaining tail is counted without invoking the sanitizer so range requests
|
|
491
|
+
do not pay sanitizer cost for irrelevant content. If counting reaches a
|
|
492
|
+
cap, the selected range remains usable and the total is explicitly marked
|
|
493
|
+
as a lower bound.
|
|
288
494
|
"""
|
|
289
|
-
sanitizer = load_line_sanitizer(
|
|
495
|
+
sanitizer = load_line_sanitizer(
|
|
496
|
+
context=context,
|
|
497
|
+
private_roots=private_roots,
|
|
498
|
+
)
|
|
290
499
|
selected: list[str] = []
|
|
291
500
|
redacted = 0
|
|
292
501
|
total_lines = 0
|
|
502
|
+
input_bytes = 0
|
|
503
|
+
input_lines = 0
|
|
293
504
|
collect_all = requested is None
|
|
294
505
|
start = requested.start if requested is not None else 1
|
|
295
506
|
end = requested.end if requested is not None else 0
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
507
|
+
total_lines_exact = False
|
|
508
|
+
limit_reason: str | None = None
|
|
509
|
+
redacted_lines_exact = True
|
|
510
|
+
iterator = None if callable(getattr(handle, "readline", None)) else iter(handle)
|
|
511
|
+
|
|
512
|
+
while True:
|
|
513
|
+
boundary_reason: str | None = None
|
|
514
|
+
if total_lines >= MAX_SOURCE_INPUT_LINES:
|
|
515
|
+
boundary_reason = "source_input_lines_exceeded"
|
|
516
|
+
elif input_budget.remaining_lines() <= 0:
|
|
517
|
+
boundary_reason = "cumulative_input_lines_exceeded"
|
|
518
|
+
source_remaining = MAX_SOURCE_INPUT_BYTES - input_bytes
|
|
519
|
+
cumulative_remaining = input_budget.remaining_bytes()
|
|
520
|
+
if boundary_reason is None and source_remaining <= 0:
|
|
521
|
+
boundary_reason = "source_input_bytes_exceeded"
|
|
522
|
+
elif boundary_reason is None and cumulative_remaining <= 0:
|
|
523
|
+
boundary_reason = "cumulative_input_bytes_exceeded"
|
|
524
|
+
if boundary_reason is not None:
|
|
525
|
+
if expected_size_bytes is not None:
|
|
526
|
+
try:
|
|
527
|
+
if handle.tell() == expected_size_bytes:
|
|
528
|
+
total_lines_exact = True
|
|
529
|
+
break
|
|
530
|
+
except (AttributeError, OSError):
|
|
531
|
+
pass
|
|
532
|
+
limit_reason = boundary_reason
|
|
533
|
+
break
|
|
534
|
+
read_char_cap = min(MAX_SOURCE_LINE_BYTES, source_remaining, cumulative_remaining)
|
|
535
|
+
try:
|
|
536
|
+
if iterator is None:
|
|
537
|
+
raw_line = handle.readline(read_char_cap + 1)
|
|
538
|
+
else:
|
|
539
|
+
raw_line = next(iterator, "")
|
|
540
|
+
except (OSError, UnicodeError):
|
|
541
|
+
limit_reason = "unsafe_path"
|
|
542
|
+
break
|
|
543
|
+
if raw_line == "":
|
|
544
|
+
total_lines_exact = True
|
|
545
|
+
break
|
|
546
|
+
raw_bytes = byte_len(raw_line)
|
|
547
|
+
cumulative_reason = input_budget.record_read(raw_bytes)
|
|
548
|
+
input_bytes += raw_bytes
|
|
549
|
+
input_lines += 1
|
|
550
|
+
if len(raw_line) > MAX_SOURCE_LINE_BYTES or raw_bytes > MAX_SOURCE_LINE_BYTES:
|
|
551
|
+
limit_reason = "source_line_bytes_exceeded"
|
|
552
|
+
break
|
|
553
|
+
if raw_bytes > source_remaining:
|
|
554
|
+
limit_reason = "source_input_bytes_exceeded"
|
|
555
|
+
break
|
|
556
|
+
if cumulative_reason is not None:
|
|
557
|
+
limit_reason = cumulative_reason
|
|
558
|
+
break
|
|
559
|
+
total_lines += 1
|
|
560
|
+
|
|
561
|
+
must_sanitize = collect_all or total_lines <= end
|
|
562
|
+
if must_sanitize:
|
|
563
|
+
sanitized, did_redact = sanitizer.sanitize(raw_line) # type: ignore[attr-defined]
|
|
564
|
+
if did_redact:
|
|
565
|
+
redacted += 1
|
|
566
|
+
if collect_all or start <= total_lines <= end:
|
|
567
|
+
selected.append(sanitized)
|
|
568
|
+
else:
|
|
569
|
+
redacted_lines_exact = False
|
|
570
|
+
if SECRET_CONTENT_RE.search(raw_line):
|
|
571
|
+
redacted += 1
|
|
572
|
+
|
|
573
|
+
selection_complete = (
|
|
574
|
+
total_lines_exact
|
|
575
|
+
if collect_all
|
|
576
|
+
else total_lines >= end or (total_lines_exact and total_lines >= start)
|
|
577
|
+
)
|
|
578
|
+
sanitized_through = total_lines if collect_all else min(total_lines, end)
|
|
579
|
+
return _SourceScanResult(
|
|
580
|
+
selected_lines=tuple(selected),
|
|
581
|
+
total_lines=total_lines,
|
|
582
|
+
redacted_lines=redacted,
|
|
583
|
+
total_lines_exact=total_lines_exact,
|
|
584
|
+
input_bytes_read=input_bytes,
|
|
585
|
+
input_lines_read=input_lines,
|
|
586
|
+
sanitized_through_line=sanitized_through,
|
|
587
|
+
limit_reason=limit_reason,
|
|
588
|
+
selection_complete=selection_complete,
|
|
589
|
+
redacted_lines_exact=redacted_lines_exact,
|
|
590
|
+
)
|
|
303
591
|
|
|
304
592
|
|
|
305
593
|
def byte_len(text: str) -> int:
|
|
@@ -886,6 +1174,9 @@ def read_manifest(path: Path) -> list[SourceSpec]:
|
|
|
886
1174
|
lines=lines,
|
|
887
1175
|
label=cap_label(item.get("label")),
|
|
888
1176
|
origin="manifest",
|
|
1177
|
+
sanitization_context=parse_sanitization_context(
|
|
1178
|
+
item.get("sanitization_context", item.get("context"))
|
|
1179
|
+
),
|
|
889
1180
|
))
|
|
890
1181
|
return out
|
|
891
1182
|
|
|
@@ -917,6 +1208,9 @@ def parse_source_spec(raw: str) -> SourceSpec:
|
|
|
917
1208
|
lines=lines,
|
|
918
1209
|
label=cap_label(values.get("label")),
|
|
919
1210
|
origin="cli",
|
|
1211
|
+
sanitization_context=parse_sanitization_context(
|
|
1212
|
+
values.get("sanitization_context", values.get("context"))
|
|
1213
|
+
),
|
|
920
1214
|
)
|
|
921
1215
|
|
|
922
1216
|
|
|
@@ -1081,7 +1375,69 @@ def open_regular_under_root(root: Path, rel: Path) -> tuple[Any | None, str]:
|
|
|
1081
1375
|
return None, "unsafe_path"
|
|
1082
1376
|
|
|
1083
1377
|
|
|
1084
|
-
def
|
|
1378
|
+
def _open_source_identity(handle: Any) -> tuple[int, int, int, int, int, int, int, int] | None:
|
|
1379
|
+
try:
|
|
1380
|
+
st = os.fstat(handle.fileno())
|
|
1381
|
+
except (AttributeError, OSError):
|
|
1382
|
+
return None
|
|
1383
|
+
return (
|
|
1384
|
+
st.st_dev,
|
|
1385
|
+
st.st_ino,
|
|
1386
|
+
st.st_mode,
|
|
1387
|
+
st.st_uid,
|
|
1388
|
+
st.st_nlink,
|
|
1389
|
+
st.st_size,
|
|
1390
|
+
int(getattr(st, "st_mtime_ns", int(st.st_mtime * 1_000_000_000))),
|
|
1391
|
+
int(getattr(st, "st_ctime_ns", int(st.st_ctime * 1_000_000_000))),
|
|
1392
|
+
)
|
|
1393
|
+
|
|
1394
|
+
|
|
1395
|
+
def _input_limit_metadata(reason: str) -> dict[str, Any]:
|
|
1396
|
+
caps = {
|
|
1397
|
+
"source_line_bytes_exceeded": ("source_line_bytes", MAX_SOURCE_LINE_BYTES),
|
|
1398
|
+
"source_input_bytes_exceeded": ("source_bytes", MAX_SOURCE_INPUT_BYTES),
|
|
1399
|
+
"source_input_lines_exceeded": ("source_lines", MAX_SOURCE_INPUT_LINES),
|
|
1400
|
+
"cumulative_input_bytes_exceeded": ("cumulative_bytes", MAX_TOTAL_SOURCE_INPUT_BYTES),
|
|
1401
|
+
"cumulative_input_lines_exceeded": ("cumulative_lines", MAX_TOTAL_SOURCE_INPUT_LINES),
|
|
1402
|
+
}
|
|
1403
|
+
kind, cap = caps.get(reason, ("unknown", 0))
|
|
1404
|
+
return {"kind": kind, "cap_bytes" if "bytes" in kind else "cap_lines": cap}
|
|
1405
|
+
|
|
1406
|
+
|
|
1407
|
+
def _snapshot_to_source(
|
|
1408
|
+
snapshot: _SourceSnapshot,
|
|
1409
|
+
*,
|
|
1410
|
+
root: Path,
|
|
1411
|
+
rel: Path,
|
|
1412
|
+
spec: SourceSpec,
|
|
1413
|
+
) -> ResolvedSource:
|
|
1414
|
+
return ResolvedSource(
|
|
1415
|
+
spec=spec,
|
|
1416
|
+
abs_path=root / rel,
|
|
1417
|
+
display_path=snapshot.display_path,
|
|
1418
|
+
redacted_path=snapshot.redacted_path,
|
|
1419
|
+
requested_lines=spec.lines or snapshot.requested_lines,
|
|
1420
|
+
selected_lines=list(snapshot.selected_lines),
|
|
1421
|
+
total_lines=snapshot.total_lines,
|
|
1422
|
+
redacted_lines=snapshot.redacted_lines,
|
|
1423
|
+
total_lines_exact=snapshot.total_lines_exact,
|
|
1424
|
+
input_bytes_read=snapshot.input_bytes_read,
|
|
1425
|
+
input_lines_read=snapshot.input_lines_read,
|
|
1426
|
+
sanitized_through_line=snapshot.sanitized_through_line,
|
|
1427
|
+
input_limit_reason=snapshot.input_limit_reason,
|
|
1428
|
+
redacted_lines_exact=snapshot.redacted_lines_exact,
|
|
1429
|
+
)
|
|
1430
|
+
|
|
1431
|
+
|
|
1432
|
+
def resolve_source(
|
|
1433
|
+
root: Path,
|
|
1434
|
+
spec: SourceSpec,
|
|
1435
|
+
*,
|
|
1436
|
+
source_cache: _SourceSnapshotCache | None = None,
|
|
1437
|
+
input_budget: _SourceInputBudget | None = None,
|
|
1438
|
+
expected_identity: tuple[int, int, int, int, int, int, int, int] | None = None,
|
|
1439
|
+
require_cached: bool = False,
|
|
1440
|
+
) -> tuple[ResolvedSource | None, dict[str, Any] | None]:
|
|
1085
1441
|
if spec.lines is not None and spec.lines.start < 1:
|
|
1086
1442
|
return None, omission(spec, "invalid_lines")
|
|
1087
1443
|
rel, reason = lexical_rel(spec.path)
|
|
@@ -1091,20 +1447,85 @@ def resolve_source(root: Path, spec: SourceSpec) -> tuple[ResolvedSource | None,
|
|
|
1091
1447
|
handle, reason = open_regular_under_root(root, rel)
|
|
1092
1448
|
if handle is None:
|
|
1093
1449
|
return None, omission(spec, reason, path=display, redacted_path=redacted_path)
|
|
1450
|
+
scan_budget = input_budget if input_budget is not None else _SourceInputBudget()
|
|
1451
|
+
requested = spec.lines
|
|
1452
|
+
cache_key = _SourceSnapshotCache.key(rel, requested, spec.sanitization_context)
|
|
1094
1453
|
try:
|
|
1095
1454
|
with handle:
|
|
1096
|
-
|
|
1097
|
-
|
|
1455
|
+
before_identity = _open_source_identity(handle)
|
|
1456
|
+
if expected_identity is not None and before_identity != expected_identity:
|
|
1457
|
+
return None, omission(
|
|
1458
|
+
spec,
|
|
1459
|
+
"graph_source_changed_since_repo_map_snapshot",
|
|
1460
|
+
path=display,
|
|
1461
|
+
redacted_path=redacted_path,
|
|
1462
|
+
)
|
|
1463
|
+
cached = source_cache.entries.get(cache_key) if source_cache is not None else None
|
|
1464
|
+
if require_cached and cached is None:
|
|
1465
|
+
return None, omission(
|
|
1466
|
+
spec,
|
|
1467
|
+
"graph_source_snapshot_unavailable",
|
|
1468
|
+
path=display,
|
|
1469
|
+
redacted_path=redacted_path,
|
|
1470
|
+
)
|
|
1471
|
+
if cached is not None:
|
|
1472
|
+
if before_identity != cached.identity:
|
|
1473
|
+
return None, omission(
|
|
1474
|
+
spec,
|
|
1475
|
+
"source_changed_during_auto",
|
|
1476
|
+
path=display,
|
|
1477
|
+
redacted_path=redacted_path,
|
|
1478
|
+
)
|
|
1479
|
+
source = _snapshot_to_source(cached, root=root, rel=rel, spec=spec)
|
|
1480
|
+
if _open_source_identity(handle) != before_identity:
|
|
1481
|
+
return None, omission(
|
|
1482
|
+
spec,
|
|
1483
|
+
"source_changed_during_auto",
|
|
1484
|
+
path=display,
|
|
1485
|
+
redacted_path=redacted_path,
|
|
1486
|
+
)
|
|
1487
|
+
return source, None
|
|
1488
|
+
scan = _scan_source_lines(
|
|
1489
|
+
handle,
|
|
1490
|
+
requested,
|
|
1491
|
+
context=spec.sanitization_context,
|
|
1492
|
+
private_roots=(
|
|
1493
|
+
(str(root),)
|
|
1494
|
+
if spec.sanitization_context == "filesystem_listing"
|
|
1495
|
+
else ()
|
|
1496
|
+
),
|
|
1497
|
+
input_budget=scan_budget,
|
|
1498
|
+
expected_size_bytes=before_identity[5] if before_identity is not None else None,
|
|
1499
|
+
)
|
|
1500
|
+
after_identity = _open_source_identity(handle)
|
|
1098
1501
|
except OSError:
|
|
1099
1502
|
return None, omission(spec, "unsafe_path", path=display, redacted_path=redacted_path)
|
|
1503
|
+
if before_identity is not None and after_identity != before_identity:
|
|
1504
|
+
return None, omission(spec, "source_changed_during_read", path=display, redacted_path=redacted_path)
|
|
1505
|
+
if scan.limit_reason == "unsafe_path":
|
|
1506
|
+
return None, omission(spec, "unsafe_path", path=display, redacted_path=redacted_path)
|
|
1507
|
+
if scan.limit_reason is not None and not scan.selection_complete:
|
|
1508
|
+
item = omission(spec, scan.limit_reason, path=display, redacted_path=redacted_path)
|
|
1509
|
+
item["input_limit"] = _input_limit_metadata(scan.limit_reason)
|
|
1510
|
+
item["input_observed"] = {
|
|
1511
|
+
"bytes": scan.input_bytes_read,
|
|
1512
|
+
"lines": scan.input_lines_read,
|
|
1513
|
+
"bytes_attempted": scan.input_bytes_read,
|
|
1514
|
+
"lines_attempted": scan.input_lines_read,
|
|
1515
|
+
"capped": True,
|
|
1516
|
+
}
|
|
1517
|
+
return None, item
|
|
1518
|
+
selected = list(scan.selected_lines)
|
|
1519
|
+
total_lines = scan.total_lines
|
|
1520
|
+
redacted_lines = scan.redacted_lines
|
|
1100
1521
|
if total_lines <= 0:
|
|
1101
1522
|
return None, omission(spec, "empty_source", path=display, redacted_path=redacted_path)
|
|
1102
1523
|
requested = requested or LineRange(1, total_lines)
|
|
1103
|
-
if requested.start > total_lines:
|
|
1524
|
+
if scan.total_lines_exact and requested.start > total_lines:
|
|
1104
1525
|
return None, omission(spec, "empty_source", path=display, redacted_path=redacted_path)
|
|
1105
1526
|
if not selected:
|
|
1106
1527
|
return None, omission(spec, "empty_source", path=display, redacted_path=redacted_path)
|
|
1107
|
-
|
|
1528
|
+
source = ResolvedSource(
|
|
1108
1529
|
spec=spec,
|
|
1109
1530
|
abs_path=root / rel,
|
|
1110
1531
|
display_path=display,
|
|
@@ -1113,7 +1534,33 @@ def resolve_source(root: Path, spec: SourceSpec) -> tuple[ResolvedSource | None,
|
|
|
1113
1534
|
selected_lines=selected,
|
|
1114
1535
|
total_lines=total_lines,
|
|
1115
1536
|
redacted_lines=redacted_lines,
|
|
1116
|
-
|
|
1537
|
+
total_lines_exact=scan.total_lines_exact,
|
|
1538
|
+
input_bytes_read=scan.input_bytes_read,
|
|
1539
|
+
input_lines_read=scan.input_lines_read,
|
|
1540
|
+
sanitized_through_line=scan.sanitized_through_line,
|
|
1541
|
+
input_limit_reason=scan.limit_reason,
|
|
1542
|
+
redacted_lines_exact=scan.redacted_lines_exact,
|
|
1543
|
+
)
|
|
1544
|
+
if source_cache is not None and before_identity is not None:
|
|
1545
|
+
snapshot = _SourceSnapshot(
|
|
1546
|
+
identity=before_identity,
|
|
1547
|
+
display_path=display,
|
|
1548
|
+
redacted_path=redacted_path,
|
|
1549
|
+
requested_lines=requested,
|
|
1550
|
+
selected_lines=tuple(selected),
|
|
1551
|
+
total_lines=total_lines,
|
|
1552
|
+
redacted_lines=redacted_lines,
|
|
1553
|
+
total_lines_exact=scan.total_lines_exact,
|
|
1554
|
+
input_bytes_read=scan.input_bytes_read,
|
|
1555
|
+
input_lines_read=scan.input_lines_read,
|
|
1556
|
+
sanitized_through_line=scan.sanitized_through_line,
|
|
1557
|
+
input_limit_reason=scan.limit_reason,
|
|
1558
|
+
redacted_lines_exact=scan.redacted_lines_exact,
|
|
1559
|
+
)
|
|
1560
|
+
source_cache.entries[cache_key] = snapshot
|
|
1561
|
+
canonical_key = _SourceSnapshotCache.key(rel, source_selected_range(source), spec.sanitization_context)
|
|
1562
|
+
source_cache.entries[canonical_key] = snapshot
|
|
1563
|
+
return source, None
|
|
1117
1564
|
|
|
1118
1565
|
|
|
1119
1566
|
def retrieval_cli(root_arg: str, display_path: str, lines: LineRange) -> str:
|
|
@@ -1152,31 +1599,99 @@ def retrieval_for(root_arg: str, display_path: str, lines: LineRange, *, redacte
|
|
|
1152
1599
|
return retrieval_cli(safe_root, display_path, lines), None
|
|
1153
1600
|
|
|
1154
1601
|
|
|
1155
|
-
|
|
1156
|
-
|
|
1602
|
+
def markdown_metadata_text(value: object) -> str:
|
|
1603
|
+
out: list[str] = []
|
|
1604
|
+
for char in str(value):
|
|
1605
|
+
code = ord(char)
|
|
1606
|
+
if not char.isprintable():
|
|
1607
|
+
out.append(f"\\u{code:04X}" if code <= 0xFFFF else f"\\U{code:08X}")
|
|
1608
|
+
elif char == "&":
|
|
1609
|
+
out.append("&")
|
|
1610
|
+
elif char == "<":
|
|
1611
|
+
out.append("<")
|
|
1612
|
+
elif char == ">":
|
|
1613
|
+
out.append(">")
|
|
1614
|
+
elif char in {"[", "]", "(", ")", "!"}:
|
|
1615
|
+
out.append("\\" + char)
|
|
1616
|
+
elif char in {"`", "\\"}:
|
|
1617
|
+
out.append("\\" + char)
|
|
1618
|
+
else:
|
|
1619
|
+
out.append(char)
|
|
1620
|
+
return "".join(out)
|
|
1621
|
+
|
|
1622
|
+
|
|
1623
|
+
def markdown_inline_code(value: object) -> str:
|
|
1624
|
+
text = "".join(
|
|
1625
|
+
(f"\\u{ord(char):04X}" if ord(char) <= 0xFFFF else f"\\U{ord(char):08X}")
|
|
1626
|
+
if not char.isprintable()
|
|
1627
|
+
else char
|
|
1628
|
+
for char in str(value)
|
|
1629
|
+
)
|
|
1630
|
+
max_run = max((len(match.group(0)) for match in re.finditer(r"`+", text)), default=0)
|
|
1631
|
+
delimiter = "`" * max(1, max_run + 1)
|
|
1632
|
+
padding = " " if text.startswith("`") or text.endswith("`") else ""
|
|
1633
|
+
return f"{delimiter}{padding}{text}{padding}{delimiter}"
|
|
1634
|
+
|
|
1635
|
+
|
|
1636
|
+
def markdown_block_delimiters(lines: list[str]) -> tuple[str, str]:
|
|
1637
|
+
max_run = 0
|
|
1638
|
+
for line in lines:
|
|
1639
|
+
line_max = max((len(match.group(0)) for match in re.finditer(r"`+", line)), default=0)
|
|
1640
|
+
max_run = max(max_run, line_max)
|
|
1641
|
+
fence = "`" * max(3, max_run + 1)
|
|
1642
|
+
return f"\n\n{fence}text\n", f"{fence}\n\n"
|
|
1157
1643
|
|
|
1158
1644
|
|
|
1159
1645
|
def render_block_header(source: ResolvedSource, *, root_arg: str, status: str, included: LineRange) -> str:
|
|
1160
|
-
title = source.spec.label or source.display_path
|
|
1646
|
+
title = markdown_metadata_text(source.spec.label or source.display_path)
|
|
1161
1647
|
requested = source.requested_lines or LineRange(1, source.total_lines)
|
|
1162
1648
|
retrieval, retrieval_omitted_reason = retrieval_for(root_arg, source.display_path, included, redacted_path=source.redacted_path)
|
|
1163
1649
|
header = [
|
|
1164
1650
|
f"## {title}",
|
|
1165
|
-
f"Source:
|
|
1651
|
+
f"Source: {markdown_inline_code(source.display_path)}",
|
|
1166
1652
|
f"Priority: {source.spec.priority}",
|
|
1167
1653
|
f"Status: {status}",
|
|
1168
1654
|
f"Included lines: {included.start}:{included.end}",
|
|
1169
1655
|
f"Requested lines: {requested.start}:{requested.end}",
|
|
1170
1656
|
]
|
|
1171
1657
|
if retrieval:
|
|
1172
|
-
header.append(f"Retrieval:
|
|
1658
|
+
header.append(f"Retrieval: {markdown_inline_code(retrieval)}")
|
|
1173
1659
|
elif retrieval_omitted_reason:
|
|
1174
1660
|
header.append(f"Retrieval omitted: {retrieval_omitted_reason}")
|
|
1175
1661
|
return "\n".join(header)
|
|
1176
1662
|
|
|
1177
1663
|
|
|
1178
1664
|
def render_block(source: ResolvedSource, lines: list[str], *, root_arg: str, status: str, included: LineRange) -> str:
|
|
1179
|
-
|
|
1665
|
+
block_open, block_close = markdown_block_delimiters(lines)
|
|
1666
|
+
return render_block_header(source, root_arg=root_arg, status=status, included=included) + block_open + "".join(lines) + ("" if not lines or lines[-1].endswith("\n") else "\n") + block_close
|
|
1667
|
+
|
|
1668
|
+
|
|
1669
|
+
def source_input_metadata(source: ResolvedSource) -> dict[str, Any]:
|
|
1670
|
+
item: dict[str, Any] = {
|
|
1671
|
+
"bytes_read": source.input_bytes_read,
|
|
1672
|
+
"lines_read": source.input_lines_read,
|
|
1673
|
+
"bytes_attempted": source.input_bytes_read,
|
|
1674
|
+
"lines_attempted": source.input_lines_read,
|
|
1675
|
+
"capped": source.input_limit_reason is not None,
|
|
1676
|
+
"total_lines_exact": source.total_lines_exact,
|
|
1677
|
+
"truncated": not source.total_lines_exact,
|
|
1678
|
+
"sanitized_through_line": source.sanitized_through_line,
|
|
1679
|
+
"redacted_lines_exact": source.redacted_lines_exact,
|
|
1680
|
+
"limits": {
|
|
1681
|
+
"source_bytes": MAX_SOURCE_INPUT_BYTES,
|
|
1682
|
+
"source_lines": MAX_SOURCE_INPUT_LINES,
|
|
1683
|
+
"source_line_bytes": MAX_SOURCE_LINE_BYTES,
|
|
1684
|
+
"cumulative_bytes": MAX_TOTAL_SOURCE_INPUT_BYTES,
|
|
1685
|
+
"cumulative_lines": MAX_TOTAL_SOURCE_INPUT_LINES,
|
|
1686
|
+
},
|
|
1687
|
+
}
|
|
1688
|
+
if source.total_lines_exact:
|
|
1689
|
+
item["total_lines"] = source.total_lines
|
|
1690
|
+
else:
|
|
1691
|
+
item["total_lines_lower_bound"] = source.total_lines
|
|
1692
|
+
if source.input_limit_reason is not None:
|
|
1693
|
+
item["limit_reason"] = source.input_limit_reason
|
|
1694
|
+
return item
|
|
1180
1695
|
|
|
1181
1696
|
|
|
1182
1697
|
def source_metadata(source: ResolvedSource, *, status: str, lines: list[str], included: LineRange, root_arg: str) -> dict[str, Any]:
|
|
@@ -1192,6 +1707,7 @@ def source_metadata(source: ResolvedSource, *, status: str, lines: list[str], in
|
|
|
1192
1707
|
}
|
|
1193
1708
|
if source.spec.label:
|
|
1194
1709
|
item["label"] = source.spec.label
|
|
1710
|
+
item["input"] = source_input_metadata(source)
|
|
1195
1711
|
retrieval, retrieval_omitted_reason = retrieval_for(root_arg, source.display_path, included, redacted_path=source.redacted_path)
|
|
1196
1712
|
if retrieval:
|
|
1197
1713
|
item["retrieval_cli"] = retrieval
|
|
@@ -1206,7 +1722,11 @@ def budget_omission(source: ResolvedSource, *, root_arg: str) -> dict[str, Any]:
|
|
|
1206
1722
|
requested = source.requested_lines or LineRange(1, source.total_lines)
|
|
1207
1723
|
item = omission(source.spec, "budget_exhausted", path=source.display_path, redacted_path=source.redacted_path)
|
|
1208
1724
|
item["requested_lines"] = requested.as_dict()
|
|
1209
|
-
|
|
1725
|
+
if source.total_lines_exact:
|
|
1726
|
+
item["total_lines"] = source.total_lines
|
|
1727
|
+
else:
|
|
1728
|
+
item["total_lines_lower_bound"] = source.total_lines
|
|
1729
|
+
item["input"] = source_input_metadata(source)
|
|
1210
1730
|
retrieval, retrieval_omitted_reason = retrieval_for(root_arg, source.display_path, requested, redacted_path=source.redacted_path)
|
|
1211
1731
|
if retrieval:
|
|
1212
1732
|
item["retrieval_cli"] = retrieval
|
|
@@ -1242,7 +1762,8 @@ def render_block_byte_len(
|
|
|
1242
1762
|
body_bytes = line_prefixes[line_count]
|
|
1243
1763
|
if line_count > 0 and not source.selected_lines[line_count - 1].endswith("\n"):
|
|
1244
1764
|
body_bytes += 1
|
|
1245
|
-
|
|
1765
|
+
block_open, block_close = markdown_block_delimiters(source.selected_lines[:line_count])
|
|
1766
|
+
return byte_len(render_block_header(source, root_arg=root_arg, status=status, included=included)) + byte_len(block_open) + body_bytes + byte_len(block_close)
|
|
1246
1767
|
|
|
1247
1768
|
|
|
1248
1769
|
def fit_partial_lines(
|
|
@@ -1741,7 +2262,13 @@ def build_pack(
|
|
|
1741
2262
|
store_artifact: bool,
|
|
1742
2263
|
delta_from_pack_id: str | None = None,
|
|
1743
2264
|
sketch_duplicate_veto: bool = False,
|
|
2265
|
+
_source_cache: _SourceSnapshotCache | None = None,
|
|
2266
|
+
_input_budget: _SourceInputBudget | None = None,
|
|
2267
|
+
_required_snapshot_sources: set[tuple[str, str]] | None = None,
|
|
2268
|
+
_expected_source_identities: dict[str, tuple[int, int, int, int, int, int, int, int]] | None = None,
|
|
2269
|
+
_snapshot_rejections: dict[tuple[str, str], dict[str, Any]] | None = None,
|
|
1744
2270
|
) -> dict[str, Any]:
|
|
2271
|
+
input_budget = _input_budget if _input_budget is not None else _SourceInputBudget()
|
|
1745
2272
|
seen: set[tuple[str, str]] = set()
|
|
1746
2273
|
resolved: list[ResolvedSource] = []
|
|
1747
2274
|
paired_candidates: list[_PairedCandidate] = []
|
|
@@ -1768,7 +2295,55 @@ def build_pack(
|
|
|
1768
2295
|
continue
|
|
1769
2296
|
if rel is not None:
|
|
1770
2297
|
seen.add(identity)
|
|
1771
|
-
|
|
2298
|
+
rel_path = rel.as_posix() if rel is not None else ""
|
|
2299
|
+
require_cached = bool(
|
|
2300
|
+
_required_snapshot_sources
|
|
2301
|
+
and (rel_path, identity_lines) in _required_snapshot_sources
|
|
2302
|
+
)
|
|
2303
|
+
expected_identity = (
|
|
2304
|
+
_expected_source_identities.get(rel_path)
|
|
2305
|
+
if require_cached and _expected_source_identities is not None
|
|
2306
|
+
else None
|
|
2307
|
+
)
|
|
2308
|
+
if require_cached and expected_identity is None:
|
|
2309
|
+
display, redacted = display_rel_path(rel_path)
|
|
2310
|
+
omitted_item = omission(
|
|
2311
|
+
spec,
|
|
2312
|
+
"graph_source_not_in_repo_map_snapshot",
|
|
2313
|
+
path=display,
|
|
2314
|
+
redacted_path=redacted,
|
|
2315
|
+
)
|
|
2316
|
+
omitted.append(omitted_item)
|
|
2317
|
+
canonical_specs.append({
|
|
2318
|
+
"path": display,
|
|
2319
|
+
"priority": spec.priority,
|
|
2320
|
+
"lines": identity_lines,
|
|
2321
|
+
"status": omitted_item.get("reason"),
|
|
2322
|
+
})
|
|
2323
|
+
continue
|
|
2324
|
+
cached_rejection = (
|
|
2325
|
+
_snapshot_rejections.get((rel_path, identity_lines))
|
|
2326
|
+
if require_cached and _snapshot_rejections is not None
|
|
2327
|
+
else None
|
|
2328
|
+
)
|
|
2329
|
+
if cached_rejection is not None:
|
|
2330
|
+
omitted_item = copy.deepcopy(cached_rejection)
|
|
2331
|
+
omitted.append(omitted_item)
|
|
2332
|
+
canonical_specs.append({
|
|
2333
|
+
"path": omitted_item.get("path"),
|
|
2334
|
+
"priority": spec.priority,
|
|
2335
|
+
"lines": identity_lines,
|
|
2336
|
+
"status": omitted_item.get("reason"),
|
|
2337
|
+
})
|
|
2338
|
+
continue
|
|
2339
|
+
source, omitted_item = resolve_source(
|
|
2340
|
+
root,
|
|
2341
|
+
spec,
|
|
2342
|
+
source_cache=_source_cache,
|
|
2343
|
+
input_budget=input_budget,
|
|
2344
|
+
expected_identity=expected_identity,
|
|
2345
|
+
require_cached=require_cached,
|
|
2346
|
+
)
|
|
1772
2347
|
if omitted_item is not None:
|
|
1773
2348
|
omitted.append(omitted_item)
|
|
1774
2349
|
canonical_specs.append({"path": omitted_item.get("path"), "priority": spec.priority, "lines": identity_lines, "status": omitted_item.get("reason")})
|
|
@@ -1843,7 +2418,27 @@ def build_pack(
|
|
|
1843
2418
|
"sources": {"total": len(specs), "included": len(included) - partial_count, "partial": partial_count, "omitted": len(omitted_sorted)},
|
|
1844
2419
|
"included_sources": included,
|
|
1845
2420
|
"omitted_sources": omitted_sorted,
|
|
1846
|
-
"redaction": {
|
|
2421
|
+
"redaction": {
|
|
2422
|
+
"redacted_lines": redacted_lines,
|
|
2423
|
+
"redacted_lines_exact": all(source.redacted_lines_exact for source in all_resolved),
|
|
2424
|
+
"redacted_before_pack": True,
|
|
2425
|
+
},
|
|
2426
|
+
"input": {
|
|
2427
|
+
"bytes_read": input_budget.bytes_read,
|
|
2428
|
+
"lines_read": input_budget.lines_read,
|
|
2429
|
+
"bytes_attempted": input_budget.bytes_attempted,
|
|
2430
|
+
"lines_attempted": input_budget.lines_attempted,
|
|
2431
|
+
"bytes_charged": input_budget.bytes_charged,
|
|
2432
|
+
"lines_charged": input_budget.lines_charged,
|
|
2433
|
+
"capped": input_budget.capped,
|
|
2434
|
+
"limits": {
|
|
2435
|
+
"source_bytes": MAX_SOURCE_INPUT_BYTES,
|
|
2436
|
+
"source_lines": MAX_SOURCE_INPUT_LINES,
|
|
2437
|
+
"source_line_bytes": MAX_SOURCE_LINE_BYTES,
|
|
2438
|
+
"cumulative_bytes": MAX_TOTAL_SOURCE_INPUT_BYTES,
|
|
2439
|
+
"cumulative_lines": MAX_TOTAL_SOURCE_INPUT_LINES,
|
|
2440
|
+
},
|
|
2441
|
+
},
|
|
1847
2442
|
"artifact": {"stored": False, "path": None, "bytes": 0, "capped": False, "cap_bytes": MAX_RECEIPT_BYTES},
|
|
1848
2443
|
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
1849
2444
|
}
|
|
@@ -1889,7 +2484,12 @@ def slice_source(root: Path, *, raw_path: str, lines: LineRange) -> tuple[dict[s
|
|
|
1889
2484
|
"query": {"type": "lines", "start": lines.start, "end": min(lines.end, source.total_lines), "returned_lines": len(source.selected_lines)},
|
|
1890
2485
|
"content": content,
|
|
1891
2486
|
"bytes": byte_len(content),
|
|
1892
|
-
"redaction": {
|
|
2487
|
+
"redaction": {
|
|
2488
|
+
"redacted_lines": source.redacted_lines,
|
|
2489
|
+
"redacted_lines_exact": source.redacted_lines_exact,
|
|
2490
|
+
"redacted_before_pack": True,
|
|
2491
|
+
},
|
|
2492
|
+
"input": source_input_metadata(source),
|
|
1893
2493
|
}
|
|
1894
2494
|
return payload, 0
|
|
1895
2495
|
|
|
@@ -1959,28 +2559,247 @@ def add_suggest_candidate(
|
|
|
1959
2559
|
)
|
|
1960
2560
|
|
|
1961
2561
|
|
|
2562
|
+
def trusted_git_executable() -> str:
|
|
2563
|
+
if os.name != "posix":
|
|
2564
|
+
raise OSError("Git execution is unavailable on this platform")
|
|
2565
|
+
executable_names = ("git",)
|
|
2566
|
+
for directory in os.defpath.split(os.pathsep):
|
|
2567
|
+
if not directory:
|
|
2568
|
+
continue
|
|
2569
|
+
for name in executable_names:
|
|
2570
|
+
candidate = Path(directory) / name
|
|
2571
|
+
try:
|
|
2572
|
+
if candidate.is_file() and os.access(candidate, os.X_OK):
|
|
2573
|
+
return str(candidate)
|
|
2574
|
+
except OSError:
|
|
2575
|
+
continue
|
|
2576
|
+
raise OSError("trusted system git executable unavailable")
|
|
2577
|
+
|
|
2578
|
+
|
|
2579
|
+
def guarded_git_environment() -> dict[str, str]:
|
|
2580
|
+
return {
|
|
2581
|
+
"PATH": os.defpath,
|
|
2582
|
+
"LANG": "C",
|
|
2583
|
+
"LC_ALL": "C",
|
|
2584
|
+
"GIT_CONFIG_GLOBAL": os.devnull,
|
|
2585
|
+
"GIT_CONFIG_SYSTEM": os.devnull,
|
|
2586
|
+
"GIT_CONFIG_NOSYSTEM": "1",
|
|
2587
|
+
"GIT_ATTR_NOSYSTEM": "1",
|
|
2588
|
+
"GIT_TERMINAL_PROMPT": "0",
|
|
2589
|
+
"GIT_ASKPASS": os.devnull,
|
|
2590
|
+
"SSH_ASKPASS": os.devnull,
|
|
2591
|
+
"GCM_INTERACTIVE": "Never",
|
|
2592
|
+
"GIT_NO_LAZY_FETCH": "1",
|
|
2593
|
+
"GIT_OPTIONAL_LOCKS": "0",
|
|
2594
|
+
"GIT_PAGER": "cat",
|
|
2595
|
+
"PAGER": "cat",
|
|
2596
|
+
}
|
|
2597
|
+
|
|
2598
|
+
|
|
2599
|
+
def guarded_git_command(root: Path, *args: str) -> list[str]:
|
|
2600
|
+
return [
|
|
2601
|
+
trusted_git_executable(),
|
|
2602
|
+
"-c",
|
|
2603
|
+
"core.fsmonitor=false",
|
|
2604
|
+
"-c",
|
|
2605
|
+
f"core.hooksPath={os.devnull}",
|
|
2606
|
+
"-c",
|
|
2607
|
+
f"core.attributesFile={os.devnull}",
|
|
2608
|
+
"-c",
|
|
2609
|
+
"credential.helper=",
|
|
2610
|
+
"-c",
|
|
2611
|
+
"core.askPass=",
|
|
2612
|
+
"-c",
|
|
2613
|
+
"credential.interactive=never",
|
|
2614
|
+
"-c",
|
|
2615
|
+
"filter.unset.clean=",
|
|
2616
|
+
"-c",
|
|
2617
|
+
"filter.unset.process=",
|
|
2618
|
+
"-c",
|
|
2619
|
+
"filter.unset.required=false",
|
|
2620
|
+
"-c",
|
|
2621
|
+
"filter.unspecified.clean=",
|
|
2622
|
+
"-c",
|
|
2623
|
+
"filter.unspecified.process=",
|
|
2624
|
+
"-c",
|
|
2625
|
+
"filter.unspecified.required=false",
|
|
2626
|
+
"-C",
|
|
2627
|
+
str(root),
|
|
2628
|
+
*args,
|
|
2629
|
+
]
|
|
2630
|
+
|
|
2631
|
+
|
|
2632
|
+
def _signal_process_group(proc: subprocess.Popen[Any], *, force: bool) -> None:
|
|
2633
|
+
requested_signal = getattr(signal, "SIGKILL", signal.SIGTERM) if force else signal.SIGTERM
|
|
2634
|
+
if os.name == "posix" and hasattr(os, "killpg"):
|
|
2635
|
+
try:
|
|
2636
|
+
os.killpg(proc.pid, requested_signal)
|
|
2637
|
+
return
|
|
2638
|
+
except ProcessLookupError:
|
|
2639
|
+
return
|
|
2640
|
+
except OSError:
|
|
2641
|
+
pass
|
|
2642
|
+
if proc.poll() is not None:
|
|
2643
|
+
return
|
|
2644
|
+
try:
|
|
2645
|
+
if force:
|
|
2646
|
+
proc.kill()
|
|
2647
|
+
else:
|
|
2648
|
+
proc.terminate()
|
|
2649
|
+
except OSError:
|
|
2650
|
+
pass
|
|
2651
|
+
|
|
2652
|
+
|
|
2653
|
+
def _run_process_capped(
|
|
2654
|
+
command: list[str],
|
|
2655
|
+
*,
|
|
2656
|
+
stdout_cap: int,
|
|
2657
|
+
stderr_cap: int,
|
|
2658
|
+
timeout_seconds: float,
|
|
2659
|
+
environment: dict[str, str] | None = None,
|
|
2660
|
+
stdin_data: bytes | None = None,
|
|
2661
|
+
stdin_cap_bytes: int | None = None,
|
|
2662
|
+
) -> tuple[int, bytes, bytes, bool, bool]:
|
|
2663
|
+
if stdin_data is not None and (
|
|
2664
|
+
stdin_cap_bytes is None or len(stdin_data) > stdin_cap_bytes
|
|
2665
|
+
):
|
|
2666
|
+
raise PackError("process stdin exceeds cap")
|
|
2667
|
+
proc = subprocess.Popen(
|
|
2668
|
+
command,
|
|
2669
|
+
stdin=subprocess.PIPE if stdin_data is not None else subprocess.DEVNULL,
|
|
2670
|
+
stdout=subprocess.PIPE,
|
|
2671
|
+
stderr=subprocess.PIPE,
|
|
2672
|
+
text=False,
|
|
2673
|
+
start_new_session=os.name == "posix",
|
|
2674
|
+
env=environment,
|
|
2675
|
+
)
|
|
2676
|
+
buffers: dict[str, list[bytes]] = {"stdout": [], "stderr": []}
|
|
2677
|
+
capped = {"stdout": False, "stderr": False}
|
|
2678
|
+
stop = threading.Event()
|
|
2679
|
+
|
|
2680
|
+
def drain(name: str, stream: Any, cap: int) -> None:
|
|
2681
|
+
total = 0
|
|
2682
|
+
try:
|
|
2683
|
+
while not stop.is_set() and total <= cap:
|
|
2684
|
+
chunk = stream.read(min(64 * 1024, cap + 1 - total))
|
|
2685
|
+
if not chunk:
|
|
2686
|
+
break
|
|
2687
|
+
buffers[name].append(chunk)
|
|
2688
|
+
total += len(chunk)
|
|
2689
|
+
if total > cap:
|
|
2690
|
+
capped[name] = True
|
|
2691
|
+
stop.set()
|
|
2692
|
+
_signal_process_group(proc, force=False)
|
|
2693
|
+
break
|
|
2694
|
+
finally:
|
|
2695
|
+
try:
|
|
2696
|
+
stream.close()
|
|
2697
|
+
except OSError:
|
|
2698
|
+
pass
|
|
2699
|
+
|
|
2700
|
+
threads = [
|
|
2701
|
+
threading.Thread(target=drain, args=("stdout", proc.stdout, stdout_cap), daemon=True),
|
|
2702
|
+
threading.Thread(target=drain, args=("stderr", proc.stderr, stderr_cap), daemon=True),
|
|
2703
|
+
]
|
|
2704
|
+
if stdin_data is not None:
|
|
2705
|
+
def write_stdin() -> None:
|
|
2706
|
+
try:
|
|
2707
|
+
assert proc.stdin is not None
|
|
2708
|
+
view = memoryview(stdin_data)
|
|
2709
|
+
for offset in range(0, len(view), 64 * 1024):
|
|
2710
|
+
if stop.is_set():
|
|
2711
|
+
break
|
|
2712
|
+
proc.stdin.write(view[offset : offset + 64 * 1024])
|
|
2713
|
+
proc.stdin.flush()
|
|
2714
|
+
except (BrokenPipeError, OSError, ValueError):
|
|
2715
|
+
pass
|
|
2716
|
+
finally:
|
|
2717
|
+
if proc.stdin is not None:
|
|
2718
|
+
try:
|
|
2719
|
+
proc.stdin.close()
|
|
2720
|
+
except OSError:
|
|
2721
|
+
pass
|
|
2722
|
+
|
|
2723
|
+
threads.append(threading.Thread(target=write_stdin, daemon=True))
|
|
2724
|
+
for thread in threads:
|
|
2725
|
+
thread.start()
|
|
2726
|
+
timed_out = False
|
|
2727
|
+
try:
|
|
2728
|
+
proc.wait(timeout=timeout_seconds)
|
|
2729
|
+
except subprocess.TimeoutExpired:
|
|
2730
|
+
timed_out = True
|
|
2731
|
+
stop.set()
|
|
2732
|
+
_signal_process_group(proc, force=False)
|
|
2733
|
+
try:
|
|
2734
|
+
proc.wait(timeout=0.2)
|
|
2735
|
+
except subprocess.TimeoutExpired:
|
|
2736
|
+
_signal_process_group(proc, force=True)
|
|
2737
|
+
try:
|
|
2738
|
+
proc.wait(timeout=2)
|
|
2739
|
+
except subprocess.TimeoutExpired:
|
|
2740
|
+
pass
|
|
2741
|
+
if capped["stdout"] or capped["stderr"] or timed_out:
|
|
2742
|
+
_signal_process_group(proc, force=True)
|
|
2743
|
+
for thread in threads:
|
|
2744
|
+
thread.join(0.5)
|
|
2745
|
+
if any(thread.is_alive() for thread in threads):
|
|
2746
|
+
stop.set()
|
|
2747
|
+
_signal_process_group(proc, force=True)
|
|
2748
|
+
for thread in threads:
|
|
2749
|
+
thread.join(0.2)
|
|
2750
|
+
stdout = b"".join(buffers["stdout"])[:stdout_cap]
|
|
2751
|
+
stderr = b"".join(buffers["stderr"])[:stderr_cap]
|
|
2752
|
+
return proc.returncode if proc.returncode is not None else -1, stdout, stderr, capped["stdout"], capped["stderr"] or timed_out
|
|
2753
|
+
|
|
2754
|
+
|
|
1962
2755
|
def run_git_diff(root: Path, diff_ref: str) -> str:
|
|
1963
2756
|
ref = diff_ref.strip()
|
|
1964
2757
|
if not ref:
|
|
1965
2758
|
raise PackError("empty --diff")
|
|
1966
|
-
|
|
2759
|
+
git_args = [
|
|
2760
|
+
"diff",
|
|
2761
|
+
"--no-ext-diff",
|
|
2762
|
+
"--no-textconv",
|
|
2763
|
+
"--ignore-submodules=all",
|
|
2764
|
+
"--unified=3",
|
|
2765
|
+
]
|
|
1967
2766
|
if ref in {"staged", "--staged", "cached", "--cached"}:
|
|
1968
|
-
|
|
2767
|
+
git_args.append("--cached")
|
|
1969
2768
|
elif ref in {"worktree", "unstaged", "working-tree"}:
|
|
1970
2769
|
pass
|
|
1971
2770
|
elif ref.startswith("-"):
|
|
1972
2771
|
raise PackError("invalid --diff: revision must not start with '-'")
|
|
1973
2772
|
else:
|
|
1974
|
-
|
|
2773
|
+
git_args.append(ref)
|
|
1975
2774
|
try:
|
|
1976
|
-
|
|
2775
|
+
reject_configured_git_filters(root)
|
|
2776
|
+
command = guarded_git_command(root, *git_args)
|
|
2777
|
+
returncode, stdout, stderr, stdout_capped, stderr_capped_or_timeout = _run_process_capped(
|
|
2778
|
+
command,
|
|
2779
|
+
stdout_cap=MAX_SUGGEST_INPUT_BYTES,
|
|
2780
|
+
stderr_cap=MAX_GIT_DIFF_STDERR_BYTES,
|
|
2781
|
+
timeout_seconds=GIT_DIFF_TIMEOUT_SECONDS,
|
|
2782
|
+
environment=guarded_git_environment(),
|
|
2783
|
+
)
|
|
1977
2784
|
except (OSError, UnicodeError, subprocess.TimeoutExpired) as exc:
|
|
1978
2785
|
raise PackError(f"could not read diff: {exc.__class__.__name__}") from exc
|
|
1979
|
-
if
|
|
1980
|
-
|
|
2786
|
+
if stdout_capped:
|
|
2787
|
+
raise PackError(f"could not read diff: diff output exceeds cap ({MAX_SUGGEST_INPUT_BYTES} bytes)")
|
|
2788
|
+
if stderr_capped_or_timeout:
|
|
2789
|
+
raise PackError("could not read diff: stderr cap or timeout exceeded")
|
|
2790
|
+
stdout_text = stdout.decode("utf-8", "replace")
|
|
2791
|
+
stderr_text = stderr.decode("utf-8", "replace")
|
|
2792
|
+
if returncode != 0:
|
|
2793
|
+
detail = sanitize_text(
|
|
2794
|
+
stderr_text or stdout_text or "git diff failed",
|
|
2795
|
+
context="command_search_diff",
|
|
2796
|
+
)[0].strip().splitlines()
|
|
1981
2797
|
message = detail[0] if detail else "git diff failed"
|
|
1982
2798
|
raise PackError(f"could not read diff: {cap_label(message, default='git diff failed', limit=160)}")
|
|
1983
|
-
return sanitize_text(
|
|
2799
|
+
return sanitize_text(
|
|
2800
|
+
stdout_text,
|
|
2801
|
+
context="command_search_diff",
|
|
2802
|
+
)[0]
|
|
1984
2803
|
|
|
1985
2804
|
|
|
1986
2805
|
def collect_diff_candidates(root: Path, diff_ref: str, query_terms: set[str], context_lines: int) -> list[SuggestCandidate]:
|
|
@@ -2091,100 +2910,269 @@ def collect_output_candidates(
|
|
|
2091
2910
|
return candidates, omitted
|
|
2092
2911
|
|
|
2093
2912
|
|
|
2094
|
-
def
|
|
2095
|
-
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
try:
|
|
2106
|
-
while total <= limit:
|
|
2107
|
-
chunk = proc.stdout.read(min(GIT_LS_FILES_READ_CHUNK_BYTES, limit + 1 - total))
|
|
2108
|
-
if not chunk:
|
|
2109
|
-
break
|
|
2110
|
-
chunks.append(chunk)
|
|
2111
|
-
total += len(chunk)
|
|
2112
|
-
if total > limit:
|
|
2113
|
-
capped = True
|
|
2114
|
-
break
|
|
2115
|
-
finally:
|
|
2116
|
-
if capped and proc.poll() is None:
|
|
2117
|
-
try:
|
|
2118
|
-
proc.terminate()
|
|
2119
|
-
except OSError:
|
|
2120
|
-
pass
|
|
2121
|
-
try:
|
|
2122
|
-
proc.stdout.close()
|
|
2123
|
-
except OSError:
|
|
2124
|
-
pass
|
|
2913
|
+
def _read_git_stdout_capped(
|
|
2914
|
+
proc: subprocess.Popen[bytes],
|
|
2915
|
+
limit: int,
|
|
2916
|
+
timeout_seconds: float,
|
|
2917
|
+
) -> tuple[bytes, bool]:
|
|
2918
|
+
if proc.stdout is None:
|
|
2919
|
+
return b"", False
|
|
2920
|
+
chunks: list[bytes] = []
|
|
2921
|
+
total = 0
|
|
2922
|
+
capped = False
|
|
2923
|
+
timed_out = False
|
|
2125
2924
|
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2925
|
+
def reader() -> None:
|
|
2926
|
+
nonlocal total, capped
|
|
2927
|
+
try:
|
|
2928
|
+
while total <= limit:
|
|
2929
|
+
chunk = proc.stdout.read(min(GIT_LS_FILES_READ_CHUNK_BYTES, limit + 1 - total))
|
|
2930
|
+
if not chunk:
|
|
2931
|
+
break
|
|
2932
|
+
chunks.append(chunk)
|
|
2933
|
+
total += len(chunk)
|
|
2934
|
+
if total > limit:
|
|
2935
|
+
capped = True
|
|
2936
|
+
break
|
|
2937
|
+
finally:
|
|
2938
|
+
if capped:
|
|
2939
|
+
_signal_process_group(proc, force=False)
|
|
2131
2940
|
try:
|
|
2132
|
-
proc.
|
|
2941
|
+
proc.stdout.close()
|
|
2133
2942
|
except OSError:
|
|
2134
2943
|
pass
|
|
2944
|
+
|
|
2945
|
+
thread = threading.Thread(target=reader, daemon=True)
|
|
2946
|
+
thread.start()
|
|
2947
|
+
thread.join(timeout_seconds)
|
|
2948
|
+
if thread.is_alive():
|
|
2949
|
+
timed_out = True
|
|
2950
|
+
_signal_process_group(proc, force=False)
|
|
2951
|
+
try:
|
|
2952
|
+
proc.wait(timeout=0.2 if timed_out else 2)
|
|
2953
|
+
except subprocess.TimeoutExpired:
|
|
2954
|
+
_signal_process_group(proc, force=True)
|
|
2135
2955
|
try:
|
|
2136
2956
|
proc.wait(timeout=2)
|
|
2137
2957
|
except subprocess.TimeoutExpired:
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
raw_output
|
|
2148
|
-
|
|
2149
|
-
|
|
2958
|
+
pass
|
|
2959
|
+
if capped or timed_out:
|
|
2960
|
+
_signal_process_group(proc, force=True)
|
|
2961
|
+
thread.join(0.5)
|
|
2962
|
+
raw_output = b"".join(chunks)[:limit]
|
|
2963
|
+
complete = (
|
|
2964
|
+
proc.returncode == 0
|
|
2965
|
+
and not capped
|
|
2966
|
+
and not timed_out
|
|
2967
|
+
and (not raw_output or raw_output.endswith(b"\0"))
|
|
2968
|
+
)
|
|
2969
|
+
return raw_output, complete
|
|
2150
2970
|
|
|
2151
|
-
|
|
2152
|
-
|
|
2971
|
+
|
|
2972
|
+
def _git_ls_files_raw(root: Path) -> tuple[bytes, bool, int | None]:
|
|
2153
2973
|
try:
|
|
2154
2974
|
proc = subprocess.Popen(
|
|
2155
|
-
|
|
2975
|
+
guarded_git_command(root, "ls-files", "-z"),
|
|
2976
|
+
stdin=subprocess.DEVNULL,
|
|
2156
2977
|
stdout=subprocess.PIPE,
|
|
2157
2978
|
stderr=subprocess.DEVNULL,
|
|
2158
2979
|
text=False,
|
|
2980
|
+
start_new_session=os.name == "posix",
|
|
2981
|
+
env=guarded_git_environment(),
|
|
2982
|
+
)
|
|
2983
|
+
raw, complete = _read_git_stdout_capped(
|
|
2984
|
+
proc,
|
|
2985
|
+
MAX_GIT_LS_FILES_OUTPUT_BYTES,
|
|
2986
|
+
10,
|
|
2159
2987
|
)
|
|
2160
|
-
raw,
|
|
2161
|
-
git_returncode = proc.returncode
|
|
2988
|
+
return raw, complete, proc.returncode
|
|
2162
2989
|
except (OSError, subprocess.TimeoutExpired):
|
|
2163
|
-
|
|
2990
|
+
return b"", False, None
|
|
2991
|
+
|
|
2992
|
+
|
|
2993
|
+
def _iter_nul_fields(raw: bytes):
|
|
2994
|
+
view = memoryview(raw)
|
|
2995
|
+
start = 0
|
|
2996
|
+
while start < len(raw):
|
|
2997
|
+
end = raw.find(b"\0", start)
|
|
2998
|
+
if end < 0:
|
|
2999
|
+
return
|
|
3000
|
+
yield view[start:end]
|
|
3001
|
+
start = end + 1
|
|
3002
|
+
|
|
3003
|
+
|
|
3004
|
+
def git_ls_files(root: Path, diagnostics: dict[str, Any] | None = None) -> list[str]:
|
|
3005
|
+
raw, git_complete, git_returncode = _git_ls_files_raw(root)
|
|
2164
3006
|
if raw:
|
|
2165
3007
|
if not raw.endswith(b"\0"):
|
|
2166
3008
|
raw = raw.rsplit(b"\0", 1)[0] if b"\0" in raw else b""
|
|
2167
|
-
|
|
2168
|
-
|
|
3009
|
+
retained_parts: list[bytes] = []
|
|
3010
|
+
file_cap_reached = False
|
|
3011
|
+
for part_view in _iter_nul_fields(raw):
|
|
3012
|
+
if not part_view:
|
|
3013
|
+
continue
|
|
3014
|
+
if len(retained_parts) >= MAX_QUERY_SCAN_FILES:
|
|
3015
|
+
file_cap_reached = True
|
|
3016
|
+
break
|
|
3017
|
+
retained_parts.append(bytes(part_view))
|
|
3018
|
+
if diagnostics is not None:
|
|
3019
|
+
diagnostics.update({
|
|
3020
|
+
"mode": "git",
|
|
3021
|
+
"truncated": not git_complete or file_cap_reached,
|
|
3022
|
+
"truncation_reason": (
|
|
3023
|
+
"git_output_cap_or_timeout"
|
|
3024
|
+
if not git_complete
|
|
3025
|
+
else "file_cap" if file_cap_reached else None
|
|
3026
|
+
),
|
|
3027
|
+
})
|
|
3028
|
+
return [part.decode("utf-8", "replace") for part in retained_parts]
|
|
3029
|
+
if git_returncode == 0:
|
|
3030
|
+
if diagnostics is not None:
|
|
3031
|
+
diagnostics.update({"mode": "git", "truncated": False, "truncation_reason": None})
|
|
3032
|
+
return []
|
|
3033
|
+
if git_returncode is not None and git_returncode < 0:
|
|
3034
|
+
if diagnostics is not None:
|
|
3035
|
+
diagnostics.update({
|
|
3036
|
+
"mode": "git",
|
|
3037
|
+
"truncated": True,
|
|
3038
|
+
"truncation_reason": "git_output_cap_or_timeout",
|
|
3039
|
+
})
|
|
2169
3040
|
return []
|
|
2170
3041
|
out: list[str] = []
|
|
2171
3042
|
skip_dirs = {".git", ".omx", ".context-guard", "node_modules", "dist", "build", "__pycache__"}
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
|
|
2178
|
-
|
|
2179
|
-
|
|
3043
|
+
started = time.monotonic()
|
|
3044
|
+
visited_dirs = 0
|
|
3045
|
+
visited_entries = 0
|
|
3046
|
+
truncation_reason: str | None = None
|
|
3047
|
+
pending: deque[tuple[Path, int]] = deque([(root, 0)])
|
|
3048
|
+
while pending:
|
|
3049
|
+
if time.monotonic() - started > MAX_QUERY_WALK_SECONDS:
|
|
3050
|
+
truncation_reason = "time_cap"
|
|
3051
|
+
break
|
|
3052
|
+
if visited_dirs >= MAX_QUERY_WALK_DIRS:
|
|
3053
|
+
truncation_reason = "directory_cap"
|
|
3054
|
+
break
|
|
3055
|
+
current_path, depth = pending.popleft()
|
|
3056
|
+
visited_dirs += 1
|
|
3057
|
+
try:
|
|
3058
|
+
iterator = os.scandir(current_path)
|
|
3059
|
+
except OSError:
|
|
3060
|
+
truncation_reason = "unsafe_path"
|
|
3061
|
+
break
|
|
3062
|
+
child_dirs: list[Path] = []
|
|
3063
|
+
try:
|
|
3064
|
+
with iterator:
|
|
3065
|
+
for entry in iterator:
|
|
3066
|
+
if time.monotonic() - started > MAX_QUERY_WALK_SECONDS:
|
|
3067
|
+
truncation_reason = "time_cap"
|
|
3068
|
+
break
|
|
3069
|
+
if visited_entries >= MAX_QUERY_WALK_ENTRIES:
|
|
3070
|
+
truncation_reason = "entry_cap"
|
|
3071
|
+
break
|
|
3072
|
+
visited_entries += 1
|
|
3073
|
+
name = entry.name
|
|
3074
|
+
try:
|
|
3075
|
+
is_dir = entry.is_dir(follow_symlinks=False)
|
|
3076
|
+
is_file = entry.is_file(follow_symlinks=False)
|
|
3077
|
+
except OSError:
|
|
3078
|
+
continue
|
|
3079
|
+
if is_dir:
|
|
3080
|
+
if name in skip_dirs or name.startswith(".pytest"):
|
|
3081
|
+
continue
|
|
3082
|
+
if depth >= MAX_QUERY_WALK_DEPTH:
|
|
3083
|
+
truncation_reason = truncation_reason or "depth_cap"
|
|
3084
|
+
continue
|
|
3085
|
+
child_dirs.append(current_path / name)
|
|
3086
|
+
elif is_file:
|
|
3087
|
+
try:
|
|
3088
|
+
rel = (current_path / name).relative_to(root).as_posix()
|
|
3089
|
+
except ValueError:
|
|
3090
|
+
truncation_reason = "unsafe_path"
|
|
3091
|
+
break
|
|
3092
|
+
out.append(rel)
|
|
3093
|
+
if len(out) >= MAX_QUERY_SCAN_FILES:
|
|
3094
|
+
truncation_reason = "file_cap"
|
|
3095
|
+
break
|
|
3096
|
+
except OSError:
|
|
3097
|
+
truncation_reason = "unsafe_path"
|
|
3098
|
+
break
|
|
3099
|
+
if truncation_reason in {"time_cap", "entry_cap", "file_cap", "unsafe_path"}:
|
|
3100
|
+
break
|
|
3101
|
+
for child in reversed(sorted(child_dirs, key=lambda path: path.name)):
|
|
3102
|
+
pending.appendleft((child, depth + 1))
|
|
3103
|
+
if diagnostics is not None:
|
|
3104
|
+
diagnostics.update({
|
|
3105
|
+
"mode": "walk",
|
|
3106
|
+
"truncated": truncation_reason is not None,
|
|
3107
|
+
"truncation_reason": truncation_reason,
|
|
3108
|
+
"visited_dirs": min(visited_dirs, MAX_QUERY_WALK_DIRS),
|
|
3109
|
+
"visited_entries": min(visited_entries, MAX_QUERY_WALK_ENTRIES),
|
|
3110
|
+
})
|
|
2180
3111
|
return out
|
|
2181
3112
|
|
|
2182
3113
|
|
|
2183
|
-
def
|
|
3114
|
+
def reject_configured_git_filters(root: Path) -> None:
|
|
3115
|
+
raw_paths, complete, returncode = _git_ls_files_raw(root)
|
|
3116
|
+
if returncode != 0 or not complete:
|
|
3117
|
+
raise PackError("could not verify git filters: tracked path scan failed or truncated")
|
|
3118
|
+
if not raw_paths:
|
|
3119
|
+
return
|
|
3120
|
+
if len(raw_paths) > MAX_GIT_ATTR_INPUT_BYTES:
|
|
3121
|
+
raise PackError("could not verify git filters: tracked path input exceeds cap")
|
|
3122
|
+
|
|
3123
|
+
try:
|
|
3124
|
+
command = guarded_git_command(
|
|
3125
|
+
root,
|
|
3126
|
+
"check-attr",
|
|
3127
|
+
"-z",
|
|
3128
|
+
"--stdin",
|
|
3129
|
+
"filter",
|
|
3130
|
+
)
|
|
3131
|
+
returncode, stdout, _stderr, stdout_capped, failed_or_timed_out = _run_process_capped(
|
|
3132
|
+
command,
|
|
3133
|
+
stdout_cap=MAX_GIT_ATTR_OUTPUT_BYTES,
|
|
3134
|
+
stderr_cap=MAX_GIT_DIFF_STDERR_BYTES,
|
|
3135
|
+
timeout_seconds=GIT_ATTR_TIMEOUT_SECONDS,
|
|
3136
|
+
environment=guarded_git_environment(),
|
|
3137
|
+
stdin_data=raw_paths,
|
|
3138
|
+
stdin_cap_bytes=MAX_GIT_ATTR_INPUT_BYTES,
|
|
3139
|
+
)
|
|
3140
|
+
except (OSError, UnicodeError, subprocess.TimeoutExpired) as exc:
|
|
3141
|
+
raise PackError(f"could not verify git filters: {exc.__class__.__name__}") from exc
|
|
3142
|
+
if stdout_capped or failed_or_timed_out or returncode != 0:
|
|
3143
|
+
raise PackError("could not verify git filters: check-attr failed or exceeded cap")
|
|
3144
|
+
if not stdout.endswith(b"\0"):
|
|
3145
|
+
raise PackError("could not verify git filters: malformed check-attr output")
|
|
3146
|
+
output_fields = iter(_iter_nul_fields(stdout))
|
|
3147
|
+
for expected_path in _iter_nul_fields(raw_paths):
|
|
3148
|
+
try:
|
|
3149
|
+
path = next(output_fields)
|
|
3150
|
+
attribute = next(output_fields)
|
|
3151
|
+
value = next(output_fields)
|
|
3152
|
+
except StopIteration as exc:
|
|
3153
|
+
raise PackError("could not verify git filters: incomplete check-attr output") from exc
|
|
3154
|
+
if path != expected_path or bytes(attribute) != b"filter":
|
|
3155
|
+
raise PackError("could not verify git filters: mismatched check-attr output")
|
|
3156
|
+
if bytes(value) not in {b"unspecified", b"unset"}:
|
|
3157
|
+
raise PackError("git diff blocked: configured filter attribute")
|
|
3158
|
+
try:
|
|
3159
|
+
next(output_fields)
|
|
3160
|
+
except StopIteration:
|
|
3161
|
+
return
|
|
3162
|
+
raise PackError("could not verify git filters: excess check-attr output")
|
|
3163
|
+
|
|
3164
|
+
|
|
3165
|
+
def collect_query_candidates(
|
|
3166
|
+
root: Path,
|
|
3167
|
+
query_terms: set[str],
|
|
3168
|
+
context_lines: int,
|
|
3169
|
+
*,
|
|
3170
|
+
diagnostics: dict[str, Any] | None = None,
|
|
3171
|
+
) -> list[SuggestCandidate]:
|
|
2184
3172
|
if not query_terms:
|
|
2185
3173
|
return []
|
|
2186
3174
|
candidates: list[SuggestCandidate] = []
|
|
2187
|
-
for rel_path in git_ls_files(root):
|
|
3175
|
+
for rel_path in git_ls_files(root, diagnostics):
|
|
2188
3176
|
rel, reason = lexical_rel(rel_path)
|
|
2189
3177
|
if rel is None or reason:
|
|
2190
3178
|
continue
|
|
@@ -2274,16 +3262,28 @@ def suggested_source_payload(source: ResolvedSource, candidate: SuggestCandidate
|
|
|
2274
3262
|
return payload
|
|
2275
3263
|
|
|
2276
3264
|
|
|
2277
|
-
def normalize_suggest_source(
|
|
3265
|
+
def normalize_suggest_source(
|
|
3266
|
+
root: Path,
|
|
3267
|
+
candidate: SuggestCandidate,
|
|
3268
|
+
*,
|
|
3269
|
+
source_cache: _SourceSnapshotCache | None = None,
|
|
3270
|
+
input_budget: _SourceInputBudget | None = None,
|
|
3271
|
+
) -> tuple[ResolvedSource | None, dict[str, Any] | None]:
|
|
3272
|
+
effective_lines = candidate.lines or LineRange(1, SUGGEST_WHOLE_FILE_MAX_LINES)
|
|
2278
3273
|
spec = SourceSpec(
|
|
2279
3274
|
path=candidate.path,
|
|
2280
3275
|
priority=candidate.score,
|
|
2281
|
-
lines=
|
|
3276
|
+
lines=effective_lines,
|
|
2282
3277
|
label=candidate.label,
|
|
2283
3278
|
input_index=candidate.input_index,
|
|
2284
3279
|
origin="suggest",
|
|
2285
3280
|
)
|
|
2286
|
-
source, omitted_item = resolve_source(
|
|
3281
|
+
source, omitted_item = resolve_source(
|
|
3282
|
+
root,
|
|
3283
|
+
spec,
|
|
3284
|
+
source_cache=source_cache,
|
|
3285
|
+
input_budget=input_budget,
|
|
3286
|
+
)
|
|
2287
3287
|
if omitted_item is not None:
|
|
2288
3288
|
omitted_item["reason"] = omitted_item.get("reason") or candidate.reason
|
|
2289
3289
|
omitted_item["suggest_reason"] = candidate.reason
|
|
@@ -2291,20 +3291,6 @@ def normalize_suggest_source(root: Path, candidate: SuggestCandidate) -> tuple[R
|
|
|
2291
3291
|
assert source is not None
|
|
2292
3292
|
if source.redacted_path:
|
|
2293
3293
|
return None, omission(spec, "redacted_path", path=source.display_path, redacted_path=True)
|
|
2294
|
-
if spec.lines is None and source.total_lines > SUGGEST_WHOLE_FILE_MAX_LINES:
|
|
2295
|
-
capped = SourceSpec(
|
|
2296
|
-
path=candidate.path,
|
|
2297
|
-
priority=candidate.score,
|
|
2298
|
-
lines=LineRange(1, min(SUGGEST_WHOLE_FILE_MAX_LINES, source.total_lines)),
|
|
2299
|
-
label=candidate.label,
|
|
2300
|
-
input_index=candidate.input_index,
|
|
2301
|
-
origin="suggest",
|
|
2302
|
-
)
|
|
2303
|
-
source, omitted_item = resolve_source(root, capped)
|
|
2304
|
-
if omitted_item is not None:
|
|
2305
|
-
omitted_item["suggest_reason"] = candidate.reason
|
|
2306
|
-
return None, omitted_item
|
|
2307
|
-
assert source is not None
|
|
2308
3294
|
return source, None
|
|
2309
3295
|
|
|
2310
3296
|
|
|
@@ -2844,7 +3830,15 @@ def build_adaptive_k_advisory(
|
|
|
2844
3830
|
}
|
|
2845
3831
|
|
|
2846
3832
|
|
|
2847
|
-
def suggest_pack(
|
|
3833
|
+
def suggest_pack(
|
|
3834
|
+
root: Path,
|
|
3835
|
+
args: argparse.Namespace,
|
|
3836
|
+
*,
|
|
3837
|
+
root_arg: str,
|
|
3838
|
+
_source_cache: _SourceSnapshotCache | None = None,
|
|
3839
|
+
_input_budget: _SourceInputBudget | None = None,
|
|
3840
|
+
) -> tuple[dict[str, Any], int]:
|
|
3841
|
+
input_budget = _input_budget if _input_budget is not None else _SourceInputBudget()
|
|
2848
3842
|
query_text, _query_redactions = sanitize_text(args.query or "")
|
|
2849
3843
|
query = " ".join(query_text.split())
|
|
2850
3844
|
query_terms = suggest_tokens(query)
|
|
@@ -2874,7 +3868,23 @@ def suggest_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tupl
|
|
|
2874
3868
|
candidates.extend(test_candidates)
|
|
2875
3869
|
omitted.extend(output_omitted)
|
|
2876
3870
|
omitted.extend(test_omitted)
|
|
2877
|
-
|
|
3871
|
+
query_scan: dict[str, Any] = {}
|
|
3872
|
+
candidates.extend(
|
|
3873
|
+
collect_query_candidates(
|
|
3874
|
+
root,
|
|
3875
|
+
query_terms,
|
|
3876
|
+
context_lines,
|
|
3877
|
+
diagnostics=query_scan,
|
|
3878
|
+
)
|
|
3879
|
+
)
|
|
3880
|
+
if query_scan.get("truncated"):
|
|
3881
|
+
omitted.append({
|
|
3882
|
+
"path": "repository",
|
|
3883
|
+
"status": "omitted",
|
|
3884
|
+
"reason": "query_scan_truncated",
|
|
3885
|
+
"scan_truncation_reason": query_scan.get("truncation_reason"),
|
|
3886
|
+
"priority": 0,
|
|
3887
|
+
})
|
|
2878
3888
|
|
|
2879
3889
|
candidates.sort(key=lambda item: (-item.score, item.input_index, item.path, item.lines.identity() if item.lines else "0:0"))
|
|
2880
3890
|
seen: set[tuple[str, str]] = set()
|
|
@@ -2901,7 +3911,12 @@ def suggest_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tupl
|
|
|
2901
3911
|
continue
|
|
2902
3912
|
if rel is not None:
|
|
2903
3913
|
seen.add(identity)
|
|
2904
|
-
source, omitted_item = normalize_suggest_source(
|
|
3914
|
+
source, omitted_item = normalize_suggest_source(
|
|
3915
|
+
root,
|
|
3916
|
+
candidate,
|
|
3917
|
+
source_cache=_source_cache,
|
|
3918
|
+
input_budget=input_budget,
|
|
3919
|
+
)
|
|
2905
3920
|
if omitted_item is not None:
|
|
2906
3921
|
omitted_item["priority"] = candidate.score
|
|
2907
3922
|
omitted_item["suggest_reason"] = candidate.reason
|
|
@@ -2941,7 +3956,12 @@ def suggest_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tupl
|
|
|
2941
3956
|
input_index=candidate.input_index,
|
|
2942
3957
|
origin="suggest",
|
|
2943
3958
|
)
|
|
2944
|
-
source, omitted_item = resolve_source(
|
|
3959
|
+
source, omitted_item = resolve_source(
|
|
3960
|
+
root,
|
|
3961
|
+
partial_spec,
|
|
3962
|
+
source_cache=_source_cache,
|
|
3963
|
+
input_budget=input_budget,
|
|
3964
|
+
)
|
|
2945
3965
|
if omitted_item is not None:
|
|
2946
3966
|
omitted_item["priority"] = candidate.score
|
|
2947
3967
|
omitted_item["suggest_reason"] = candidate.reason
|
|
@@ -3000,6 +4020,17 @@ def suggest_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tupl
|
|
|
3000
4020
|
"Byte and token values are pack-size proxies, not billing claims.",
|
|
3001
4021
|
],
|
|
3002
4022
|
}
|
|
4023
|
+
if query_scan:
|
|
4024
|
+
payload["query_scan"] = {
|
|
4025
|
+
**query_scan,
|
|
4026
|
+
"limits": {
|
|
4027
|
+
"files": MAX_QUERY_SCAN_FILES,
|
|
4028
|
+
"directories": MAX_QUERY_WALK_DIRS,
|
|
4029
|
+
"entries": MAX_QUERY_WALK_ENTRIES,
|
|
4030
|
+
"depth": MAX_QUERY_WALK_DEPTH,
|
|
4031
|
+
"seconds": MAX_QUERY_WALK_SECONDS,
|
|
4032
|
+
},
|
|
4033
|
+
}
|
|
3003
4034
|
if build_hint_omitted_reason:
|
|
3004
4035
|
payload["build_hint_omitted_reason"] = build_hint_omitted_reason
|
|
3005
4036
|
if getattr(args, "adaptive_k", False):
|
|
@@ -3025,6 +4056,57 @@ def line_range_identity(value: object) -> str:
|
|
|
3025
4056
|
return str(value)
|
|
3026
4057
|
|
|
3027
4058
|
|
|
4059
|
+
def apply_adaptive_k_manifest(
|
|
4060
|
+
manifest: dict[str, Any],
|
|
4061
|
+
advisory: dict[str, Any],
|
|
4062
|
+
) -> tuple[dict[str, Any], dict[str, Any]]:
|
|
4063
|
+
raw_sources = manifest.get("sources", [])
|
|
4064
|
+
sources = copy.deepcopy(raw_sources) if isinstance(raw_sources, list) else []
|
|
4065
|
+
gates = advisory.get("regression_gates", {})
|
|
4066
|
+
gates_passed = isinstance(gates, dict) and gates.get("status") == "pass"
|
|
4067
|
+
recommended_k = max(0, int(advisory.get("recommended_k", 0) or 0))
|
|
4068
|
+
protected_prefixes = ("file:", "output:", "test-output:", "diff:")
|
|
4069
|
+
protected_indexes = {
|
|
4070
|
+
index
|
|
4071
|
+
for index, item in enumerate(sources)
|
|
4072
|
+
if isinstance(item, dict)
|
|
4073
|
+
and str(item.get("label", "")).startswith(protected_prefixes)
|
|
4074
|
+
}
|
|
4075
|
+
retained: list[dict[str, Any]] = []
|
|
4076
|
+
if gates_passed:
|
|
4077
|
+
target_count = max(recommended_k, len(protected_indexes))
|
|
4078
|
+
for index, item in enumerate(sources):
|
|
4079
|
+
if not isinstance(item, dict):
|
|
4080
|
+
continue
|
|
4081
|
+
if index in protected_indexes or len(retained) < target_count:
|
|
4082
|
+
retained.append(item)
|
|
4083
|
+
else:
|
|
4084
|
+
retained = [item for item in sources if isinstance(item, dict)]
|
|
4085
|
+
omitted_count = len(sources) - len(retained)
|
|
4086
|
+
status = "applied" if gates_passed and omitted_count else "no_change"
|
|
4087
|
+
if not gates_passed:
|
|
4088
|
+
status = "gate_failed"
|
|
4089
|
+
applied_manifest = {"version": 1, "sources": retained}
|
|
4090
|
+
return applied_manifest, {
|
|
4091
|
+
"schema_version": ADAPTIVE_K_APPLICATION_SCHEMA_VERSION,
|
|
4092
|
+
"mode": "explicit_opt_in",
|
|
4093
|
+
"status": status,
|
|
4094
|
+
"recommended_k": recommended_k,
|
|
4095
|
+
"input_source_count": len(sources),
|
|
4096
|
+
"applied_source_count": len(retained),
|
|
4097
|
+
"omitted_source_count": omitted_count,
|
|
4098
|
+
"regression_gates_passed": gates_passed,
|
|
4099
|
+
"explicit_sources_retained": all(
|
|
4100
|
+
sources[index] in retained for index in protected_indexes
|
|
4101
|
+
),
|
|
4102
|
+
"claim_boundary": {
|
|
4103
|
+
"deterministic_local_only": True,
|
|
4104
|
+
"exact_source_fallback_retained": True,
|
|
4105
|
+
"provider_token_or_cost_savings_claim_allowed": False,
|
|
4106
|
+
},
|
|
4107
|
+
}
|
|
4108
|
+
|
|
4109
|
+
|
|
3028
4110
|
def copy_explain_fields(item: dict[str, Any], fields: tuple[str, ...]) -> dict[str, Any]:
|
|
3029
4111
|
out: dict[str, Any] = {}
|
|
3030
4112
|
for field in fields:
|
|
@@ -3094,7 +4176,12 @@ def is_repo_map_text_path(path: str) -> bool:
|
|
|
3094
4176
|
return Path(path).suffix.lower() in REPO_MAP_TEXT_EXTENSIONS
|
|
3095
4177
|
|
|
3096
4178
|
|
|
3097
|
-
def read_repo_map_text(
|
|
4179
|
+
def read_repo_map_text(
|
|
4180
|
+
root: Path,
|
|
4181
|
+
rel_path: str,
|
|
4182
|
+
*,
|
|
4183
|
+
source_identities_out: dict[str, tuple[int, int, int, int, int, int, int, int]] | None = None,
|
|
4184
|
+
) -> tuple[dict[str, Any] | None, dict[str, Any] | None]:
|
|
3098
4185
|
rel, reason = lexical_rel(rel_path)
|
|
3099
4186
|
if rel is None:
|
|
3100
4187
|
return None, {"path": repo_map_safe_raw_path_label(rel_path), "reason": reason}
|
|
@@ -3106,14 +4193,24 @@ def read_repo_map_text(root: Path, rel_path: str) -> tuple[dict[str, Any] | None
|
|
|
3106
4193
|
return None, {"path": display, "reason": open_reason, "retrieval_omitted_reason": "redacted_path" if redacted_path else None}
|
|
3107
4194
|
try:
|
|
3108
4195
|
with handle:
|
|
4196
|
+
before_identity = _open_source_identity(handle)
|
|
3109
4197
|
text = handle.read(MAX_REPO_MAP_BYTES_PER_FILE + 1)
|
|
4198
|
+
after_identity = _open_source_identity(handle)
|
|
3110
4199
|
except (OSError, UnicodeError):
|
|
3111
4200
|
return None, {"path": display, "reason": "unsafe_path", "retrieval_omitted_reason": "redacted_path" if redacted_path else None}
|
|
4201
|
+
if before_identity is None or after_identity != before_identity:
|
|
4202
|
+
return None, {
|
|
4203
|
+
"path": display,
|
|
4204
|
+
"reason": "source_changed_during_repo_map",
|
|
4205
|
+
"retrieval_omitted_reason": "redacted_path" if redacted_path else None,
|
|
4206
|
+
}
|
|
3112
4207
|
capped = byte_len(text) > MAX_REPO_MAP_BYTES_PER_FILE
|
|
3113
4208
|
if capped:
|
|
3114
4209
|
text = text.encode("utf-8", errors="replace")[:MAX_REPO_MAP_BYTES_PER_FILE].decode("utf-8", errors="ignore")
|
|
3115
4210
|
risk_counts = secret_risk_counts(text)
|
|
3116
4211
|
sanitized_text, redacted_lines = sanitize_text(text)
|
|
4212
|
+
if source_identities_out is not None:
|
|
4213
|
+
source_identities_out[rel.as_posix()] = before_identity
|
|
3117
4214
|
return {
|
|
3118
4215
|
"path": display,
|
|
3119
4216
|
"raw_path": rel.as_posix(),
|
|
@@ -3152,7 +4249,13 @@ def repo_map_scan_paths(paths: list[str], *, seed_paths: set[str], query_terms:
|
|
|
3152
4249
|
return [path for _index, path in ranked[:MAX_REPO_MAP_SCAN_FILES]]
|
|
3153
4250
|
|
|
3154
4251
|
|
|
3155
|
-
def repo_map_records(
|
|
4252
|
+
def repo_map_records(
|
|
4253
|
+
root: Path,
|
|
4254
|
+
*,
|
|
4255
|
+
seed_paths: set[str],
|
|
4256
|
+
query_terms: set[str],
|
|
4257
|
+
source_identities_out: dict[str, tuple[int, int, int, int, int, int, int, int]] | None = None,
|
|
4258
|
+
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], dict[str, Any]]:
|
|
3156
4259
|
paths = git_ls_files(root)
|
|
3157
4260
|
candidate_paths = paths[:MAX_REPO_MAP_FILES]
|
|
3158
4261
|
path_cap_reached = len(paths) > MAX_REPO_MAP_FILES
|
|
@@ -3161,7 +4264,14 @@ def repo_map_records(root: Path, *, seed_paths: set[str], query_terms: set[str])
|
|
|
3161
4264
|
records: list[dict[str, Any]] = []
|
|
3162
4265
|
omitted: list[dict[str, Any]] = []
|
|
3163
4266
|
for rel_path in scan_paths:
|
|
3164
|
-
|
|
4267
|
+
if source_identities_out is None:
|
|
4268
|
+
record, omission_item = read_repo_map_text(root, rel_path)
|
|
4269
|
+
else:
|
|
4270
|
+
record, omission_item = read_repo_map_text(
|
|
4271
|
+
root,
|
|
4272
|
+
rel_path,
|
|
4273
|
+
source_identities_out=source_identities_out,
|
|
4274
|
+
)
|
|
3165
4275
|
if record is not None:
|
|
3166
4276
|
records.append(record)
|
|
3167
4277
|
elif omission_item is not None and omission_item.get("reason") != "unsupported_file_type":
|
|
@@ -3423,9 +4533,18 @@ def build_graph_rank(
|
|
|
3423
4533
|
query_terms: set[str],
|
|
3424
4534
|
seed_paths: set[str],
|
|
3425
4535
|
secret_scan: dict[str, Any],
|
|
4536
|
+
complete_secret_paths: set[str] | None = None,
|
|
3426
4537
|
) -> list[dict[str, Any]]:
|
|
3427
4538
|
signature_paths = {str(item.get("path", "")) for item in signatures}
|
|
3428
|
-
secret_paths =
|
|
4539
|
+
secret_paths = (
|
|
4540
|
+
complete_secret_paths
|
|
4541
|
+
if complete_secret_paths is not None
|
|
4542
|
+
else {
|
|
4543
|
+
str(item.get("path", ""))
|
|
4544
|
+
for item in secret_scan.get("files_with_risks", [])
|
|
4545
|
+
if isinstance(item, dict)
|
|
4546
|
+
}
|
|
4547
|
+
)
|
|
3429
4548
|
degree: dict[str, int] = {}
|
|
3430
4549
|
for edge in edges:
|
|
3431
4550
|
degree[edge["from"]] = degree.get(edge["from"], 0) + 1
|
|
@@ -3522,13 +4641,27 @@ def build_repo_map_payload(
|
|
|
3522
4641
|
build_payload: dict[str, Any],
|
|
3523
4642
|
*,
|
|
3524
4643
|
root_arg: str,
|
|
4644
|
+
complete_secret_paths_out: set[str] | None = None,
|
|
4645
|
+
source_identities_out: dict[str, tuple[int, int, int, int, int, int, int, int]] | None = None,
|
|
3525
4646
|
) -> dict[str, Any]:
|
|
3526
4647
|
query_terms = suggest_tokens(str(suggest_payload.get("query", "")))
|
|
3527
4648
|
seed_paths = repo_map_seed_paths(args, suggest_payload, build_payload)
|
|
3528
|
-
records, omitted, caps = repo_map_records(
|
|
4649
|
+
records, omitted, caps = repo_map_records(
|
|
4650
|
+
root,
|
|
4651
|
+
seed_paths=seed_paths,
|
|
4652
|
+
query_terms=query_terms,
|
|
4653
|
+
source_identities_out=source_identities_out,
|
|
4654
|
+
)
|
|
3529
4655
|
record_by_path = {str(record["path"]): record for record in records}
|
|
3530
4656
|
signatures = extract_signatures(records)
|
|
3531
4657
|
secret_scan = build_secret_scan(records)
|
|
4658
|
+
complete_secret_paths = {
|
|
4659
|
+
str(record.get("path", ""))
|
|
4660
|
+
for record in records
|
|
4661
|
+
if record.get("secret_risk_counts")
|
|
4662
|
+
}
|
|
4663
|
+
if complete_secret_paths_out is not None:
|
|
4664
|
+
complete_secret_paths_out.update(complete_secret_paths)
|
|
3532
4665
|
edges = collect_import_edges(records)
|
|
3533
4666
|
graph_rank = build_graph_rank(
|
|
3534
4667
|
records,
|
|
@@ -3537,6 +4670,7 @@ def build_repo_map_payload(
|
|
|
3537
4670
|
query_terms=query_terms,
|
|
3538
4671
|
seed_paths=seed_paths,
|
|
3539
4672
|
secret_scan=secret_scan,
|
|
4673
|
+
complete_secret_paths=complete_secret_paths,
|
|
3540
4674
|
)
|
|
3541
4675
|
retrieval = repo_map_retrieval(record_by_path, signatures, graph_rank, root_arg=root_arg)
|
|
3542
4676
|
tree = build_token_tree(records)
|
|
@@ -3586,7 +4720,153 @@ def line_identity_from_dict(value: object) -> str:
|
|
|
3586
4720
|
return f"{value.get('start')}:{value.get('end')}"
|
|
3587
4721
|
|
|
3588
4722
|
|
|
3589
|
-
def
|
|
4723
|
+
def apply_symbol_memory_graph(
|
|
4724
|
+
manifest: dict[str, Any],
|
|
4725
|
+
repo_map: dict[str, Any],
|
|
4726
|
+
*,
|
|
4727
|
+
complete_secret_paths: set[str] | None = None,
|
|
4728
|
+
) -> tuple[dict[str, Any], dict[str, Any]]:
|
|
4729
|
+
"""Add bounded direct graph neighbors to an explicit auto-pack manifest."""
|
|
4730
|
+
|
|
4731
|
+
raw_sources = manifest.get("sources")
|
|
4732
|
+
if not isinstance(raw_sources, list):
|
|
4733
|
+
raise PackError("manifest sources must be a list")
|
|
4734
|
+
existing_sources = [
|
|
4735
|
+
copy.deepcopy(item) for item in raw_sources if isinstance(item, dict)
|
|
4736
|
+
]
|
|
4737
|
+
existing_paths = {
|
|
4738
|
+
str(item.get("path", "")) for item in existing_sources if item.get("path")
|
|
4739
|
+
}
|
|
4740
|
+
graph = repo_map.get("graph") if isinstance(repo_map.get("graph"), dict) else {}
|
|
4741
|
+
edges = graph.get("edges") if isinstance(graph.get("edges"), list) else []
|
|
4742
|
+
rank_items = (
|
|
4743
|
+
repo_map.get("graph_rank")
|
|
4744
|
+
if isinstance(repo_map.get("graph_rank"), list)
|
|
4745
|
+
else []
|
|
4746
|
+
)
|
|
4747
|
+
rank_by_path = {
|
|
4748
|
+
str(item.get("path", "")): item
|
|
4749
|
+
for item in rank_items
|
|
4750
|
+
if isinstance(item, dict) and item.get("path")
|
|
4751
|
+
}
|
|
4752
|
+
secret_scan = (
|
|
4753
|
+
repo_map.get("secret_scan")
|
|
4754
|
+
if isinstance(repo_map.get("secret_scan"), dict)
|
|
4755
|
+
else {}
|
|
4756
|
+
)
|
|
4757
|
+
risky_paths = (
|
|
4758
|
+
complete_secret_paths
|
|
4759
|
+
if complete_secret_paths is not None
|
|
4760
|
+
else {
|
|
4761
|
+
str(item.get("path", ""))
|
|
4762
|
+
for item in secret_scan.get("files_with_risks", [])
|
|
4763
|
+
if isinstance(item, dict) and item.get("path")
|
|
4764
|
+
}
|
|
4765
|
+
)
|
|
4766
|
+
direct_neighbors: set[str] = set()
|
|
4767
|
+
for edge in edges:
|
|
4768
|
+
if not isinstance(edge, dict):
|
|
4769
|
+
continue
|
|
4770
|
+
source = edge.get("from")
|
|
4771
|
+
target = edge.get("to")
|
|
4772
|
+
if not isinstance(source, str) or not isinstance(target, str):
|
|
4773
|
+
continue
|
|
4774
|
+
if source in existing_paths and target not in existing_paths:
|
|
4775
|
+
direct_neighbors.add(target)
|
|
4776
|
+
if target in existing_paths and source not in existing_paths:
|
|
4777
|
+
direct_neighbors.add(source)
|
|
4778
|
+
|
|
4779
|
+
eligible: list[tuple[int, str, int]] = []
|
|
4780
|
+
excluded_secret_risk_count = 0
|
|
4781
|
+
for path in direct_neighbors:
|
|
4782
|
+
if path in risky_paths:
|
|
4783
|
+
excluded_secret_risk_count += 1
|
|
4784
|
+
continue
|
|
4785
|
+
item = rank_by_path.get(path)
|
|
4786
|
+
if item is None or repo_map_path_has_sensitive_evidence(path):
|
|
4787
|
+
continue
|
|
4788
|
+
score = int(item.get("score", 0) or 0)
|
|
4789
|
+
line_count = int(item.get("line_count", 0) or 0)
|
|
4790
|
+
if score <= 0 or line_count <= 0:
|
|
4791
|
+
continue
|
|
4792
|
+
eligible.append((score, path, line_count))
|
|
4793
|
+
eligible.sort(key=lambda item: (-item[0], item[1]))
|
|
4794
|
+
|
|
4795
|
+
seed_priorities = [
|
|
4796
|
+
int(item.get("priority", 0) or 0) for item in existing_sources
|
|
4797
|
+
]
|
|
4798
|
+
maximum_graph_priority = max(1, min(seed_priorities, default=2) - 1)
|
|
4799
|
+
selected_sources: list[dict[str, Any]] = []
|
|
4800
|
+
for score, path, line_count in eligible[:MAX_GRAPH_APPLICATION_SOURCES]:
|
|
4801
|
+
source = {
|
|
4802
|
+
"path": path,
|
|
4803
|
+
"priority": max(1, min(score, maximum_graph_priority)),
|
|
4804
|
+
"label": f"graph:{path}"[:MAX_LABEL_CHARS],
|
|
4805
|
+
"lines": {"start": 1, "end": min(line_count, MAX_GRAPH_APPLICATION_LINES)},
|
|
4806
|
+
}
|
|
4807
|
+
existing_sources.append(source)
|
|
4808
|
+
selected_sources.append(
|
|
4809
|
+
{
|
|
4810
|
+
"path": source["path"],
|
|
4811
|
+
"priority": source["priority"],
|
|
4812
|
+
"lines": copy.deepcopy(source["lines"]),
|
|
4813
|
+
"reason": "direct_import_neighbor",
|
|
4814
|
+
}
|
|
4815
|
+
)
|
|
4816
|
+
result_manifest = build_suggest_manifest(existing_sources)
|
|
4817
|
+
return result_manifest, {
|
|
4818
|
+
"schema_version": GRAPH_APPLICATION_SCHEMA_VERSION,
|
|
4819
|
+
"mode": "explicit_opt_in",
|
|
4820
|
+
"selected_source_count": len(selected_sources),
|
|
4821
|
+
"selected_sources": selected_sources,
|
|
4822
|
+
"candidate_count": len(eligible),
|
|
4823
|
+
"candidate_cap": MAX_GRAPH_APPLICATION_SOURCES,
|
|
4824
|
+
"candidate_cap_reached": len(eligible) > MAX_GRAPH_APPLICATION_SOURCES,
|
|
4825
|
+
"excluded_secret_risk_count": excluded_secret_risk_count,
|
|
4826
|
+
"exact_source_fallback_retained": True,
|
|
4827
|
+
"deterministic_local_only": True,
|
|
4828
|
+
"provider_token_or_cost_savings_claim_allowed": False,
|
|
4829
|
+
}
|
|
4830
|
+
|
|
4831
|
+
|
|
4832
|
+
def bind_graph_sources_to_repo_snapshot(
|
|
4833
|
+
root: Path,
|
|
4834
|
+
specs: list[SourceSpec],
|
|
4835
|
+
required_sources: set[tuple[str, str]],
|
|
4836
|
+
source_identities: dict[str, tuple[int, int, int, int, int, int, int, int]],
|
|
4837
|
+
*,
|
|
4838
|
+
source_cache: _SourceSnapshotCache,
|
|
4839
|
+
input_budget: _SourceInputBudget,
|
|
4840
|
+
) -> dict[tuple[str, str], dict[str, Any]]:
|
|
4841
|
+
"""Warm immutable source snapshots only for repo-map-bound graph additions."""
|
|
4842
|
+
|
|
4843
|
+
rejections: dict[tuple[str, str], dict[str, Any]] = {}
|
|
4844
|
+
for spec in specs:
|
|
4845
|
+
rel, _reason = lexical_rel(spec.path)
|
|
4846
|
+
if rel is None:
|
|
4847
|
+
continue
|
|
4848
|
+
lines_identity = spec.lines.identity() if spec.lines is not None else "all"
|
|
4849
|
+
source_identity = (rel.as_posix(), lines_identity)
|
|
4850
|
+
if source_identity not in required_sources:
|
|
4851
|
+
continue
|
|
4852
|
+
expected_identity = source_identities.get(rel.as_posix())
|
|
4853
|
+
if expected_identity is None:
|
|
4854
|
+
continue
|
|
4855
|
+
_source, omitted_item = resolve_source(
|
|
4856
|
+
root,
|
|
4857
|
+
spec,
|
|
4858
|
+
source_cache=source_cache,
|
|
4859
|
+
input_budget=input_budget,
|
|
4860
|
+
expected_identity=expected_identity,
|
|
4861
|
+
)
|
|
4862
|
+
if omitted_item is not None:
|
|
4863
|
+
rejections[source_identity] = copy.deepcopy(omitted_item)
|
|
4864
|
+
return rejections
|
|
4865
|
+
|
|
4866
|
+
|
|
4867
|
+
def build_symbol_memory_payload(
|
|
4868
|
+
repo_map: dict[str, Any], *, applied: bool = False
|
|
4869
|
+
) -> dict[str, Any]:
|
|
3590
4870
|
retrieval_by_path_lines: dict[tuple[str, str], dict[str, Any]] = {}
|
|
3591
4871
|
for item in repo_map.get("retrieval", []):
|
|
3592
4872
|
if not isinstance(item, dict):
|
|
@@ -3637,7 +4917,7 @@ def build_symbol_memory_payload(repo_map: dict[str, Any]) -> dict[str, Any]:
|
|
|
3637
4917
|
retrieval = repo_map.get("retrieval", []) if isinstance(repo_map.get("retrieval"), list) else []
|
|
3638
4918
|
return {
|
|
3639
4919
|
"schema_version": SYMBOL_MEMORY_SCHEMA_VERSION,
|
|
3640
|
-
"mode": "advisory",
|
|
4920
|
+
"mode": "applied" if applied else "advisory",
|
|
3641
4921
|
"source": "contextguard.pack-repo-map.v1",
|
|
3642
4922
|
"summary": {
|
|
3643
4923
|
"symbols": len(symbols),
|
|
@@ -3657,8 +4937,9 @@ def build_symbol_memory_payload(repo_map: dict[str, Any]) -> dict[str, Any]:
|
|
|
3657
4937
|
"claim_boundary": {
|
|
3658
4938
|
"deterministic_local_only": True,
|
|
3659
4939
|
"no_network_model_embedding_lsp_or_tree_sitter_dependency": True,
|
|
3660
|
-
"advisory_does_not_change_manifest_pack_or_receipt":
|
|
3661
|
-
"
|
|
4940
|
+
"advisory_does_not_change_manifest_pack_or_receipt": not applied,
|
|
4941
|
+
"explicit_graph_application_changes_manifest_and_pack": applied,
|
|
4942
|
+
"graph_rank_is_explain_only": not applied,
|
|
3662
4943
|
"provider_token_or_cost_savings_claim_allowed": False,
|
|
3663
4944
|
},
|
|
3664
4945
|
}
|
|
@@ -3804,10 +5085,18 @@ def build_auto_explain_payload(
|
|
|
3804
5085
|
explain["repo_map"] = copy.deepcopy(repo_map_payload)
|
|
3805
5086
|
elif root is not None:
|
|
3806
5087
|
explain["repo_map"] = build_repo_map_payload(root, args, suggest_payload, build_payload, root_arg=root_arg)
|
|
5088
|
+
if isinstance(payload.get("graph_application"), dict):
|
|
5089
|
+
explain["graph_application"] = copy.deepcopy(payload["graph_application"])
|
|
5090
|
+
if isinstance(payload.get("adaptive_k_application"), dict):
|
|
5091
|
+
explain["adaptive_k_application"] = copy.deepcopy(
|
|
5092
|
+
payload["adaptive_k_application"]
|
|
5093
|
+
)
|
|
3807
5094
|
return explain
|
|
3808
5095
|
|
|
3809
5096
|
|
|
3810
5097
|
def auto_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tuple[dict[str, Any], int]:
|
|
5098
|
+
source_cache = _SourceSnapshotCache()
|
|
5099
|
+
input_budget = _SourceInputBudget()
|
|
3811
5100
|
manifest_rel = output_rel_for_collision_check(args.manifest_out, "--manifest-out") if args.manifest_out else None
|
|
3812
5101
|
pack_rel = output_rel_for_collision_check(args.pack_out, "--pack-out") if args.pack_out else None
|
|
3813
5102
|
if manifest_rel is not None and pack_rel is not None:
|
|
@@ -3824,8 +5113,39 @@ def auto_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tuple[d
|
|
|
3824
5113
|
validate_output_path_under_root(root, args.pack_out, "--pack-out")
|
|
3825
5114
|
suggest_args = copy.copy(args)
|
|
3826
5115
|
suggest_args.manifest_out = None
|
|
3827
|
-
|
|
5116
|
+
apply_adaptive_k = bool(getattr(args, "apply_adaptive_k", False))
|
|
5117
|
+
if apply_adaptive_k:
|
|
5118
|
+
suggest_args.adaptive_k = True
|
|
5119
|
+
suggest_payload, rc = suggest_pack(
|
|
5120
|
+
root,
|
|
5121
|
+
suggest_args,
|
|
5122
|
+
root_arg=root_arg,
|
|
5123
|
+
_source_cache=source_cache,
|
|
5124
|
+
_input_budget=input_budget,
|
|
5125
|
+
)
|
|
3828
5126
|
manifest = suggest_payload["manifest"]
|
|
5127
|
+
adaptive_k_application: dict[str, Any] | None = None
|
|
5128
|
+
if apply_adaptive_k and isinstance(suggest_payload.get("adaptive_k"), dict):
|
|
5129
|
+
manifest, adaptive_k_application = apply_adaptive_k_manifest(
|
|
5130
|
+
manifest,
|
|
5131
|
+
suggest_payload["adaptive_k"],
|
|
5132
|
+
)
|
|
5133
|
+
suggest_payload["manifest"] = manifest
|
|
5134
|
+
retained_identities = {
|
|
5135
|
+
(str(item.get("path", "")), line_range_identity(item.get("lines")))
|
|
5136
|
+
for item in manifest.get("sources", [])
|
|
5137
|
+
if isinstance(item, dict)
|
|
5138
|
+
}
|
|
5139
|
+
suggest_payload["sources"] = [
|
|
5140
|
+
item
|
|
5141
|
+
for item in suggest_payload.get("sources", [])
|
|
5142
|
+
if isinstance(item, dict)
|
|
5143
|
+
and (
|
|
5144
|
+
str(item.get("path", "")),
|
|
5145
|
+
line_range_identity(item.get("lines")),
|
|
5146
|
+
)
|
|
5147
|
+
in retained_identities
|
|
5148
|
+
]
|
|
3829
5149
|
specs = manifest_to_source_specs(manifest)
|
|
3830
5150
|
budget = bounded_int(args.budget_bytes, DEFAULT_BUDGET_BYTES, MIN_BUDGET_BYTES, MAX_BUDGET_BYTES)
|
|
3831
5151
|
build_payload = build_pack(
|
|
@@ -3836,7 +5156,83 @@ def auto_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tuple[d
|
|
|
3836
5156
|
store_artifact=False,
|
|
3837
5157
|
delta_from_pack_id=args.delta_from_pack_id,
|
|
3838
5158
|
sketch_duplicate_veto=getattr(args, "sketch_duplicate_veto", False),
|
|
5159
|
+
_source_cache=source_cache,
|
|
5160
|
+
_input_budget=input_budget,
|
|
3839
5161
|
)
|
|
5162
|
+
if apply_adaptive_k:
|
|
5163
|
+
suggest_payload["estimated_pack_bytes"] = build_payload.get("pack_bytes", 0)
|
|
5164
|
+
suggest_payload["token_proxy"] = copy.deepcopy(
|
|
5165
|
+
build_payload.get("token_proxy", {})
|
|
5166
|
+
)
|
|
5167
|
+
repo_map_payload: dict[str, Any] | None = None
|
|
5168
|
+
graph_application: dict[str, Any] | None = None
|
|
5169
|
+
apply_symbol_memory = bool(getattr(args, "apply_symbol_memory", False))
|
|
5170
|
+
complete_secret_paths: set[str] | None = set() if apply_symbol_memory else None
|
|
5171
|
+
repo_map_source_identities: dict[
|
|
5172
|
+
str,
|
|
5173
|
+
tuple[int, int, int, int, int, int, int, int],
|
|
5174
|
+
] = {}
|
|
5175
|
+
if getattr(args, "symbol_memory", False) or apply_symbol_memory or args.explain:
|
|
5176
|
+
repo_map_payload = build_repo_map_payload(
|
|
5177
|
+
root,
|
|
5178
|
+
args,
|
|
5179
|
+
suggest_payload,
|
|
5180
|
+
build_payload,
|
|
5181
|
+
root_arg=root_arg,
|
|
5182
|
+
complete_secret_paths_out=complete_secret_paths,
|
|
5183
|
+
source_identities_out=(
|
|
5184
|
+
repo_map_source_identities if apply_symbol_memory else None
|
|
5185
|
+
),
|
|
5186
|
+
)
|
|
5187
|
+
if apply_symbol_memory and isinstance(repo_map_payload, dict):
|
|
5188
|
+
repo_map_payload["safety"]["explain_only"] = False
|
|
5189
|
+
repo_map_payload["safety"]["caveats"] = [
|
|
5190
|
+
"Repo-map bytes are local sampled UTF-8 bytes and estimated chars_div_4 token proxies, not provider-token or savings claims.",
|
|
5191
|
+
"Graph ranking is applied only to the bounded direct-neighbor expansion recorded in graph_application; exact source retrieval remains available.",
|
|
5192
|
+
]
|
|
5193
|
+
pre_graph_sources = {
|
|
5194
|
+
(str(item.get("path", "")), line_range_identity(item.get("lines")))
|
|
5195
|
+
for item in manifest.get("sources", [])
|
|
5196
|
+
if isinstance(item, dict) and item.get("path")
|
|
5197
|
+
}
|
|
5198
|
+
manifest, graph_application = apply_symbol_memory_graph(
|
|
5199
|
+
manifest,
|
|
5200
|
+
repo_map_payload,
|
|
5201
|
+
complete_secret_paths=complete_secret_paths,
|
|
5202
|
+
)
|
|
5203
|
+
suggest_payload["manifest"] = manifest
|
|
5204
|
+
specs = manifest_to_source_specs(manifest)
|
|
5205
|
+
graph_snapshot_sources = {
|
|
5206
|
+
(str(item.get("path", "")), line_range_identity(item.get("lines")))
|
|
5207
|
+
for item in manifest.get("sources", [])
|
|
5208
|
+
if isinstance(item, dict) and item.get("path")
|
|
5209
|
+
} - pre_graph_sources
|
|
5210
|
+
graph_snapshot_rejections = bind_graph_sources_to_repo_snapshot(
|
|
5211
|
+
root,
|
|
5212
|
+
specs,
|
|
5213
|
+
graph_snapshot_sources,
|
|
5214
|
+
repo_map_source_identities,
|
|
5215
|
+
source_cache=source_cache,
|
|
5216
|
+
input_budget=input_budget,
|
|
5217
|
+
)
|
|
5218
|
+
build_payload = build_pack(
|
|
5219
|
+
root,
|
|
5220
|
+
specs,
|
|
5221
|
+
budget_bytes=budget,
|
|
5222
|
+
root_arg=root_arg,
|
|
5223
|
+
store_artifact=False,
|
|
5224
|
+
delta_from_pack_id=args.delta_from_pack_id,
|
|
5225
|
+
sketch_duplicate_veto=getattr(args, "sketch_duplicate_veto", False),
|
|
5226
|
+
_source_cache=source_cache,
|
|
5227
|
+
_input_budget=input_budget,
|
|
5228
|
+
_required_snapshot_sources=graph_snapshot_sources,
|
|
5229
|
+
_expected_source_identities=repo_map_source_identities,
|
|
5230
|
+
_snapshot_rejections=graph_snapshot_rejections,
|
|
5231
|
+
)
|
|
5232
|
+
suggest_payload["estimated_pack_bytes"] = build_payload.get("pack_bytes", 0)
|
|
5233
|
+
suggest_payload["token_proxy"] = copy.deepcopy(
|
|
5234
|
+
build_payload.get("token_proxy", {})
|
|
5235
|
+
)
|
|
3840
5236
|
if not args.no_artifact:
|
|
3841
5237
|
receipt_rel = Path(PACK_DIR) / f"{build_payload['pack_id']}.json"
|
|
3842
5238
|
if manifest_rel is not None:
|
|
@@ -3883,7 +5279,7 @@ def auto_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tuple[d
|
|
|
3883
5279
|
"suggest": suggest_payload,
|
|
3884
5280
|
"build": build_payload,
|
|
3885
5281
|
"sources": {
|
|
3886
|
-
"suggested": len(
|
|
5282
|
+
"suggested": len(manifest.get("sources", [])),
|
|
3887
5283
|
"included": build_payload.get("sources", {}).get("included", 0),
|
|
3888
5284
|
"partial": build_payload.get("sources", {}).get("partial", 0),
|
|
3889
5285
|
"omitted": build_payload.get("sources", {}).get("omitted", 0),
|
|
@@ -3897,13 +5293,18 @@ def auto_pack(root: Path, args: argparse.Namespace, *, root_arg: str) -> tuple[d
|
|
|
3897
5293
|
}
|
|
3898
5294
|
if build_hint_omitted_reason:
|
|
3899
5295
|
payload["build_hint_omitted_reason"] = build_hint_omitted_reason
|
|
3900
|
-
if getattr(args, "adaptive_k", False) and isinstance(
|
|
5296
|
+
if (getattr(args, "adaptive_k", False) or apply_adaptive_k) and isinstance(
|
|
5297
|
+
suggest_payload.get("adaptive_k"), dict
|
|
5298
|
+
):
|
|
3901
5299
|
payload["adaptive_k"] = copy.deepcopy(suggest_payload["adaptive_k"])
|
|
3902
|
-
|
|
3903
|
-
|
|
3904
|
-
|
|
3905
|
-
|
|
3906
|
-
|
|
5300
|
+
if adaptive_k_application is not None:
|
|
5301
|
+
payload["adaptive_k_application"] = adaptive_k_application
|
|
5302
|
+
if graph_application is not None:
|
|
5303
|
+
payload["graph_application"] = graph_application
|
|
5304
|
+
if (getattr(args, "symbol_memory", False) or apply_symbol_memory) and isinstance(repo_map_payload, dict):
|
|
5305
|
+
payload["symbol_memory"] = build_symbol_memory_payload(
|
|
5306
|
+
repo_map_payload, applied=apply_symbol_memory
|
|
5307
|
+
)
|
|
3907
5308
|
if args.explain:
|
|
3908
5309
|
payload["explain"] = build_auto_explain_payload(
|
|
3909
5310
|
args,
|
|
@@ -3939,6 +5340,8 @@ def print_adaptive_k_text(payload: dict[str, Any]) -> None:
|
|
|
3939
5340
|
reason_text = ",".join(str(item) for item in reason_codes[:5])
|
|
3940
5341
|
else:
|
|
3941
5342
|
reason_text = str(reason_codes)
|
|
5343
|
+
application = payload.get("adaptive_k_application")
|
|
5344
|
+
applied = isinstance(application, dict) and application.get("status") == "applied"
|
|
3942
5345
|
print(
|
|
3943
5346
|
"adaptive-k: "
|
|
3944
5347
|
f"recommended={adaptive.get('recommended_k', 0)}/{adaptive.get('requested_top', 0)} "
|
|
@@ -3946,7 +5349,7 @@ def print_adaptive_k_text(payload: dict[str, Any]) -> None:
|
|
|
3946
5349
|
f"gates={regression_gates.get('status', 'pass')} "
|
|
3947
5350
|
f"candidates={score_distribution.get('candidate_count', 0)} "
|
|
3948
5351
|
f"budget_limited={budget_fit.get('budget_limited', False)} "
|
|
3949
|
-
f"apply=
|
|
5352
|
+
f"apply={str(applied).lower()} reasons={reason_text or 'none'}"
|
|
3950
5353
|
)
|
|
3951
5354
|
|
|
3952
5355
|
|
|
@@ -4125,10 +5528,27 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
4125
5528
|
)
|
|
4126
5529
|
auto.add_argument("--explain", action="store_true", help="include deterministic local selection/build explanation metadata")
|
|
4127
5530
|
auto.add_argument("--adaptive-k", action="store_true", help="include local score/budget top-k advisory metadata without changing the manifest or pack")
|
|
5531
|
+
auto.add_argument(
|
|
5532
|
+
"--apply-adaptive-k",
|
|
5533
|
+
action="store_true",
|
|
5534
|
+
help=(
|
|
5535
|
+
"explicitly prune heuristic-selected sources to the locally recommended top-k "
|
|
5536
|
+
"after regression gates pass while always retaining explicit file/output/diff sources; "
|
|
5537
|
+
"implies --adaptive-k"
|
|
5538
|
+
),
|
|
5539
|
+
)
|
|
4128
5540
|
auto.add_argument("--adaptive-k-policy", choices=ADAPTIVE_K_POLICIES, default="balanced", help="local adaptive-k recommendation policy used when --adaptive-k is set")
|
|
4129
5541
|
auto.add_argument("--adaptive-k-min-recall-proxy", type=adaptive_k_threshold, default=0.0, help="metadata-only minimum recall proxy gate for --adaptive-k")
|
|
4130
5542
|
auto.add_argument("--adaptive-k-min-precision-proxy", type=adaptive_k_threshold, default=0.0, help="metadata-only minimum precision proxy gate for --adaptive-k")
|
|
4131
5543
|
auto.add_argument("--symbol-memory", action="store_true", help="include repo-map derived symbol/graph advisory metadata with exact source verification hints")
|
|
5544
|
+
auto.add_argument(
|
|
5545
|
+
"--apply-symbol-memory",
|
|
5546
|
+
action="store_true",
|
|
5547
|
+
help=(
|
|
5548
|
+
"explicitly add up to four direct import-neighbor slices from the local "
|
|
5549
|
+
"repo map to the manifest and pack; implies --symbol-memory"
|
|
5550
|
+
),
|
|
5551
|
+
)
|
|
4132
5552
|
return parser
|
|
4133
5553
|
|
|
4134
5554
|
|