@matt82198/aesop 0.1.0-beta.4 → 0.1.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. package/CHANGELOG.md +187 -5
  2. package/LICENSE +66 -21
  3. package/README.md +143 -23
  4. package/aesop.config.example.json +14 -1
  5. package/bin/CLAUDE.md +42 -5
  6. package/bin/cli.js +479 -95
  7. package/daemons/CLAUDE.md +5 -2
  8. package/daemons/backup-fleet.sh +209 -51
  9. package/daemons/run-watchdog.sh +211 -62
  10. package/dash/dash-extra.mjs +73 -4
  11. package/dash/watchdog-gui.sh +41 -35
  12. package/docs/ARCHITECTURE.md +296 -0
  13. package/docs/CONCEPTS.md +254 -0
  14. package/docs/CONFIGURE.md +256 -0
  15. package/docs/FIRST-WAVE.md +276 -0
  16. package/docs/INSTALL.md +224 -0
  17. package/docs/README.md +176 -27
  18. package/docs/RELEASING.md +159 -0
  19. package/docs/archive/README.md +3 -0
  20. package/docs/{spikes → archive/spikes}/tiered-cognition/README.md +1 -3
  21. package/docs/autonomous-swe.md +149 -0
  22. package/docs/case-study-portfolio.md +61 -0
  23. package/docs/reproduce.md +121 -0
  24. package/docs/self-stats-data.json +11 -0
  25. package/docs/templates/FLEET-OPS-ANALYSIS.example.md +27 -0
  26. package/docs/templates/FLEET-OPS-RECOMMENDATIONS.example.md +24 -0
  27. package/docs/templates/PROPOSALS-LOG.example.md +64 -0
  28. package/hooks/CLAUDE.md +51 -0
  29. package/hooks/install-waveguard.sh +68 -0
  30. package/hooks/pre-commit-waveguard.sh +26 -0
  31. package/hooks/pre-push-policy.sh +253 -15
  32. package/mcp/CLAUDE.md +213 -0
  33. package/mcp/package.json +26 -0
  34. package/mcp/server.mjs +543 -0
  35. package/monitor/CHARTER.md +76 -110
  36. package/monitor/CLAUDE.md +3 -3
  37. package/monitor/collect-signals.mjs +218 -20
  38. package/package.json +17 -6
  39. package/scan/CLAUDE.md +30 -0
  40. package/scan/fleet-scan.example.mjs +292 -0
  41. package/skills/CLAUDE.md +1 -0
  42. package/skills/healthcheck/SKILL.md +44 -0
  43. package/state_store/CLAUDE.md +39 -0
  44. package/state_store/__init__.py +24 -0
  45. package/state_store/api.py +35 -0
  46. package/state_store/export.py +19 -0
  47. package/state_store/ingest.py +41 -0
  48. package/state_store/projections.py +126 -0
  49. package/state_store/store.py +197 -0
  50. package/tools/CLAUDE.md +37 -149
  51. package/tools/alert_bridge.py +453 -0
  52. package/tools/bench_runner.py +438 -0
  53. package/tools/buildlog.py +4 -7
  54. package/tools/ci_merge_wait.py +308 -14
  55. package/tools/common.py +62 -0
  56. package/tools/cost_ceiling.py +191 -0
  57. package/tools/dash.js +102 -0
  58. package/tools/doctor.js +220 -0
  59. package/tools/fleet_ledger.py +189 -17
  60. package/tools/fleet_prompt_extractor.py +134 -0
  61. package/tools/halt.py +172 -0
  62. package/tools/healthcheck.py +299 -0
  63. package/tools/heartbeat.py +4 -7
  64. package/tools/metrics_gate.py +8 -2
  65. package/tools/orchestrator_status.py +4 -9
  66. package/tools/reconcile.py +277 -0
  67. package/tools/rotate_logs.py +152 -62
  68. package/tools/scanner_selftest.py +28 -3
  69. package/tools/secret_scan.py +367 -91
  70. package/tools/self_stats.py +797 -0
  71. package/tools/session_usage_summary.py +198 -0
  72. package/tools/status.js +187 -0
  73. package/tools/svg_to_png.mjs +50 -0
  74. package/tools/transcript_replay.py +236 -0
  75. package/tools/transcript_timeline.py +184 -0
  76. package/tools/verify_agent_inspector.py +289 -0
  77. package/tools/verify_dash.py +361 -542
  78. package/tools/verify_prboard.py +344 -0
  79. package/tools/verify_submit_encoding.py +12 -4
  80. package/tools/watch.js +49 -0
  81. package/ui/CLAUDE.md +59 -41
  82. package/ui/agents.py +337 -34
  83. package/ui/api/submit.py +44 -5
  84. package/ui/api/tracker.py +15 -7
  85. package/ui/collectors.py +294 -52
  86. package/ui/config.py +31 -2
  87. package/ui/cost.py +355 -0
  88. package/ui/csrf.py +9 -4
  89. package/ui/handler.py +450 -12
  90. package/ui/render.py +26 -6
  91. package/ui/sse.py +114 -13
  92. package/ui/wave_prs.py +230 -0
  93. package/ui/web/dist/assets/index-0qQYnvMC.js +9 -0
  94. package/ui/web/dist/assets/index-BdIlFieV.css +1 -0
  95. package/ui/web/dist/index.html +14 -0
  96. package/monitor/.signal-state.json +0 -3
  97. package/monitor/ACTIONS.log +0 -1
  98. package/monitor/BRIEF.md +0 -24
  99. package/monitor/SIGNALS.json +0 -54
  100. package/tools/__pycache__/buildlog.cpython-314.pyc +0 -0
  101. package/tools/__pycache__/ci_merge_wait.cpython-314.pyc +0 -0
  102. package/tools/__pycache__/ensure_state.cpython-314.pyc +0 -0
  103. package/tools/__pycache__/eod_sweep.cpython-314.pyc +0 -0
  104. package/tools/__pycache__/fleet_ledger.cpython-314.pyc +0 -0
  105. package/tools/__pycache__/heartbeat.cpython-314.pyc +0 -0
  106. package/tools/__pycache__/inbox_drain.cpython-314.pyc +0 -0
  107. package/tools/__pycache__/launch_tui.cpython-314.pyc +0 -0
  108. package/tools/__pycache__/orchestrator_status.cpython-314.pyc +0 -0
  109. package/tools/__pycache__/power_selftest.cpython-314.pyc +0 -0
  110. package/tools/__pycache__/prepublish_scan.cpython-314.pyc +0 -0
  111. package/tools/__pycache__/rotate_logs.cpython-314.pyc +0 -0
  112. package/tools/__pycache__/scanner_selftest.cpython-314.pyc +0 -0
  113. package/tools/__pycache__/secret_scan.cpython-314.pyc +0 -0
  114. package/tools/__pycache__/stall_check.cpython-314.pyc +0 -0
  115. package/tools/__pycache__/verify_dash.cpython-314.pyc +0 -0
  116. package/tools/__pycache__/verify_submit_encoding.cpython-314.pyc +0 -0
  117. package/ui/__pycache__/agents.cpython-314.pyc +0 -0
  118. package/ui/__pycache__/collectors.cpython-314.pyc +0 -0
  119. package/ui/__pycache__/config.cpython-314.pyc +0 -0
  120. package/ui/__pycache__/csrf.cpython-314.pyc +0 -0
  121. package/ui/__pycache__/handler.cpython-314.pyc +0 -0
  122. package/ui/__pycache__/render.cpython-314.pyc +0 -0
  123. package/ui/__pycache__/serve.cpython-314.pyc +0 -0
  124. package/ui/__pycache__/sse.cpython-314.pyc +0 -0
  125. package/ui/api/__pycache__/__init__.cpython-314.pyc +0 -0
  126. package/ui/api/__pycache__/submit.cpython-314.pyc +0 -0
  127. package/ui/api/__pycache__/tracker.cpython-314.pyc +0 -0
  128. package/ui/templates/dashboard.html +0 -1202
  129. /package/docs/{spikes → archive/spikes}/tiered-cognition/ACTIVATION.md +0 -0
  130. /package/docs/{spikes → archive/spikes}/tiered-cognition/DESIGN.md +0 -0
  131. /package/docs/{spikes → archive/spikes}/tiered-cognition/FINDINGS.md +0 -0
  132. /package/docs/{spikes → archive/spikes}/tiered-cognition/aesop-cognition.example.md +0 -0
  133. /package/docs/{spikes → archive/spikes}/tiered-cognition/force-model-policy.merged.mjs +0 -0
  134. /package/docs/{spikes → archive/spikes}/tiered-cognition/strip-tools-hook.mjs +0 -0
@@ -3,9 +3,10 @@
3
3
  secret_scan.py — Pre-push secret/credential detection gate.
4
4
 
5
5
  Modes:
6
- secret_scan.py --staged [--repo PATH] Scan git staged files (default repo=cwd)
7
- secret_scan.py --history [--repo PATH] Scan all blobs in git history
8
- secret_scan.py PATH [PATH...] Scan files/dirs directly (recurse dirs)
6
+ secret_scan.py --staged [--repo PATH] Scan git staged files (default repo=cwd)
7
+ secret_scan.py --range COMMIT_RANGE [--repo PATH] Scan files changed in range (e.g., main..HEAD or abc123..def456)
8
+ secret_scan.py --history [--repo PATH] Scan all blobs in git history
9
+ secret_scan.py PATH [PATH...] Scan files/dirs directly (recurse dirs)
9
10
 
10
11
  Exit codes: 0=clean, 1=findings, 2=usage error
11
12
  Output: one line per finding or summary (never prints full secrets)
@@ -22,6 +23,19 @@ Self-scan invariant: this file must scan CLEAN with NO pragma. Any pattern liter
22
23
  would match its own regex is runtime-assembled from fragments (see pem_private_key) so
23
24
  the pattern text never appears contiguously in this source.
24
25
 
26
+ Scan contract (wave-25 P2 fix): --staged and --range scan committed GIT OBJECTS, not the
27
+ working-tree copy of changed files. --staged reads each path's STAGED INDEX blob
28
+ (`git show :<path>`); --range reads each changed path's blob at the TIP of the range
29
+ (`git show <tip>:<path>`). This closes two bypasses that existed when these modes read
30
+ $repo/<path> off disk: (1) stage a secret, then edit the on-disk file without
31
+ re-staging -- the dirty blob is still what gets committed/pushed; (2) commit a secret,
32
+ then edit it away in the worktree without a new commit -- the dirty blob is still what
33
+ the pushed commit carries. --history is unaffected (it already walks committed diffs).
34
+ Any git command needed to enumerate the files-to-scan that itself fails (bad ref,
35
+ unresolvable range, git error) is NOT treated as "zero files changed" -- it raises
36
+ GitScanError and the caller fails CLOSED (non-zero exit), so an unresolvable range can
37
+ never silently report CLEAN.
38
+
25
39
  NOTE: Public repo version has NO vault allowlist.
26
40
  """
27
41
 
@@ -50,10 +64,30 @@ PATTERNS = {
50
64
  "slack_token": (r"xox[baprs]-[A-Za-z0-9-]{10,}", 0),
51
65
  "openai_anthropic_key": (r"sk-[A-Za-z0-9_\-]{20,}", 0),
52
66
  "generic_secret_assignment": (
53
- r"(password|passwd|secret|api[_-]?key|token|authorization)\s*[:=]\s*[\"'](?!.*(?:xxx|changeme|your-|<|$\{|example)\b).{8,}[\"']",
67
+ # Unquoted branch requires 8+ contiguous chars from a bare-literal charset
68
+ # (no '.', '(', ')' — real code expressions like `secrets.token_urlsafe(32)`,
69
+ # `self.serve.SESSION_TOKEN`, or `headers.get(...)` all contain one of those)
70
+ # followed by whitespace/EOL, so an assignment from a variable/attribute/call
71
+ # doesn't false-positive as a hardcoded secret the way a bare env-file value
72
+ # (`API_TOKEN=verylongtokenvalue123456789`) legitimately does.
73
+ r"\b(password|passwd|secret|api[_-]?key|token|authorization)\b\s*(?::=|=)\s*(?:[\"'](?!.*(?:xxx|changeme|your-|<|$\{|example)\b).{8,}[\"']|(?!['\"]|xxx|changeme|your-|example)[^\s\$\<\{\n\(\)\.\"']{8,}(?=\s|$))",
54
74
  re.IGNORECASE,
55
75
  ),
56
- "connection_string": (r"://[^:]+:[^@/\s]+@(?!localhost(?:[:/]|$)|127\.0\.0\.1(?:[:/]|$)|example\.com(?:[:/]|$))[^\s]+", 0),
76
+ "connection_string": (
77
+ (r"://"
78
+ r"[^:]+:[^@/\s]+@"
79
+ r"(?!"
80
+ r"localhost(?:[:/]|$)|"
81
+ r"127\.0\.0\.1(?:[:/]|$)|"
82
+ r"127\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(?:[:/]|$)|"
83
+ r"[a-zA-Z0-9.-]*\.local(?:[:/]|$)|"
84
+ r"[a-zA-Z0-9.-]*\.localdomain(?:[:/]|$)|"
85
+ r"(?:[a-zA-Z0-9-]+\.)*example(?:\.[a-zA-Z]{2,})?(?:[:/]|$)|"
86
+ r"(?:[a-zA-Z0-9-]+\.)*test(?:\.[a-zA-Z]{2,})?(?:[:/]|$)"
87
+ r")"
88
+ r"[^\s]+"),
89
+ 0
90
+ ),
57
91
  "env_access": (
58
92
  r"(?i:(?:os\.getenv|os\.environ|System\.getenv|process\.env)\s*[\[\(][\"']?[A-Z_]*(?:password|secret|api[_-]?key|token|auth|key)[A-Z_0-9]*[\"']?[\)\]])",
59
93
  0,
@@ -76,6 +110,40 @@ CREDENTIAL_FILENAMES = [
76
110
  # Placeholders that don't count as secrets
77
111
  PLACEHOLDERS = {"xxx", "changeme", "your-key-here", "example", "test", "demo"}
78
112
 
113
+ # Rules that CAN be softened by pragma (doc-shaped rules only)
114
+ SOFTENED_BY_PRAGMA = {"generic_secret_assignment", "env_access"}
115
+
116
+ # Rules that are ALWAYS fatal, pragma never applies
117
+ FATAL_RULES = {
118
+ "pem_private_key",
119
+ "aws_access_key",
120
+ "github_token",
121
+ "slack_token",
122
+ "openai_anthropic_key",
123
+ "connection_string",
124
+ }
125
+
126
+
127
+ def _classify_finding(rule_name, has_file_pragma):
128
+ """Shared fatal/softened decision, used by every scan_* variant (disk-file,
129
+ blob, large-file-chunked) so the pragma contract can't drift between them.
130
+ FATAL_RULES are always fatal. SOFTENED_BY_PRAGMA rules are fatal unless the
131
+ pragma is present. Everything else (e.g. env_assignment) is always fatal."""
132
+ if rule_name in FATAL_RULES:
133
+ return True
134
+ if rule_name in SOFTENED_BY_PRAGMA:
135
+ return not has_file_pragma
136
+ return True
137
+
138
+
139
+ class GitScanError(Exception):
140
+ """Raised when a git command needed to enumerate files-to-scan fails.
141
+ Callers MUST treat this as fail-CLOSED (block / exit non-zero) -- it is
142
+ NOT equivalent to "the range/index genuinely contains zero changed
143
+ files" (wave-25 P2 fix: get_range_files previously returned [] on ANY
144
+ git-diff error, indistinguishable from a real empty range, so main()
145
+ printed CLEAN and exited 0 on an unresolvable range)."""
146
+
79
147
 
80
148
  def has_pragma(filepath):
81
149
  """Check if file has 'secretscan: allow-pattern-docs' pragma in first 10 lines."""
@@ -101,12 +169,19 @@ def is_binary_file(filepath):
101
169
 
102
170
 
103
171
  def should_skip_file(filepath):
104
- """Check if file should be skipped entirely (.git/, __pycache__, .pyc, .pyo)."""
172
+ """Check if file should be skipped entirely (.git/, node_modules/, __pycache__, .pyc, .pyo)."""
105
173
  # Skip .git directories — match a path COMPONENT, not a substring
106
174
  # (".git" in str(path) also matched ".gitignore" and skipped scanning it).
107
175
  if ".git" in filepath.parts:
108
176
  return True
109
177
 
178
+ # Skip node_modules — third-party deps, always git-ignored, never our code.
179
+ # CI installs them via `npm ci` for the dashboard build, so a whole-tree
180
+ # scan would otherwise walk thousands of package files (README example
181
+ # connection strings, files literally named token.js → false positives).
182
+ if "node_modules" in filepath.parts:
183
+ return True
184
+
110
185
  # Skip __pycache__ directories
111
186
  if "__pycache__" in filepath.parts:
112
187
  return True
@@ -156,22 +231,8 @@ def scan_file(filepath):
156
231
  - Large files: scan first 1MB; emit SKIPPED-LARGE to stderr if file is larger
157
232
  - Binary files: decode as latin-1; emit SKIPPED-BINARY to stderr if not fully scanned
158
233
  """
159
- # Rules that CAN be softened by pragma (doc-shaped rules only)
160
- SOFTENED_BY_PRAGMA = {"generic_secret_assignment", "env_access"}
161
-
162
- # Rules that are ALWAYS fatal, pragma never applies
163
- FATAL_RULES = {
164
- "pem_private_key",
165
- "aws_access_key",
166
- "github_token",
167
- "slack_token",
168
- "openai_anthropic_key",
169
- "connection_string",
170
- }
171
-
172
234
  SIZE_THRESHOLD = 1024 * 1024 # 1MB
173
235
  MAX_READ_SIZE = 2 * 1024 * 1024 # 2MB max to read
174
- SCAN_PREFIX_SIZE = 1024 * 1024 # Scan first 1MB of large files
175
236
 
176
237
  findings = []
177
238
 
@@ -197,35 +258,8 @@ def scan_file(filepath):
197
258
  is_binary = is_binary_file(filepath)
198
259
  is_large = file_size > SIZE_THRESHOLD
199
260
 
200
- if is_large:
201
- # Scan first 1MB of large file for FATAL_RULES
202
- with open(filepath, "rb") as f:
203
- content = f.read(SCAN_PREFIX_SIZE)
204
-
205
- # Decode as latin-1 (handles any binary content)
206
- try:
207
- content_str = content.decode("latin-1")
208
- except Exception:
209
- content_str = content.decode("utf-8", errors="ignore")
210
-
211
- # Emit skip notice to stderr
212
- print(f"SKIPPED-LARGE {filepath} (scanned first {SCAN_PREFIX_SIZE // 1024}KB)", file=sys.stderr)
213
-
214
- # Scan content for FATAL_RULES only
215
- for line_num, line in enumerate(content_str.split("\n"), start=1):
216
- for rule_name in FATAL_RULES:
217
- if rule_name not in PATTERNS:
218
- continue
219
- pattern, flags = PATTERNS[rule_name]
220
- matches = re.finditer(pattern, line, flags)
221
- for match in matches:
222
- match_str = match.group(0)
223
- if is_placeholder(match_str):
224
- continue
225
- findings.append((line_num, rule_name, match_str, True))
226
-
227
- elif is_binary:
228
- # Scan binary file as latin-1 for FATAL_RULES only
261
+ if is_binary:
262
+ # Binary file (any size): scan as-is for FATAL_RULES only
229
263
  with open(filepath, "rb") as f:
230
264
  content = f.read(MAX_READ_SIZE)
231
265
 
@@ -251,8 +285,38 @@ def scan_file(filepath):
251
285
  continue
252
286
  findings.append((line_num, rule_name, match_str, True))
253
287
 
288
+ elif is_large:
289
+ # Large text file: scan entire file in chunks for all rules
290
+ print(f"SKIPPED-LARGE {filepath} (scanned in chunks)", file=sys.stderr)
291
+ line_num = 0
292
+ try:
293
+ with open(filepath, "r", encoding="utf-8", errors="ignore") as f:
294
+ # Read in 1MB chunks to avoid loading entire large file into memory
295
+ for chunk in iter(lambda: f.read(1024 * 1024), ""):
296
+ for chunk_line in chunk.split("\n"):
297
+ line_num += 1
298
+ for rule_name, (pattern, flags) in PATTERNS.items():
299
+ # Skip env_assignment rule if not an .env-like file
300
+ if rule_name == "env_assignment" and not is_env_file(filepath):
301
+ continue
302
+
303
+ matches = re.finditer(pattern, chunk_line, flags)
304
+ for match in matches:
305
+ match_str = match.group(0)
306
+
307
+ # Skip if it's a placeholder
308
+ if is_placeholder(match_str):
309
+ continue
310
+
311
+ is_fatal = _classify_finding(rule_name, has_file_pragma)
312
+ findings.append((line_num, rule_name, match_str, is_fatal))
313
+ except (IOError, OSError) as e:
314
+ # FAIL CLOSED: if we cannot fully scan a large text file, exit with error
315
+ print(f"FATAL: Cannot fully scan large text file {filepath}: {e}", file=sys.stderr)
316
+ sys.exit(1)
317
+
254
318
  else:
255
- # Normal text file: scan all rules
319
+ # Normal small text file: scan all rules
256
320
  with open(filepath, "r", encoding="utf-8", errors="ignore") as f:
257
321
  for line_num, line in enumerate(f, start=1):
258
322
  for rule_name, (pattern, flags) in PATTERNS.items():
@@ -268,17 +332,7 @@ def scan_file(filepath):
268
332
  if is_placeholder(match_str):
269
333
  continue
270
334
 
271
- # Determine fatality based on rule category
272
- if rule_name in FATAL_RULES:
273
- # These are always fatal, pragma never applies
274
- is_fatal = True
275
- elif has_file_pragma and rule_name in SOFTENED_BY_PRAGMA:
276
- # Only these rules can be softened by pragma
277
- is_fatal = False
278
- else:
279
- # Other rules are fatal unless pragma and softened
280
- is_fatal = not has_file_pragma if rule_name in SOFTENED_BY_PRAGMA else True
281
-
335
+ is_fatal = _classify_finding(rule_name, has_file_pragma)
282
336
  findings.append((line_num, rule_name, match_str, is_fatal))
283
337
 
284
338
  except Exception:
@@ -287,25 +341,190 @@ def scan_file(filepath):
287
341
  return findings
288
342
 
289
343
 
344
+ def scan_blob(label, content_bytes):
345
+ """
346
+ Scan raw blob bytes for secrets, reporting findings against `label` as the
347
+ display path. Mirrors scan_file()'s rule/fatality logic, but reads content
348
+ already fetched from a git object (via get_git_blob) instead of opening the
349
+ working-tree disk file. Used by --staged (the STAGED INDEX blob) and
350
+ --range (the blob at the TIP of the range) so a secret present only in the
351
+ git object being pushed -- but absent from an out-of-sync working copy --
352
+ is not missed (wave-25 P2 fix; see module docstring "Scan contract").
353
+
354
+ Returns list of (line_num, rule, match_str, is_fatal), like scan_file().
355
+ """
356
+ filepath = Path(label)
357
+ SIZE_THRESHOLD = 1024 * 1024 # 1MB
358
+ MAX_READ_SIZE = 2 * 1024 * 1024 # 2MB max to read (matches scan_file's binary cap)
359
+
360
+ findings = []
361
+
362
+ if should_skip_file(filepath):
363
+ return findings
364
+
365
+ filename = filepath.name.lower()
366
+ for pattern in CREDENTIAL_FILENAMES:
367
+ if re.match(pattern, filename, re.IGNORECASE):
368
+ findings.append(
369
+ (0, "credential_filename", f"File name matches credential pattern: {filepath.name}", True)
370
+ )
371
+ break
372
+
373
+ # Pragma check over the first 10 lines, mirroring has_pragma() but reading
374
+ # from the in-memory blob instead of opening a path from disk.
375
+ has_file_pragma = False
376
+ try:
377
+ probe = content_bytes.decode("utf-8", errors="ignore")
378
+ for i, line in enumerate(probe.split("\n")):
379
+ if i >= 10:
380
+ break
381
+ if "secretscan: allow-pattern-docs" in line:
382
+ has_file_pragma = True
383
+ break
384
+ except Exception:
385
+ pass
386
+
387
+ is_binary = b"\x00" in content_bytes[:8192]
388
+ is_large = len(content_bytes) > SIZE_THRESHOLD
389
+
390
+ if is_binary:
391
+ raw = content_bytes[:MAX_READ_SIZE]
392
+ try:
393
+ content_str = raw.decode("latin-1")
394
+ except Exception:
395
+ content_str = raw.decode("utf-8", errors="ignore")
396
+
397
+ print(f"SKIPPED-BINARY {filepath} (scanned via latin-1)", file=sys.stderr)
398
+
399
+ for line_num, line in enumerate(content_str.split("\n"), start=1):
400
+ for rule_name in FATAL_RULES:
401
+ pattern, flags = PATTERNS[rule_name]
402
+ for match in re.finditer(pattern, line, flags):
403
+ match_str = match.group(0)
404
+ if is_placeholder(match_str):
405
+ continue
406
+ findings.append((line_num, rule_name, match_str, True))
407
+ return findings
408
+
409
+ if is_large:
410
+ print(f"SKIPPED-LARGE {filepath} (scanned in full from git object)", file=sys.stderr)
411
+
412
+ try:
413
+ content_str = content_bytes.decode("utf-8", errors="ignore")
414
+ except Exception:
415
+ content_str = content_bytes.decode("latin-1", errors="ignore")
416
+
417
+ env_file = is_env_file(filepath)
418
+ for line_num, line in enumerate(content_str.split("\n"), start=1):
419
+ for rule_name, (pattern, flags) in PATTERNS.items():
420
+ if rule_name == "env_assignment" and not env_file:
421
+ continue
422
+
423
+ for match in re.finditer(pattern, line, flags):
424
+ match_str = match.group(0)
425
+ if is_placeholder(match_str):
426
+ continue
427
+ is_fatal = _classify_finding(rule_name, has_file_pragma)
428
+ findings.append((line_num, rule_name, match_str, is_fatal))
429
+
430
+ return findings
431
+
432
+
433
+ def _range_tip_ref(commit_range):
434
+ """Extract the right-hand (tip) ref from a two-dot or three-dot commit
435
+ range string, e.g. 'main..HEAD' -> 'HEAD', 'a...b' -> 'b'. Falls back to
436
+ the whole string if it isn't a recognizable range (single ref/sha)."""
437
+ if "..." in commit_range:
438
+ tip = commit_range.split("...", 1)[1]
439
+ elif ".." in commit_range:
440
+ tip = commit_range.split("..", 1)[1]
441
+ else:
442
+ tip = commit_range
443
+ return tip or "HEAD"
444
+
445
+
446
+ def get_git_blob(repo_path, ref_path):
447
+ """Fetch raw blob bytes via `git show <ref_path>` -- e.g. ref_path=':foo.py'
448
+ reads foo.py from the STAGED INDEX, 'abc123:foo.py' reads foo.py as it
449
+ exists in commit abc123.
450
+
451
+ Raises GitScanError if the git show command fails (non-zero returncode or
452
+ exception). Since callers use --diff-filter=d to exclude deleted paths from
453
+ enumeration, a git-show failure for an enumerated path is a real error
454
+ (e.g., corruption, permissions, unreadable blob), not a legitimate 'file
455
+ absent' case. Failing closed (raising GitScanError) ensures such errors
456
+ block the push, consistent with how enumeration failures already fail closed.
457
+ """
458
+ try:
459
+ result = subprocess.run(
460
+ ["git", "show", ref_path],
461
+ cwd=repo_path,
462
+ capture_output=True,
463
+ timeout=15,
464
+ )
465
+ except Exception as e:
466
+ raise GitScanError(
467
+ f"git show {ref_path!r} raised {e!r}"
468
+ )
469
+ if result.returncode != 0:
470
+ raise GitScanError(
471
+ f"git show {ref_path!r} failed (rc={result.returncode}): "
472
+ f"{result.stderr.decode('utf-8', errors='ignore').strip()}"
473
+ )
474
+ return result.stdout
475
+
476
+
290
477
  def get_staged_files(repo_path):
291
- """Get list of staged files from git repo."""
478
+ """Get list of staged file paths (relative to repo root) from git repo.
479
+
480
+ Raises GitScanError if the underlying `git diff --cached` invocation
481
+ fails; callers must fail CLOSED on that, not treat it as "nothing staged".
482
+ """
292
483
  try:
293
484
  result = subprocess.run(
294
- ["git", "diff", "--cached", "--name-only"],
485
+ ["git", "diff", "--cached", "--name-only", "--diff-filter=d"],
295
486
  cwd=repo_path,
296
487
  capture_output=True,
297
488
  text=True,
298
489
  timeout=10,
299
490
  )
300
- if result.returncode != 0:
301
- return []
302
- return [
303
- Path(repo_path) / f
304
- for f in result.stdout.strip().split("\n")
305
- if f.strip()
306
- ]
307
- except Exception:
308
- return []
491
+ except Exception as e:
492
+ raise GitScanError(f"git diff --cached raised {e!r}")
493
+
494
+ if result.returncode != 0:
495
+ raise GitScanError(
496
+ f"git diff --cached failed (rc={result.returncode}): {result.stderr.strip()}"
497
+ )
498
+
499
+ return [f for f in result.stdout.strip().split("\n") if f.strip()]
500
+
501
+
502
+ def get_range_files(repo_path, commit_range):
503
+ """Get list of file paths (relative to repo root) changed in commit range
504
+ (e.g. 'main..HEAD' or 'abc123..def456').
505
+
506
+ Raises GitScanError if the underlying `git diff` invocation fails (e.g.
507
+ an unresolvable ref); callers must fail CLOSED on that -- it is not the
508
+ same as a range that genuinely touches zero files.
509
+ """
510
+ try:
511
+ result = subprocess.run(
512
+ ["git", "diff", "--name-only", "--diff-filter=d", commit_range],
513
+ cwd=repo_path,
514
+ capture_output=True,
515
+ text=True,
516
+ timeout=10,
517
+ )
518
+ except Exception as e:
519
+ raise GitScanError(f"git diff --name-only {commit_range!r} raised {e!r}")
520
+
521
+ if result.returncode != 0:
522
+ raise GitScanError(
523
+ f"git diff --name-only {commit_range!r} failed (rc={result.returncode}): "
524
+ f"{result.stderr.strip()}"
525
+ )
526
+
527
+ return [f for f in result.stdout.strip().split("\n") if f.strip()]
309
528
 
310
529
 
311
530
  def get_history_files(repo_path):
@@ -378,6 +597,11 @@ def main():
378
597
  action="store_true",
379
598
  help="Scan git staged files (requires --repo or uses cwd)",
380
599
  )
600
+ parser.add_argument(
601
+ "--range",
602
+ metavar="COMMIT_RANGE",
603
+ help="Scan files changed in commit range (e.g., 'main..HEAD' or 'abc123..def456')",
604
+ )
381
605
  parser.add_argument(
382
606
  "--history",
383
607
  action="store_true",
@@ -394,36 +618,90 @@ def main():
394
618
 
395
619
  args = parser.parse_args()
396
620
 
397
- # Validate usage
398
- if sum([args.staged, args.history, bool(args.paths)]) != 1:
399
- print("ERROR: Use exactly one of --staged, --history, or path arguments", file=sys.stderr)
621
+ # Validate usage: exactly one of --staged, --range, --history, or paths
622
+ mode_count = sum([args.staged, bool(args.range), args.history, bool(args.paths)])
623
+ if mode_count != 1:
624
+ print("ERROR: Use exactly one of --staged, --range, --history, or path arguments", file=sys.stderr)
400
625
  sys.exit(2)
401
626
 
402
- # Collect files to scan
403
- if args.staged:
404
- files = get_staged_files(args.repo)
405
- elif args.history:
406
- # History mode: scan all files from git log
407
- history_files = get_history_files(args.repo)
408
- files = [] # We'll handle history differently
409
- else:
410
- files = scan_paths(args.paths)
411
-
412
- # Scan files
413
627
  all_findings = []
414
628
  fatal_findings = []
415
629
  allowed_doc_count = 0
630
+ file_count = 0
631
+
632
+ if args.staged:
633
+ # Scan the STAGED INDEX blob for each changed path, NOT the working-tree
634
+ # copy -- a worktree edit made after `git add` must not be able to hide
635
+ # a secret still sitting in the index that would actually get committed.
636
+ try:
637
+ relpaths = get_staged_files(args.repo)
638
+ except GitScanError as e:
639
+ print(f"FATAL: could not determine staged files: {e}", file=sys.stderr)
640
+ print("Failing CLOSED: refusing to report CLEAN when the staged-file listing could not be determined.", file=sys.stderr)
641
+ sys.exit(1)
642
+
643
+ file_count = len(relpaths)
644
+ for relpath in relpaths:
645
+ try:
646
+ content = get_git_blob(args.repo, f":{relpath}")
647
+ except GitScanError as e:
648
+ print(f"FATAL: could not read staged blob for {relpath}: {e}", file=sys.stderr)
649
+ print("Failing CLOSED: refusing to report CLEAN when a staged object cannot be read.", file=sys.stderr)
650
+ sys.exit(1)
651
+ label = str(Path(args.repo) / relpath)
652
+ findings = scan_blob(label, content)
653
+ for line_num, rule, match_str, is_fatal in findings:
654
+ all_findings.append((label, line_num, rule, match_str, is_fatal))
655
+ if is_fatal:
656
+ fatal_findings.append((label, line_num, rule, match_str))
657
+ else:
658
+ allowed_doc_count += 1
416
659
 
417
- if args.history:
418
- # History scanning mode
660
+ elif args.range:
661
+ # Scan the COMMITTED blob at the TIP of the range for each changed
662
+ # path, NOT the working-tree copy -- a secret committed then edited
663
+ # away in the worktree (without a new commit) must still be caught,
664
+ # since the pushed commit still carries the dirty blob.
665
+ try:
666
+ relpaths = get_range_files(args.repo, args.range)
667
+ except GitScanError as e:
668
+ print(f"FATAL: could not resolve commit range {args.range!r}: {e}", file=sys.stderr)
669
+ print("Failing CLOSED: refusing to report CLEAN on an unresolvable range.", file=sys.stderr)
670
+ sys.exit(1)
671
+
672
+ file_count = len(relpaths)
673
+ tip_ref = _range_tip_ref(args.range)
674
+ for relpath in relpaths:
675
+ try:
676
+ content = get_git_blob(args.repo, f"{tip_ref}:{relpath}")
677
+ except GitScanError as e:
678
+ print(f"FATAL: could not read blob at {tip_ref}:{relpath}: {e}", file=sys.stderr)
679
+ print("Failing CLOSED: refusing to report CLEAN when a committed object cannot be read.", file=sys.stderr)
680
+ sys.exit(1)
681
+ label = str(Path(args.repo) / relpath)
682
+ findings = scan_blob(label, content)
683
+ for line_num, rule, match_str, is_fatal in findings:
684
+ all_findings.append((label, line_num, rule, match_str, is_fatal))
685
+ if is_fatal:
686
+ fatal_findings.append((label, line_num, rule, match_str))
687
+ else:
688
+ allowed_doc_count += 1
689
+
690
+ elif args.history:
691
+ # History scanning mode: unaffected by the blob-scan fix above, since
692
+ # it already walks committed diff content via `git log -p`.
693
+ history_files = get_history_files(args.repo)
694
+ file_count = len(set(f for f, _ in history_files))
419
695
  for filepath, content in history_files:
420
696
  findings = scan_content(content)
421
697
  for line_num, rule, match_str, is_fatal in findings:
422
698
  all_findings.append((filepath, line_num, rule, match_str, is_fatal))
423
699
  if is_fatal:
424
700
  fatal_findings.append((filepath, line_num, rule, match_str))
701
+
425
702
  else:
426
- # Regular file scanning mode
703
+ files = scan_paths(args.paths)
704
+ file_count = len(files)
427
705
  for filepath in files:
428
706
  findings = scan_file(filepath)
429
707
  for line_num, rule, match_str, is_fatal in findings:
@@ -442,8 +720,6 @@ def main():
442
720
  print(f"ALLOWED-DOC {filepath}:{line_num} {rule} ({masked})")
443
721
 
444
722
  # Summary and exit
445
- file_count = len(files) if not args.history else len(set(f for f, _, _, _, _ in all_findings))
446
-
447
723
  if len(fatal_findings) == 0:
448
724
  if allowed_doc_count == 0:
449
725
  if args.history: