@matt82198/aesop 0.1.0-beta.5 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. package/CHANGELOG.md +158 -5
  2. package/LICENSE +66 -21
  3. package/README.md +97 -33
  4. package/aesop.config.example.json +6 -0
  5. package/bin/CLAUDE.md +12 -3
  6. package/bin/cli.js +211 -40
  7. package/daemons/CLAUDE.md +1 -1
  8. package/daemons/backup-fleet.sh +209 -51
  9. package/daemons/run-watchdog.sh +208 -62
  10. package/dash/dash-extra.mjs +73 -4
  11. package/dash/watchdog-gui.sh +41 -35
  12. package/docs/ARCHITECTURE.md +296 -0
  13. package/docs/CONCEPTS.md +254 -0
  14. package/docs/CONFIGURE.md +256 -0
  15. package/docs/FIRST-WAVE.md +276 -0
  16. package/docs/INSTALL.md +224 -0
  17. package/docs/README.md +176 -27
  18. package/docs/autonomous-swe.md +149 -0
  19. package/docs/reproduce.md +121 -0
  20. package/hooks/CLAUDE.md +28 -0
  21. package/hooks/install-waveguard.sh +68 -0
  22. package/hooks/pre-commit-waveguard.sh +26 -0
  23. package/hooks/pre-push-policy.sh +253 -15
  24. package/monitor/CLAUDE.md +3 -3
  25. package/monitor/collect-signals.mjs +133 -20
  26. package/package.json +15 -7
  27. package/scan/CLAUDE.md +30 -0
  28. package/skills/CLAUDE.md +1 -0
  29. package/state_store/ingest.py +18 -1
  30. package/state_store/projections.py +78 -4
  31. package/state_store/store.py +102 -7
  32. package/tools/CLAUDE.md +25 -18
  33. package/tools/alert_bridge.py +14 -10
  34. package/tools/bench_runner.py +438 -0
  35. package/tools/buildlog.py +4 -7
  36. package/tools/ci_merge_wait.py +215 -34
  37. package/tools/common.py +62 -0
  38. package/tools/cost_ceiling.py +191 -0
  39. package/tools/dash.js +102 -0
  40. package/tools/doctor.js +220 -0
  41. package/tools/fleet_ledger.py +189 -17
  42. package/tools/halt.py +172 -0
  43. package/tools/healthcheck.py +24 -21
  44. package/tools/heartbeat.py +4 -7
  45. package/tools/metrics_gate.py +8 -2
  46. package/tools/orchestrator_status.py +4 -9
  47. package/tools/reconcile.py +277 -0
  48. package/tools/rotate_logs.py +152 -62
  49. package/tools/scanner_selftest.py +28 -3
  50. package/tools/secret_scan.py +359 -90
  51. package/tools/self_stats.py +292 -4
  52. package/tools/status.js +187 -0
  53. package/tools/verify_agent_inspector.py +289 -0
  54. package/tools/verify_prboard.py +344 -0
  55. package/tools/watch.js +49 -0
  56. package/ui/CLAUDE.md +2 -0
  57. package/ui/agents.py +332 -33
  58. package/ui/api/tracker.py +15 -7
  59. package/ui/collectors.py +125 -32
  60. package/ui/config.py +22 -2
  61. package/ui/cost.py +98 -3
  62. package/ui/csrf.py +2 -1
  63. package/ui/handler.py +209 -21
  64. package/ui/sse.py +102 -16
  65. package/ui/wave_prs.py +230 -0
  66. package/ui/web/dist/assets/index-0qQYnvMC.js +9 -0
  67. package/ui/web/dist/assets/index-BdIlFieV.css +1 -0
  68. package/ui/web/dist/index.html +2 -2
  69. package/monitor/.signal-state.json +0 -3
  70. package/monitor/ACTIONS.log +0 -1
  71. package/monitor/BRIEF.md +0 -24
  72. package/monitor/SIGNALS.json +0 -54
  73. package/state_store/__pycache__/__init__.cpython-314.pyc +0 -0
  74. package/state_store/__pycache__/api.cpython-314.pyc +0 -0
  75. package/state_store/__pycache__/export.cpython-314.pyc +0 -0
  76. package/state_store/__pycache__/ingest.cpython-314.pyc +0 -0
  77. package/state_store/__pycache__/projections.cpython-314.pyc +0 -0
  78. package/state_store/__pycache__/store.cpython-314.pyc +0 -0
  79. package/tools/__pycache__/alert_bridge.cpython-314.pyc +0 -0
  80. package/tools/__pycache__/buildlog.cpython-314.pyc +0 -0
  81. package/tools/__pycache__/ci_merge_wait.cpython-314.pyc +0 -0
  82. package/tools/__pycache__/ensure_state.cpython-314.pyc +0 -0
  83. package/tools/__pycache__/eod_sweep.cpython-314.pyc +0 -0
  84. package/tools/__pycache__/fleet_ledger.cpython-314.pyc +0 -0
  85. package/tools/__pycache__/fleet_prompt_extractor.cpython-314.pyc +0 -0
  86. package/tools/__pycache__/healthcheck.cpython-314.pyc +0 -0
  87. package/tools/__pycache__/heartbeat.cpython-314.pyc +0 -0
  88. package/tools/__pycache__/inbox_drain.cpython-314.pyc +0 -0
  89. package/tools/__pycache__/launch_tui.cpython-314.pyc +0 -0
  90. package/tools/__pycache__/metrics_gate.cpython-314.pyc +0 -0
  91. package/tools/__pycache__/orchestrator_status.cpython-314.pyc +0 -0
  92. package/tools/__pycache__/power_selftest.cpython-314.pyc +0 -0
  93. package/tools/__pycache__/prepublish_scan.cpython-314.pyc +0 -0
  94. package/tools/__pycache__/rotate_logs.cpython-314.pyc +0 -0
  95. package/tools/__pycache__/scanner_selftest.cpython-314.pyc +0 -0
  96. package/tools/__pycache__/secret_scan.cpython-314.pyc +0 -0
  97. package/tools/__pycache__/self_stats.cpython-314.pyc +0 -0
  98. package/tools/__pycache__/session_usage_summary.cpython-314.pyc +0 -0
  99. package/tools/__pycache__/stall_check.cpython-314.pyc +0 -0
  100. package/tools/__pycache__/transcript_replay.cpython-314.pyc +0 -0
  101. package/tools/__pycache__/transcript_timeline.cpython-314.pyc +0 -0
  102. package/tools/__pycache__/verify_dash.cpython-314.pyc +0 -0
  103. package/tools/__pycache__/verify_submit_encoding.cpython-314.pyc +0 -0
  104. package/ui/__pycache__/agents.cpython-314.pyc +0 -0
  105. package/ui/__pycache__/collectors.cpython-314.pyc +0 -0
  106. package/ui/__pycache__/config.cpython-314.pyc +0 -0
  107. package/ui/__pycache__/cost.cpython-314.pyc +0 -0
  108. package/ui/__pycache__/csrf.cpython-314.pyc +0 -0
  109. package/ui/__pycache__/handler.cpython-314.pyc +0 -0
  110. package/ui/__pycache__/render.cpython-314.pyc +0 -0
  111. package/ui/__pycache__/serve.cpython-314.pyc +0 -0
  112. package/ui/__pycache__/sse.cpython-314.pyc +0 -0
  113. package/ui/api/__pycache__/__init__.cpython-314.pyc +0 -0
  114. package/ui/api/__pycache__/submit.cpython-314.pyc +0 -0
  115. package/ui/api/__pycache__/tracker.cpython-314.pyc +0 -0
  116. package/ui/web/.gitattributes +0 -13
  117. package/ui/web/dist/assets/index-2LZDQirC.js +0 -9
  118. package/ui/web/dist/assets/index-D4M1qyOv.css +0 -1
  119. package/ui/web/index.html +0 -13
  120. package/ui/web/package-lock.json +0 -2225
  121. package/ui/web/package.json +0 -26
  122. package/ui/web/src/App.test.tsx +0 -74
  123. package/ui/web/src/App.tsx +0 -142
  124. package/ui/web/src/CONTRIBUTING-UI.md +0 -49
  125. package/ui/web/src/components/AgentRow.css +0 -187
  126. package/ui/web/src/components/AgentRow.test.tsx +0 -209
  127. package/ui/web/src/components/AgentRow.tsx +0 -207
  128. package/ui/web/src/components/AgentsPanel.css +0 -108
  129. package/ui/web/src/components/AgentsPanel.test.tsx +0 -41
  130. package/ui/web/src/components/AgentsPanel.tsx +0 -58
  131. package/ui/web/src/components/AlertsPanel.css +0 -88
  132. package/ui/web/src/components/AlertsPanel.test.tsx +0 -51
  133. package/ui/web/src/components/AlertsPanel.tsx +0 -67
  134. package/ui/web/src/components/BacklogPanel.test.tsx +0 -126
  135. package/ui/web/src/components/BacklogPanel.tsx +0 -122
  136. package/ui/web/src/components/CostChart.css +0 -110
  137. package/ui/web/src/components/CostChart.test.tsx +0 -144
  138. package/ui/web/src/components/CostChart.tsx +0 -152
  139. package/ui/web/src/components/CostTable.css +0 -93
  140. package/ui/web/src/components/CostTable.test.tsx +0 -165
  141. package/ui/web/src/components/CostTable.tsx +0 -94
  142. package/ui/web/src/components/EventsFeed.css +0 -68
  143. package/ui/web/src/components/EventsFeed.test.tsx +0 -36
  144. package/ui/web/src/components/EventsFeed.tsx +0 -31
  145. package/ui/web/src/components/HealthHeader.css +0 -137
  146. package/ui/web/src/components/HealthHeader.test.tsx +0 -278
  147. package/ui/web/src/components/HealthHeader.tsx +0 -281
  148. package/ui/web/src/components/InboxForm.css +0 -135
  149. package/ui/web/src/components/InboxForm.test.tsx +0 -208
  150. package/ui/web/src/components/InboxForm.tsx +0 -116
  151. package/ui/web/src/components/MessagesTail.module.css +0 -144
  152. package/ui/web/src/components/MessagesTail.test.tsx +0 -176
  153. package/ui/web/src/components/MessagesTail.tsx +0 -94
  154. package/ui/web/src/components/ReposPanel.css +0 -90
  155. package/ui/web/src/components/ReposPanel.test.tsx +0 -45
  156. package/ui/web/src/components/ReposPanel.tsx +0 -67
  157. package/ui/web/src/components/Scorecard.css +0 -106
  158. package/ui/web/src/components/Scorecard.test.tsx +0 -117
  159. package/ui/web/src/components/Scorecard.tsx +0 -85
  160. package/ui/web/src/components/Timeline.module.css +0 -151
  161. package/ui/web/src/components/Timeline.test.tsx +0 -215
  162. package/ui/web/src/components/Timeline.tsx +0 -99
  163. package/ui/web/src/components/TrackerBoard.test.tsx +0 -121
  164. package/ui/web/src/components/TrackerBoard.tsx +0 -107
  165. package/ui/web/src/components/TrackerCard.test.tsx +0 -180
  166. package/ui/web/src/components/TrackerCard.tsx +0 -160
  167. package/ui/web/src/components/TrackerForm.test.tsx +0 -189
  168. package/ui/web/src/components/TrackerForm.tsx +0 -144
  169. package/ui/web/src/lib/api.ts +0 -218
  170. package/ui/web/src/lib/format.test.ts +0 -89
  171. package/ui/web/src/lib/format.ts +0 -103
  172. package/ui/web/src/lib/sanitizeUrl.test.ts +0 -84
  173. package/ui/web/src/lib/sanitizeUrl.ts +0 -38
  174. package/ui/web/src/lib/types.ts +0 -230
  175. package/ui/web/src/lib/useHashRoute.test.ts +0 -60
  176. package/ui/web/src/lib/useHashRoute.ts +0 -23
  177. package/ui/web/src/lib/useSSE.ts +0 -175
  178. package/ui/web/src/main.tsx +0 -10
  179. package/ui/web/src/styles/global.css +0 -179
  180. package/ui/web/src/styles/theme.css +0 -184
  181. package/ui/web/src/styles/work.css +0 -572
  182. package/ui/web/src/test/fixtures.ts +0 -385
  183. package/ui/web/src/test/setup.ts +0 -49
  184. package/ui/web/src/views/Activity.module.css +0 -43
  185. package/ui/web/src/views/Activity.test.tsx +0 -89
  186. package/ui/web/src/views/Activity.tsx +0 -31
  187. package/ui/web/src/views/Cost.css +0 -87
  188. package/ui/web/src/views/Cost.test.tsx +0 -142
  189. package/ui/web/src/views/Cost.tsx +0 -54
  190. package/ui/web/src/views/Overview.css +0 -51
  191. package/ui/web/src/views/Overview.test.tsx +0 -76
  192. package/ui/web/src/views/Overview.tsx +0 -46
  193. package/ui/web/src/views/Work.test.tsx +0 -82
  194. package/ui/web/src/views/Work.tsx +0 -79
  195. package/ui/web/src/vite-env.d.ts +0 -10
  196. package/ui/web/tsconfig.json +0 -22
  197. package/ui/web/vite.config.ts +0 -25
  198. package/ui/web/vitest.config.ts +0 -12
@@ -158,8 +158,8 @@ def main():
158
158
  (
159
159
  # Doc-shaped rule: generic assignment IS softened by pragma.
160
160
  "tp_pragma_generic_softened",
161
- "# secretscan: allow-pattern-docs\\n"
162
- + "pass" + "word = \"this_is_a_doc_sample_value_12345\"\\n",
161
+ "# secretscan: allow-pattern-docs\n"
162
+ + "password = \"this_is_a_doc_sample_value_12345\"\n",
163
163
  "pattern_docs.py",
164
164
  0,
165
165
  "ALLOWED-DOC"
@@ -235,7 +235,7 @@ def main():
235
235
  "tp_connection_real_credentials",
236
236
  "# Real production DB\\ndb_url = '"
237
237
  + "postgresql:" + "//produser:RealPassword123"
238
- + "@db.prod.example.com:5432/production" + "'\\n",
238
+ + "@db.production.io:5432/production" + "'\\n",
239
239
  "config_prod.py",
240
240
  1,
241
241
  "connection_string"
@@ -266,6 +266,31 @@ def main():
266
266
  0,
267
267
  "SKIPPED-LARGE"
268
268
  ),
269
+ (
270
+ # REGRESSION: Unquoted password assignment should be detected
271
+ # (previously required quotes to match the pattern)
272
+ "tp_unquoted_password_assignment",
273
+ "password = supersecretvalue123\n",
274
+ "config.py",
275
+ 1,
276
+ "generic_secret_assignment"
277
+ ),
278
+ (
279
+ # REGRESSION: Unquoted API key assignment should be detected
280
+ "tp_unquoted_api_key_assignment",
281
+ "api_key = longsecretapikey1234567890\n",
282
+ "settings.py",
283
+ 1,
284
+ "generic_secret_assignment"
285
+ ),
286
+ (
287
+ # REGRESSION: Unquoted token in .env should be detected
288
+ "tp_unquoted_token_env_file",
289
+ "API_TOKEN=verylongtokenvalue123456789\n",
290
+ ".env",
291
+ 1,
292
+ "env_assignment"
293
+ ),
269
294
  ]
270
295
 
271
296
  results = []
@@ -3,9 +3,10 @@
3
3
  secret_scan.py — Pre-push secret/credential detection gate.
4
4
 
5
5
  Modes:
6
- secret_scan.py --staged [--repo PATH] Scan git staged files (default repo=cwd)
7
- secret_scan.py --history [--repo PATH] Scan all blobs in git history
8
- secret_scan.py PATH [PATH...] Scan files/dirs directly (recurse dirs)
6
+ secret_scan.py --staged [--repo PATH] Scan git staged files (default repo=cwd)
7
+ secret_scan.py --range COMMIT_RANGE [--repo PATH] Scan files changed in range (e.g., main..HEAD or abc123..def456)
8
+ secret_scan.py --history [--repo PATH] Scan all blobs in git history
9
+ secret_scan.py PATH [PATH...] Scan files/dirs directly (recurse dirs)
9
10
 
10
11
  Exit codes: 0=clean, 1=findings, 2=usage error
11
12
  Output: one line per finding or summary (never prints full secrets)
@@ -22,6 +23,19 @@ Self-scan invariant: this file must scan CLEAN with NO pragma. Any pattern liter
22
23
  would match its own regex is runtime-assembled from fragments (see pem_private_key) so
23
24
  the pattern text never appears contiguously in this source.
24
25
 
26
+ Scan contract (wave-25 P2 fix): --staged and --range scan committed GIT OBJECTS, not the
27
+ working-tree copy of changed files. --staged reads each path's STAGED INDEX blob
28
+ (`git show :<path>`); --range reads each changed path's blob at the TIP of the range
29
+ (`git show <tip>:<path>`). This closes two bypasses that existed when these modes read
30
+ $repo/<path> off disk: (1) stage a secret, then edit the on-disk file without
31
+ re-staging -- the dirty blob is still what gets committed/pushed; (2) commit a secret,
32
+ then edit it away in the worktree without a new commit -- the dirty blob is still what
33
+ the pushed commit carries. --history is unaffected (it already walks committed diffs).
34
+ Any git command needed to enumerate the files-to-scan that itself fails (bad ref,
35
+ unresolvable range, git error) is NOT treated as "zero files changed" -- it raises
36
+ GitScanError and the caller fails CLOSED (non-zero exit), so an unresolvable range can
37
+ never silently report CLEAN.
38
+
25
39
  NOTE: Public repo version has NO vault allowlist.
26
40
  """
27
41
 
@@ -50,10 +64,30 @@ PATTERNS = {
50
64
  "slack_token": (r"xox[baprs]-[A-Za-z0-9-]{10,}", 0),
51
65
  "openai_anthropic_key": (r"sk-[A-Za-z0-9_\-]{20,}", 0),
52
66
  "generic_secret_assignment": (
53
- r"(password|passwd|secret|api[_-]?key|token|authorization)\s*[:=]\s*[\"'](?!.*(?:xxx|changeme|your-|<|$\{|example)\b).{8,}[\"']",
67
+ # Unquoted branch requires 8+ contiguous chars from a bare-literal charset
68
+ # (no '.', '(', ')' — real code expressions like `secrets.token_urlsafe(32)`,
69
+ # `self.serve.SESSION_TOKEN`, or `headers.get(...)` all contain one of those)
70
+ # followed by whitespace/EOL, so an assignment from a variable/attribute/call
71
+ # doesn't false-positive as a hardcoded secret the way a bare env-file value
72
+ # (`API_TOKEN=verylongtokenvalue123456789`) legitimately does.
73
+ r"\b(password|passwd|secret|api[_-]?key|token|authorization)\b\s*(?::=|=)\s*(?:[\"'](?!.*(?:xxx|changeme|your-|<|$\{|example)\b).{8,}[\"']|(?!['\"]|xxx|changeme|your-|example)[^\s\$\<\{\n\(\)\.\"']{8,}(?=\s|$))",
54
74
  re.IGNORECASE,
55
75
  ),
56
- "connection_string": (r"://[^:]+:[^@/\s]+@(?!localhost(?:[:/]|$)|127\.0\.0\.1(?:[:/]|$)|example\.com(?:[:/]|$))[^\s]+", 0),
76
+ "connection_string": (
77
+ (r"://"
78
+ r"[^:]+:[^@/\s]+@"
79
+ r"(?!"
80
+ r"localhost(?:[:/]|$)|"
81
+ r"127\.0\.0\.1(?:[:/]|$)|"
82
+ r"127\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(?:[:/]|$)|"
83
+ r"[a-zA-Z0-9.-]*\.local(?:[:/]|$)|"
84
+ r"[a-zA-Z0-9.-]*\.localdomain(?:[:/]|$)|"
85
+ r"(?:[a-zA-Z0-9-]+\.)*example(?:\.[a-zA-Z]{2,})?(?:[:/]|$)|"
86
+ r"(?:[a-zA-Z0-9-]+\.)*test(?:\.[a-zA-Z]{2,})?(?:[:/]|$)"
87
+ r")"
88
+ r"[^\s]+"),
89
+ 0
90
+ ),
57
91
  "env_access": (
58
92
  r"(?i:(?:os\.getenv|os\.environ|System\.getenv|process\.env)\s*[\[\(][\"']?[A-Z_]*(?:password|secret|api[_-]?key|token|auth|key)[A-Z_0-9]*[\"']?[\)\]])",
59
93
  0,
@@ -76,6 +110,40 @@ CREDENTIAL_FILENAMES = [
76
110
  # Placeholders that don't count as secrets
77
111
  PLACEHOLDERS = {"xxx", "changeme", "your-key-here", "example", "test", "demo"}
78
112
 
113
+ # Rules that CAN be softened by pragma (doc-shaped rules only)
114
+ SOFTENED_BY_PRAGMA = {"generic_secret_assignment", "env_access"}
115
+
116
+ # Rules that are ALWAYS fatal, pragma never applies
117
+ FATAL_RULES = {
118
+ "pem_private_key",
119
+ "aws_access_key",
120
+ "github_token",
121
+ "slack_token",
122
+ "openai_anthropic_key",
123
+ "connection_string",
124
+ }
125
+
126
+
127
+ def _classify_finding(rule_name, has_file_pragma):
128
+ """Shared fatal/softened decision, used by every scan_* variant (disk-file,
129
+ blob, large-file-chunked) so the pragma contract can't drift between them.
130
+ FATAL_RULES are always fatal. SOFTENED_BY_PRAGMA rules are fatal unless the
131
+ pragma is present. Everything else (e.g. env_assignment) is always fatal."""
132
+ if rule_name in FATAL_RULES:
133
+ return True
134
+ if rule_name in SOFTENED_BY_PRAGMA:
135
+ return not has_file_pragma
136
+ return True
137
+
138
+
139
+ class GitScanError(Exception):
140
+ """Raised when a git command needed to enumerate files-to-scan fails.
141
+ Callers MUST treat this as fail-CLOSED (block / exit non-zero) -- it is
142
+ NOT equivalent to "the range/index genuinely contains zero changed
143
+ files" (wave-25 P2 fix: get_range_files previously returned [] on ANY
144
+ git-diff error, indistinguishable from a real empty range, so main()
145
+ printed CLEAN and exited 0 on an unresolvable range)."""
146
+
79
147
 
80
148
  def has_pragma(filepath):
81
149
  """Check if file has 'secretscan: allow-pattern-docs' pragma in first 10 lines."""
@@ -163,22 +231,8 @@ def scan_file(filepath):
163
231
  - Large files: scan first 1MB; emit SKIPPED-LARGE to stderr if file is larger
164
232
  - Binary files: decode as latin-1; emit SKIPPED-BINARY to stderr if not fully scanned
165
233
  """
166
- # Rules that CAN be softened by pragma (doc-shaped rules only)
167
- SOFTENED_BY_PRAGMA = {"generic_secret_assignment", "env_access"}
168
-
169
- # Rules that are ALWAYS fatal, pragma never applies
170
- FATAL_RULES = {
171
- "pem_private_key",
172
- "aws_access_key",
173
- "github_token",
174
- "slack_token",
175
- "openai_anthropic_key",
176
- "connection_string",
177
- }
178
-
179
234
  SIZE_THRESHOLD = 1024 * 1024 # 1MB
180
235
  MAX_READ_SIZE = 2 * 1024 * 1024 # 2MB max to read
181
- SCAN_PREFIX_SIZE = 1024 * 1024 # Scan first 1MB of large files
182
236
 
183
237
  findings = []
184
238
 
@@ -204,35 +258,8 @@ def scan_file(filepath):
204
258
  is_binary = is_binary_file(filepath)
205
259
  is_large = file_size > SIZE_THRESHOLD
206
260
 
207
- if is_large:
208
- # Scan first 1MB of large file for FATAL_RULES
209
- with open(filepath, "rb") as f:
210
- content = f.read(SCAN_PREFIX_SIZE)
211
-
212
- # Decode as latin-1 (handles any binary content)
213
- try:
214
- content_str = content.decode("latin-1")
215
- except Exception:
216
- content_str = content.decode("utf-8", errors="ignore")
217
-
218
- # Emit skip notice to stderr
219
- print(f"SKIPPED-LARGE {filepath} (scanned first {SCAN_PREFIX_SIZE // 1024}KB)", file=sys.stderr)
220
-
221
- # Scan content for FATAL_RULES only
222
- for line_num, line in enumerate(content_str.split("\n"), start=1):
223
- for rule_name in FATAL_RULES:
224
- if rule_name not in PATTERNS:
225
- continue
226
- pattern, flags = PATTERNS[rule_name]
227
- matches = re.finditer(pattern, line, flags)
228
- for match in matches:
229
- match_str = match.group(0)
230
- if is_placeholder(match_str):
231
- continue
232
- findings.append((line_num, rule_name, match_str, True))
233
-
234
- elif is_binary:
235
- # Scan binary file as latin-1 for FATAL_RULES only
261
+ if is_binary:
262
+ # Binary file (any size): scan as-is for FATAL_RULES only
236
263
  with open(filepath, "rb") as f:
237
264
  content = f.read(MAX_READ_SIZE)
238
265
 
@@ -258,8 +285,38 @@ def scan_file(filepath):
258
285
  continue
259
286
  findings.append((line_num, rule_name, match_str, True))
260
287
 
288
+ elif is_large:
289
+ # Large text file: scan entire file in chunks for all rules
290
+ print(f"SKIPPED-LARGE {filepath} (scanned in chunks)", file=sys.stderr)
291
+ line_num = 0
292
+ try:
293
+ with open(filepath, "r", encoding="utf-8", errors="ignore") as f:
294
+ # Read in 1MB chunks to avoid loading entire large file into memory
295
+ for chunk in iter(lambda: f.read(1024 * 1024), ""):
296
+ for chunk_line in chunk.split("\n"):
297
+ line_num += 1
298
+ for rule_name, (pattern, flags) in PATTERNS.items():
299
+ # Skip env_assignment rule if not an .env-like file
300
+ if rule_name == "env_assignment" and not is_env_file(filepath):
301
+ continue
302
+
303
+ matches = re.finditer(pattern, chunk_line, flags)
304
+ for match in matches:
305
+ match_str = match.group(0)
306
+
307
+ # Skip if it's a placeholder
308
+ if is_placeholder(match_str):
309
+ continue
310
+
311
+ is_fatal = _classify_finding(rule_name, has_file_pragma)
312
+ findings.append((line_num, rule_name, match_str, is_fatal))
313
+ except (IOError, OSError) as e:
314
+ # FAIL CLOSED: if we cannot fully scan a large text file, exit with error
315
+ print(f"FATAL: Cannot fully scan large text file {filepath}: {e}", file=sys.stderr)
316
+ sys.exit(1)
317
+
261
318
  else:
262
- # Normal text file: scan all rules
319
+ # Normal small text file: scan all rules
263
320
  with open(filepath, "r", encoding="utf-8", errors="ignore") as f:
264
321
  for line_num, line in enumerate(f, start=1):
265
322
  for rule_name, (pattern, flags) in PATTERNS.items():
@@ -275,17 +332,7 @@ def scan_file(filepath):
275
332
  if is_placeholder(match_str):
276
333
  continue
277
334
 
278
- # Determine fatality based on rule category
279
- if rule_name in FATAL_RULES:
280
- # These are always fatal, pragma never applies
281
- is_fatal = True
282
- elif has_file_pragma and rule_name in SOFTENED_BY_PRAGMA:
283
- # Only these rules can be softened by pragma
284
- is_fatal = False
285
- else:
286
- # Other rules are fatal unless pragma and softened
287
- is_fatal = not has_file_pragma if rule_name in SOFTENED_BY_PRAGMA else True
288
-
335
+ is_fatal = _classify_finding(rule_name, has_file_pragma)
289
336
  findings.append((line_num, rule_name, match_str, is_fatal))
290
337
 
291
338
  except Exception:
@@ -294,25 +341,190 @@ def scan_file(filepath):
294
341
  return findings
295
342
 
296
343
 
344
+ def scan_blob(label, content_bytes):
345
+ """
346
+ Scan raw blob bytes for secrets, reporting findings against `label` as the
347
+ display path. Mirrors scan_file()'s rule/fatality logic, but reads content
348
+ already fetched from a git object (via get_git_blob) instead of opening the
349
+ working-tree disk file. Used by --staged (the STAGED INDEX blob) and
350
+ --range (the blob at the TIP of the range) so a secret present only in the
351
+ git object being pushed -- but absent from an out-of-sync working copy --
352
+ is not missed (wave-25 P2 fix; see module docstring "Scan contract").
353
+
354
+ Returns list of (line_num, rule, match_str, is_fatal), like scan_file().
355
+ """
356
+ filepath = Path(label)
357
+ SIZE_THRESHOLD = 1024 * 1024 # 1MB
358
+ MAX_READ_SIZE = 2 * 1024 * 1024 # 2MB max to read (matches scan_file's binary cap)
359
+
360
+ findings = []
361
+
362
+ if should_skip_file(filepath):
363
+ return findings
364
+
365
+ filename = filepath.name.lower()
366
+ for pattern in CREDENTIAL_FILENAMES:
367
+ if re.match(pattern, filename, re.IGNORECASE):
368
+ findings.append(
369
+ (0, "credential_filename", f"File name matches credential pattern: {filepath.name}", True)
370
+ )
371
+ break
372
+
373
+ # Pragma check over the first 10 lines, mirroring has_pragma() but reading
374
+ # from the in-memory blob instead of opening a path from disk.
375
+ has_file_pragma = False
376
+ try:
377
+ probe = content_bytes.decode("utf-8", errors="ignore")
378
+ for i, line in enumerate(probe.split("\n")):
379
+ if i >= 10:
380
+ break
381
+ if "secretscan: allow-pattern-docs" in line:
382
+ has_file_pragma = True
383
+ break
384
+ except Exception:
385
+ pass
386
+
387
+ is_binary = b"\x00" in content_bytes[:8192]
388
+ is_large = len(content_bytes) > SIZE_THRESHOLD
389
+
390
+ if is_binary:
391
+ raw = content_bytes[:MAX_READ_SIZE]
392
+ try:
393
+ content_str = raw.decode("latin-1")
394
+ except Exception:
395
+ content_str = raw.decode("utf-8", errors="ignore")
396
+
397
+ print(f"SKIPPED-BINARY {filepath} (scanned via latin-1)", file=sys.stderr)
398
+
399
+ for line_num, line in enumerate(content_str.split("\n"), start=1):
400
+ for rule_name in FATAL_RULES:
401
+ pattern, flags = PATTERNS[rule_name]
402
+ for match in re.finditer(pattern, line, flags):
403
+ match_str = match.group(0)
404
+ if is_placeholder(match_str):
405
+ continue
406
+ findings.append((line_num, rule_name, match_str, True))
407
+ return findings
408
+
409
+ if is_large:
410
+ print(f"SKIPPED-LARGE {filepath} (scanned in full from git object)", file=sys.stderr)
411
+
412
+ try:
413
+ content_str = content_bytes.decode("utf-8", errors="ignore")
414
+ except Exception:
415
+ content_str = content_bytes.decode("latin-1", errors="ignore")
416
+
417
+ env_file = is_env_file(filepath)
418
+ for line_num, line in enumerate(content_str.split("\n"), start=1):
419
+ for rule_name, (pattern, flags) in PATTERNS.items():
420
+ if rule_name == "env_assignment" and not env_file:
421
+ continue
422
+
423
+ for match in re.finditer(pattern, line, flags):
424
+ match_str = match.group(0)
425
+ if is_placeholder(match_str):
426
+ continue
427
+ is_fatal = _classify_finding(rule_name, has_file_pragma)
428
+ findings.append((line_num, rule_name, match_str, is_fatal))
429
+
430
+ return findings
431
+
432
+
433
+ def _range_tip_ref(commit_range):
434
+ """Extract the right-hand (tip) ref from a two-dot or three-dot commit
435
+ range string, e.g. 'main..HEAD' -> 'HEAD', 'a...b' -> 'b'. Falls back to
436
+ the whole string if it isn't a recognizable range (single ref/sha)."""
437
+ if "..." in commit_range:
438
+ tip = commit_range.split("...", 1)[1]
439
+ elif ".." in commit_range:
440
+ tip = commit_range.split("..", 1)[1]
441
+ else:
442
+ tip = commit_range
443
+ return tip or "HEAD"
444
+
445
+
446
+ def get_git_blob(repo_path, ref_path):
447
+ """Fetch raw blob bytes via `git show <ref_path>` -- e.g. ref_path=':foo.py'
448
+ reads foo.py from the STAGED INDEX, 'abc123:foo.py' reads foo.py as it
449
+ exists in commit abc123.
450
+
451
+ Raises GitScanError if the git show command fails (non-zero returncode or
452
+ exception). Since callers use --diff-filter=d to exclude deleted paths from
453
+ enumeration, a git-show failure for an enumerated path is a real error
454
+ (e.g., corruption, permissions, unreadable blob), not a legitimate 'file
455
+ absent' case. Failing closed (raising GitScanError) ensures such errors
456
+ block the push, consistent with how enumeration failures already fail closed.
457
+ """
458
+ try:
459
+ result = subprocess.run(
460
+ ["git", "show", ref_path],
461
+ cwd=repo_path,
462
+ capture_output=True,
463
+ timeout=15,
464
+ )
465
+ except Exception as e:
466
+ raise GitScanError(
467
+ f"git show {ref_path!r} raised {e!r}"
468
+ )
469
+ if result.returncode != 0:
470
+ raise GitScanError(
471
+ f"git show {ref_path!r} failed (rc={result.returncode}): "
472
+ f"{result.stderr.decode('utf-8', errors='ignore').strip()}"
473
+ )
474
+ return result.stdout
475
+
476
+
297
477
  def get_staged_files(repo_path):
298
- """Get list of staged files from git repo."""
478
+ """Get list of staged file paths (relative to repo root) from git repo.
479
+
480
+ Raises GitScanError if the underlying `git diff --cached` invocation
481
+ fails; callers must fail CLOSED on that, not treat it as "nothing staged".
482
+ """
299
483
  try:
300
484
  result = subprocess.run(
301
- ["git", "diff", "--cached", "--name-only"],
485
+ ["git", "diff", "--cached", "--name-only", "--diff-filter=d"],
302
486
  cwd=repo_path,
303
487
  capture_output=True,
304
488
  text=True,
305
489
  timeout=10,
306
490
  )
307
- if result.returncode != 0:
308
- return []
309
- return [
310
- Path(repo_path) / f
311
- for f in result.stdout.strip().split("\n")
312
- if f.strip()
313
- ]
314
- except Exception:
315
- return []
491
+ except Exception as e:
492
+ raise GitScanError(f"git diff --cached raised {e!r}")
493
+
494
+ if result.returncode != 0:
495
+ raise GitScanError(
496
+ f"git diff --cached failed (rc={result.returncode}): {result.stderr.strip()}"
497
+ )
498
+
499
+ return [f for f in result.stdout.strip().split("\n") if f.strip()]
500
+
501
+
502
+ def get_range_files(repo_path, commit_range):
503
+ """Get list of file paths (relative to repo root) changed in commit range
504
+ (e.g. 'main..HEAD' or 'abc123..def456').
505
+
506
+ Raises GitScanError if the underlying `git diff` invocation fails (e.g.
507
+ an unresolvable ref); callers must fail CLOSED on that -- it is not the
508
+ same as a range that genuinely touches zero files.
509
+ """
510
+ try:
511
+ result = subprocess.run(
512
+ ["git", "diff", "--name-only", "--diff-filter=d", commit_range],
513
+ cwd=repo_path,
514
+ capture_output=True,
515
+ text=True,
516
+ timeout=10,
517
+ )
518
+ except Exception as e:
519
+ raise GitScanError(f"git diff --name-only {commit_range!r} raised {e!r}")
520
+
521
+ if result.returncode != 0:
522
+ raise GitScanError(
523
+ f"git diff --name-only {commit_range!r} failed (rc={result.returncode}): "
524
+ f"{result.stderr.strip()}"
525
+ )
526
+
527
+ return [f for f in result.stdout.strip().split("\n") if f.strip()]
316
528
 
317
529
 
318
530
  def get_history_files(repo_path):
@@ -385,6 +597,11 @@ def main():
385
597
  action="store_true",
386
598
  help="Scan git staged files (requires --repo or uses cwd)",
387
599
  )
600
+ parser.add_argument(
601
+ "--range",
602
+ metavar="COMMIT_RANGE",
603
+ help="Scan files changed in commit range (e.g., 'main..HEAD' or 'abc123..def456')",
604
+ )
388
605
  parser.add_argument(
389
606
  "--history",
390
607
  action="store_true",
@@ -401,36 +618,90 @@ def main():
401
618
 
402
619
  args = parser.parse_args()
403
620
 
404
- # Validate usage
405
- if sum([args.staged, args.history, bool(args.paths)]) != 1:
406
- print("ERROR: Use exactly one of --staged, --history, or path arguments", file=sys.stderr)
621
+ # Validate usage: exactly one of --staged, --range, --history, or paths
622
+ mode_count = sum([args.staged, bool(args.range), args.history, bool(args.paths)])
623
+ if mode_count != 1:
624
+ print("ERROR: Use exactly one of --staged, --range, --history, or path arguments", file=sys.stderr)
407
625
  sys.exit(2)
408
626
 
409
- # Collect files to scan
410
- if args.staged:
411
- files = get_staged_files(args.repo)
412
- elif args.history:
413
- # History mode: scan all files from git log
414
- history_files = get_history_files(args.repo)
415
- files = [] # We'll handle history differently
416
- else:
417
- files = scan_paths(args.paths)
418
-
419
- # Scan files
420
627
  all_findings = []
421
628
  fatal_findings = []
422
629
  allowed_doc_count = 0
630
+ file_count = 0
423
631
 
424
- if args.history:
425
- # History scanning mode
632
+ if args.staged:
633
+ # Scan the STAGED INDEX blob for each changed path, NOT the working-tree
634
+ # copy -- a worktree edit made after `git add` must not be able to hide
635
+ # a secret still sitting in the index that would actually get committed.
636
+ try:
637
+ relpaths = get_staged_files(args.repo)
638
+ except GitScanError as e:
639
+ print(f"FATAL: could not determine staged files: {e}", file=sys.stderr)
640
+ print("Failing CLOSED: refusing to report CLEAN when the staged-file listing could not be determined.", file=sys.stderr)
641
+ sys.exit(1)
642
+
643
+ file_count = len(relpaths)
644
+ for relpath in relpaths:
645
+ try:
646
+ content = get_git_blob(args.repo, f":{relpath}")
647
+ except GitScanError as e:
648
+ print(f"FATAL: could not read staged blob for {relpath}: {e}", file=sys.stderr)
649
+ print("Failing CLOSED: refusing to report CLEAN when a staged object cannot be read.", file=sys.stderr)
650
+ sys.exit(1)
651
+ label = str(Path(args.repo) / relpath)
652
+ findings = scan_blob(label, content)
653
+ for line_num, rule, match_str, is_fatal in findings:
654
+ all_findings.append((label, line_num, rule, match_str, is_fatal))
655
+ if is_fatal:
656
+ fatal_findings.append((label, line_num, rule, match_str))
657
+ else:
658
+ allowed_doc_count += 1
659
+
660
+ elif args.range:
661
+ # Scan the COMMITTED blob at the TIP of the range for each changed
662
+ # path, NOT the working-tree copy -- a secret committed then edited
663
+ # away in the worktree (without a new commit) must still be caught,
664
+ # since the pushed commit still carries the dirty blob.
665
+ try:
666
+ relpaths = get_range_files(args.repo, args.range)
667
+ except GitScanError as e:
668
+ print(f"FATAL: could not resolve commit range {args.range!r}: {e}", file=sys.stderr)
669
+ print("Failing CLOSED: refusing to report CLEAN on an unresolvable range.", file=sys.stderr)
670
+ sys.exit(1)
671
+
672
+ file_count = len(relpaths)
673
+ tip_ref = _range_tip_ref(args.range)
674
+ for relpath in relpaths:
675
+ try:
676
+ content = get_git_blob(args.repo, f"{tip_ref}:{relpath}")
677
+ except GitScanError as e:
678
+ print(f"FATAL: could not read blob at {tip_ref}:{relpath}: {e}", file=sys.stderr)
679
+ print("Failing CLOSED: refusing to report CLEAN when a committed object cannot be read.", file=sys.stderr)
680
+ sys.exit(1)
681
+ label = str(Path(args.repo) / relpath)
682
+ findings = scan_blob(label, content)
683
+ for line_num, rule, match_str, is_fatal in findings:
684
+ all_findings.append((label, line_num, rule, match_str, is_fatal))
685
+ if is_fatal:
686
+ fatal_findings.append((label, line_num, rule, match_str))
687
+ else:
688
+ allowed_doc_count += 1
689
+
690
+ elif args.history:
691
+ # History scanning mode: unaffected by the blob-scan fix above, since
692
+ # it already walks committed diff content via `git log -p`.
693
+ history_files = get_history_files(args.repo)
694
+ file_count = len(set(f for f, _ in history_files))
426
695
  for filepath, content in history_files:
427
696
  findings = scan_content(content)
428
697
  for line_num, rule, match_str, is_fatal in findings:
429
698
  all_findings.append((filepath, line_num, rule, match_str, is_fatal))
430
699
  if is_fatal:
431
700
  fatal_findings.append((filepath, line_num, rule, match_str))
701
+
432
702
  else:
433
- # Regular file scanning mode
703
+ files = scan_paths(args.paths)
704
+ file_count = len(files)
434
705
  for filepath in files:
435
706
  findings = scan_file(filepath)
436
707
  for line_num, rule, match_str, is_fatal in findings:
@@ -449,8 +720,6 @@ def main():
449
720
  print(f"ALLOWED-DOC {filepath}:{line_num} {rule} ({masked})")
450
721
 
451
722
  # Summary and exit
452
- file_count = len(files) if not args.history else len(set(f for f, _, _, _, _ in all_findings))
453
-
454
723
  if len(fatal_findings) == 0:
455
724
  if allowed_doc_count == 0:
456
725
  if args.history: