@matt82198/aesop 0.1.0-beta.5 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +158 -5
- package/LICENSE +66 -21
- package/README.md +97 -33
- package/aesop.config.example.json +6 -0
- package/bin/CLAUDE.md +12 -3
- package/bin/cli.js +211 -40
- package/daemons/CLAUDE.md +1 -1
- package/daemons/backup-fleet.sh +209 -51
- package/daemons/run-watchdog.sh +208 -62
- package/dash/dash-extra.mjs +73 -4
- package/dash/watchdog-gui.sh +41 -35
- package/docs/ARCHITECTURE.md +296 -0
- package/docs/CONCEPTS.md +254 -0
- package/docs/CONFIGURE.md +256 -0
- package/docs/FIRST-WAVE.md +276 -0
- package/docs/INSTALL.md +224 -0
- package/docs/README.md +176 -27
- package/docs/autonomous-swe.md +149 -0
- package/docs/reproduce.md +121 -0
- package/hooks/CLAUDE.md +28 -0
- package/hooks/install-waveguard.sh +68 -0
- package/hooks/pre-commit-waveguard.sh +26 -0
- package/hooks/pre-push-policy.sh +253 -15
- package/monitor/CLAUDE.md +3 -3
- package/monitor/collect-signals.mjs +133 -20
- package/package.json +15 -7
- package/scan/CLAUDE.md +30 -0
- package/skills/CLAUDE.md +1 -0
- package/state_store/ingest.py +18 -1
- package/state_store/projections.py +78 -4
- package/state_store/store.py +102 -7
- package/tools/CLAUDE.md +25 -18
- package/tools/alert_bridge.py +14 -10
- package/tools/bench_runner.py +438 -0
- package/tools/buildlog.py +4 -7
- package/tools/ci_merge_wait.py +215 -34
- package/tools/common.py +62 -0
- package/tools/cost_ceiling.py +191 -0
- package/tools/dash.js +102 -0
- package/tools/doctor.js +220 -0
- package/tools/fleet_ledger.py +189 -17
- package/tools/halt.py +172 -0
- package/tools/healthcheck.py +24 -21
- package/tools/heartbeat.py +4 -7
- package/tools/metrics_gate.py +8 -2
- package/tools/orchestrator_status.py +4 -9
- package/tools/reconcile.py +277 -0
- package/tools/rotate_logs.py +152 -62
- package/tools/scanner_selftest.py +28 -3
- package/tools/secret_scan.py +359 -90
- package/tools/self_stats.py +292 -4
- package/tools/status.js +187 -0
- package/tools/verify_agent_inspector.py +289 -0
- package/tools/verify_prboard.py +344 -0
- package/tools/watch.js +49 -0
- package/ui/CLAUDE.md +2 -0
- package/ui/agents.py +332 -33
- package/ui/api/tracker.py +15 -7
- package/ui/collectors.py +125 -32
- package/ui/config.py +22 -2
- package/ui/cost.py +98 -3
- package/ui/csrf.py +2 -1
- package/ui/handler.py +209 -21
- package/ui/sse.py +102 -16
- package/ui/wave_prs.py +230 -0
- package/ui/web/dist/assets/index-0qQYnvMC.js +9 -0
- package/ui/web/dist/assets/index-BdIlFieV.css +1 -0
- package/ui/web/dist/index.html +2 -2
- package/monitor/.signal-state.json +0 -3
- package/monitor/ACTIONS.log +0 -1
- package/monitor/BRIEF.md +0 -24
- package/monitor/SIGNALS.json +0 -54
- package/state_store/__pycache__/__init__.cpython-314.pyc +0 -0
- package/state_store/__pycache__/api.cpython-314.pyc +0 -0
- package/state_store/__pycache__/export.cpython-314.pyc +0 -0
- package/state_store/__pycache__/ingest.cpython-314.pyc +0 -0
- package/state_store/__pycache__/projections.cpython-314.pyc +0 -0
- package/state_store/__pycache__/store.cpython-314.pyc +0 -0
- package/tools/__pycache__/alert_bridge.cpython-314.pyc +0 -0
- package/tools/__pycache__/buildlog.cpython-314.pyc +0 -0
- package/tools/__pycache__/ci_merge_wait.cpython-314.pyc +0 -0
- package/tools/__pycache__/ensure_state.cpython-314.pyc +0 -0
- package/tools/__pycache__/eod_sweep.cpython-314.pyc +0 -0
- package/tools/__pycache__/fleet_ledger.cpython-314.pyc +0 -0
- package/tools/__pycache__/fleet_prompt_extractor.cpython-314.pyc +0 -0
- package/tools/__pycache__/healthcheck.cpython-314.pyc +0 -0
- package/tools/__pycache__/heartbeat.cpython-314.pyc +0 -0
- package/tools/__pycache__/inbox_drain.cpython-314.pyc +0 -0
- package/tools/__pycache__/launch_tui.cpython-314.pyc +0 -0
- package/tools/__pycache__/metrics_gate.cpython-314.pyc +0 -0
- package/tools/__pycache__/orchestrator_status.cpython-314.pyc +0 -0
- package/tools/__pycache__/power_selftest.cpython-314.pyc +0 -0
- package/tools/__pycache__/prepublish_scan.cpython-314.pyc +0 -0
- package/tools/__pycache__/rotate_logs.cpython-314.pyc +0 -0
- package/tools/__pycache__/scanner_selftest.cpython-314.pyc +0 -0
- package/tools/__pycache__/secret_scan.cpython-314.pyc +0 -0
- package/tools/__pycache__/self_stats.cpython-314.pyc +0 -0
- package/tools/__pycache__/session_usage_summary.cpython-314.pyc +0 -0
- package/tools/__pycache__/stall_check.cpython-314.pyc +0 -0
- package/tools/__pycache__/transcript_replay.cpython-314.pyc +0 -0
- package/tools/__pycache__/transcript_timeline.cpython-314.pyc +0 -0
- package/tools/__pycache__/verify_dash.cpython-314.pyc +0 -0
- package/tools/__pycache__/verify_submit_encoding.cpython-314.pyc +0 -0
- package/ui/__pycache__/agents.cpython-314.pyc +0 -0
- package/ui/__pycache__/collectors.cpython-314.pyc +0 -0
- package/ui/__pycache__/config.cpython-314.pyc +0 -0
- package/ui/__pycache__/cost.cpython-314.pyc +0 -0
- package/ui/__pycache__/csrf.cpython-314.pyc +0 -0
- package/ui/__pycache__/handler.cpython-314.pyc +0 -0
- package/ui/__pycache__/render.cpython-314.pyc +0 -0
- package/ui/__pycache__/serve.cpython-314.pyc +0 -0
- package/ui/__pycache__/sse.cpython-314.pyc +0 -0
- package/ui/api/__pycache__/__init__.cpython-314.pyc +0 -0
- package/ui/api/__pycache__/submit.cpython-314.pyc +0 -0
- package/ui/api/__pycache__/tracker.cpython-314.pyc +0 -0
- package/ui/web/.gitattributes +0 -13
- package/ui/web/dist/assets/index-2LZDQirC.js +0 -9
- package/ui/web/dist/assets/index-D4M1qyOv.css +0 -1
- package/ui/web/index.html +0 -13
- package/ui/web/package-lock.json +0 -2225
- package/ui/web/package.json +0 -26
- package/ui/web/src/App.test.tsx +0 -74
- package/ui/web/src/App.tsx +0 -142
- package/ui/web/src/CONTRIBUTING-UI.md +0 -49
- package/ui/web/src/components/AgentRow.css +0 -187
- package/ui/web/src/components/AgentRow.test.tsx +0 -209
- package/ui/web/src/components/AgentRow.tsx +0 -207
- package/ui/web/src/components/AgentsPanel.css +0 -108
- package/ui/web/src/components/AgentsPanel.test.tsx +0 -41
- package/ui/web/src/components/AgentsPanel.tsx +0 -58
- package/ui/web/src/components/AlertsPanel.css +0 -88
- package/ui/web/src/components/AlertsPanel.test.tsx +0 -51
- package/ui/web/src/components/AlertsPanel.tsx +0 -67
- package/ui/web/src/components/BacklogPanel.test.tsx +0 -126
- package/ui/web/src/components/BacklogPanel.tsx +0 -122
- package/ui/web/src/components/CostChart.css +0 -110
- package/ui/web/src/components/CostChart.test.tsx +0 -144
- package/ui/web/src/components/CostChart.tsx +0 -152
- package/ui/web/src/components/CostTable.css +0 -93
- package/ui/web/src/components/CostTable.test.tsx +0 -165
- package/ui/web/src/components/CostTable.tsx +0 -94
- package/ui/web/src/components/EventsFeed.css +0 -68
- package/ui/web/src/components/EventsFeed.test.tsx +0 -36
- package/ui/web/src/components/EventsFeed.tsx +0 -31
- package/ui/web/src/components/HealthHeader.css +0 -137
- package/ui/web/src/components/HealthHeader.test.tsx +0 -278
- package/ui/web/src/components/HealthHeader.tsx +0 -281
- package/ui/web/src/components/InboxForm.css +0 -135
- package/ui/web/src/components/InboxForm.test.tsx +0 -208
- package/ui/web/src/components/InboxForm.tsx +0 -116
- package/ui/web/src/components/MessagesTail.module.css +0 -144
- package/ui/web/src/components/MessagesTail.test.tsx +0 -176
- package/ui/web/src/components/MessagesTail.tsx +0 -94
- package/ui/web/src/components/ReposPanel.css +0 -90
- package/ui/web/src/components/ReposPanel.test.tsx +0 -45
- package/ui/web/src/components/ReposPanel.tsx +0 -67
- package/ui/web/src/components/Scorecard.css +0 -106
- package/ui/web/src/components/Scorecard.test.tsx +0 -117
- package/ui/web/src/components/Scorecard.tsx +0 -85
- package/ui/web/src/components/Timeline.module.css +0 -151
- package/ui/web/src/components/Timeline.test.tsx +0 -215
- package/ui/web/src/components/Timeline.tsx +0 -99
- package/ui/web/src/components/TrackerBoard.test.tsx +0 -121
- package/ui/web/src/components/TrackerBoard.tsx +0 -107
- package/ui/web/src/components/TrackerCard.test.tsx +0 -180
- package/ui/web/src/components/TrackerCard.tsx +0 -160
- package/ui/web/src/components/TrackerForm.test.tsx +0 -189
- package/ui/web/src/components/TrackerForm.tsx +0 -144
- package/ui/web/src/lib/api.ts +0 -218
- package/ui/web/src/lib/format.test.ts +0 -89
- package/ui/web/src/lib/format.ts +0 -103
- package/ui/web/src/lib/sanitizeUrl.test.ts +0 -84
- package/ui/web/src/lib/sanitizeUrl.ts +0 -38
- package/ui/web/src/lib/types.ts +0 -230
- package/ui/web/src/lib/useHashRoute.test.ts +0 -60
- package/ui/web/src/lib/useHashRoute.ts +0 -23
- package/ui/web/src/lib/useSSE.ts +0 -175
- package/ui/web/src/main.tsx +0 -10
- package/ui/web/src/styles/global.css +0 -179
- package/ui/web/src/styles/theme.css +0 -184
- package/ui/web/src/styles/work.css +0 -572
- package/ui/web/src/test/fixtures.ts +0 -385
- package/ui/web/src/test/setup.ts +0 -49
- package/ui/web/src/views/Activity.module.css +0 -43
- package/ui/web/src/views/Activity.test.tsx +0 -89
- package/ui/web/src/views/Activity.tsx +0 -31
- package/ui/web/src/views/Cost.css +0 -87
- package/ui/web/src/views/Cost.test.tsx +0 -142
- package/ui/web/src/views/Cost.tsx +0 -54
- package/ui/web/src/views/Overview.css +0 -51
- package/ui/web/src/views/Overview.test.tsx +0 -76
- package/ui/web/src/views/Overview.tsx +0 -46
- package/ui/web/src/views/Work.test.tsx +0 -82
- package/ui/web/src/views/Work.tsx +0 -79
- package/ui/web/src/vite-env.d.ts +0 -10
- package/ui/web/tsconfig.json +0 -22
- package/ui/web/vite.config.ts +0 -25
- package/ui/web/vitest.config.ts +0 -12
|
@@ -158,8 +158,8 @@ def main():
|
|
|
158
158
|
(
|
|
159
159
|
# Doc-shaped rule: generic assignment IS softened by pragma.
|
|
160
160
|
"tp_pragma_generic_softened",
|
|
161
|
-
"# secretscan: allow-pattern-docs
|
|
162
|
-
+ "
|
|
161
|
+
"# secretscan: allow-pattern-docs\n"
|
|
162
|
+
+ "password = \"this_is_a_doc_sample_value_12345\"\n",
|
|
163
163
|
"pattern_docs.py",
|
|
164
164
|
0,
|
|
165
165
|
"ALLOWED-DOC"
|
|
@@ -235,7 +235,7 @@ def main():
|
|
|
235
235
|
"tp_connection_real_credentials",
|
|
236
236
|
"# Real production DB\\ndb_url = '"
|
|
237
237
|
+ "postgresql:" + "//produser:RealPassword123"
|
|
238
|
-
+ "@db.
|
|
238
|
+
+ "@db.production.io:5432/production" + "'\\n",
|
|
239
239
|
"config_prod.py",
|
|
240
240
|
1,
|
|
241
241
|
"connection_string"
|
|
@@ -266,6 +266,31 @@ def main():
|
|
|
266
266
|
0,
|
|
267
267
|
"SKIPPED-LARGE"
|
|
268
268
|
),
|
|
269
|
+
(
|
|
270
|
+
# REGRESSION: Unquoted password assignment should be detected
|
|
271
|
+
# (previously required quotes to match the pattern)
|
|
272
|
+
"tp_unquoted_password_assignment",
|
|
273
|
+
"password = supersecretvalue123\n",
|
|
274
|
+
"config.py",
|
|
275
|
+
1,
|
|
276
|
+
"generic_secret_assignment"
|
|
277
|
+
),
|
|
278
|
+
(
|
|
279
|
+
# REGRESSION: Unquoted API key assignment should be detected
|
|
280
|
+
"tp_unquoted_api_key_assignment",
|
|
281
|
+
"api_key = longsecretapikey1234567890\n",
|
|
282
|
+
"settings.py",
|
|
283
|
+
1,
|
|
284
|
+
"generic_secret_assignment"
|
|
285
|
+
),
|
|
286
|
+
(
|
|
287
|
+
# REGRESSION: Unquoted token in .env should be detected
|
|
288
|
+
"tp_unquoted_token_env_file",
|
|
289
|
+
"API_TOKEN=verylongtokenvalue123456789\n",
|
|
290
|
+
".env",
|
|
291
|
+
1,
|
|
292
|
+
"env_assignment"
|
|
293
|
+
),
|
|
269
294
|
]
|
|
270
295
|
|
|
271
296
|
results = []
|
package/tools/secret_scan.py
CHANGED
|
@@ -3,9 +3,10 @@
|
|
|
3
3
|
secret_scan.py — Pre-push secret/credential detection gate.
|
|
4
4
|
|
|
5
5
|
Modes:
|
|
6
|
-
secret_scan.py --staged [--repo PATH]
|
|
7
|
-
secret_scan.py --
|
|
8
|
-
secret_scan.py
|
|
6
|
+
secret_scan.py --staged [--repo PATH] Scan git staged files (default repo=cwd)
|
|
7
|
+
secret_scan.py --range COMMIT_RANGE [--repo PATH] Scan files changed in range (e.g., main..HEAD or abc123..def456)
|
|
8
|
+
secret_scan.py --history [--repo PATH] Scan all blobs in git history
|
|
9
|
+
secret_scan.py PATH [PATH...] Scan files/dirs directly (recurse dirs)
|
|
9
10
|
|
|
10
11
|
Exit codes: 0=clean, 1=findings, 2=usage error
|
|
11
12
|
Output: one line per finding or summary (never prints full secrets)
|
|
@@ -22,6 +23,19 @@ Self-scan invariant: this file must scan CLEAN with NO pragma. Any pattern liter
|
|
|
22
23
|
would match its own regex is runtime-assembled from fragments (see pem_private_key) so
|
|
23
24
|
the pattern text never appears contiguously in this source.
|
|
24
25
|
|
|
26
|
+
Scan contract (wave-25 P2 fix): --staged and --range scan committed GIT OBJECTS, not the
|
|
27
|
+
working-tree copy of changed files. --staged reads each path's STAGED INDEX blob
|
|
28
|
+
(`git show :<path>`); --range reads each changed path's blob at the TIP of the range
|
|
29
|
+
(`git show <tip>:<path>`). This closes two bypasses that existed when these modes read
|
|
30
|
+
$repo/<path> off disk: (1) stage a secret, then edit the on-disk file without
|
|
31
|
+
re-staging -- the dirty blob is still what gets committed/pushed; (2) commit a secret,
|
|
32
|
+
then edit it away in the worktree without a new commit -- the dirty blob is still what
|
|
33
|
+
the pushed commit carries. --history is unaffected (it already walks committed diffs).
|
|
34
|
+
Any git command needed to enumerate the files-to-scan that itself fails (bad ref,
|
|
35
|
+
unresolvable range, git error) is NOT treated as "zero files changed" -- it raises
|
|
36
|
+
GitScanError and the caller fails CLOSED (non-zero exit), so an unresolvable range can
|
|
37
|
+
never silently report CLEAN.
|
|
38
|
+
|
|
25
39
|
NOTE: Public repo version has NO vault allowlist.
|
|
26
40
|
"""
|
|
27
41
|
|
|
@@ -50,10 +64,30 @@ PATTERNS = {
|
|
|
50
64
|
"slack_token": (r"xox[baprs]-[A-Za-z0-9-]{10,}", 0),
|
|
51
65
|
"openai_anthropic_key": (r"sk-[A-Za-z0-9_\-]{20,}", 0),
|
|
52
66
|
"generic_secret_assignment": (
|
|
53
|
-
|
|
67
|
+
# Unquoted branch requires 8+ contiguous chars from a bare-literal charset
|
|
68
|
+
# (no '.', '(', ')' — real code expressions like `secrets.token_urlsafe(32)`,
|
|
69
|
+
# `self.serve.SESSION_TOKEN`, or `headers.get(...)` all contain one of those)
|
|
70
|
+
# followed by whitespace/EOL, so an assignment from a variable/attribute/call
|
|
71
|
+
# doesn't false-positive as a hardcoded secret the way a bare env-file value
|
|
72
|
+
# (`API_TOKEN=verylongtokenvalue123456789`) legitimately does.
|
|
73
|
+
r"\b(password|passwd|secret|api[_-]?key|token|authorization)\b\s*(?::=|=)\s*(?:[\"'](?!.*(?:xxx|changeme|your-|<|$\{|example)\b).{8,}[\"']|(?!['\"]|xxx|changeme|your-|example)[^\s\$\<\{\n\(\)\.\"']{8,}(?=\s|$))",
|
|
54
74
|
re.IGNORECASE,
|
|
55
75
|
),
|
|
56
|
-
"connection_string": (
|
|
76
|
+
"connection_string": (
|
|
77
|
+
(r"://"
|
|
78
|
+
r"[^:]+:[^@/\s]+@"
|
|
79
|
+
r"(?!"
|
|
80
|
+
r"localhost(?:[:/]|$)|"
|
|
81
|
+
r"127\.0\.0\.1(?:[:/]|$)|"
|
|
82
|
+
r"127\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(?:[:/]|$)|"
|
|
83
|
+
r"[a-zA-Z0-9.-]*\.local(?:[:/]|$)|"
|
|
84
|
+
r"[a-zA-Z0-9.-]*\.localdomain(?:[:/]|$)|"
|
|
85
|
+
r"(?:[a-zA-Z0-9-]+\.)*example(?:\.[a-zA-Z]{2,})?(?:[:/]|$)|"
|
|
86
|
+
r"(?:[a-zA-Z0-9-]+\.)*test(?:\.[a-zA-Z]{2,})?(?:[:/]|$)"
|
|
87
|
+
r")"
|
|
88
|
+
r"[^\s]+"),
|
|
89
|
+
0
|
|
90
|
+
),
|
|
57
91
|
"env_access": (
|
|
58
92
|
r"(?i:(?:os\.getenv|os\.environ|System\.getenv|process\.env)\s*[\[\(][\"']?[A-Z_]*(?:password|secret|api[_-]?key|token|auth|key)[A-Z_0-9]*[\"']?[\)\]])",
|
|
59
93
|
0,
|
|
@@ -76,6 +110,40 @@ CREDENTIAL_FILENAMES = [
|
|
|
76
110
|
# Placeholders that don't count as secrets
|
|
77
111
|
PLACEHOLDERS = {"xxx", "changeme", "your-key-here", "example", "test", "demo"}
|
|
78
112
|
|
|
113
|
+
# Rules that CAN be softened by pragma (doc-shaped rules only)
|
|
114
|
+
SOFTENED_BY_PRAGMA = {"generic_secret_assignment", "env_access"}
|
|
115
|
+
|
|
116
|
+
# Rules that are ALWAYS fatal, pragma never applies
|
|
117
|
+
FATAL_RULES = {
|
|
118
|
+
"pem_private_key",
|
|
119
|
+
"aws_access_key",
|
|
120
|
+
"github_token",
|
|
121
|
+
"slack_token",
|
|
122
|
+
"openai_anthropic_key",
|
|
123
|
+
"connection_string",
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _classify_finding(rule_name, has_file_pragma):
|
|
128
|
+
"""Shared fatal/softened decision, used by every scan_* variant (disk-file,
|
|
129
|
+
blob, large-file-chunked) so the pragma contract can't drift between them.
|
|
130
|
+
FATAL_RULES are always fatal. SOFTENED_BY_PRAGMA rules are fatal unless the
|
|
131
|
+
pragma is present. Everything else (e.g. env_assignment) is always fatal."""
|
|
132
|
+
if rule_name in FATAL_RULES:
|
|
133
|
+
return True
|
|
134
|
+
if rule_name in SOFTENED_BY_PRAGMA:
|
|
135
|
+
return not has_file_pragma
|
|
136
|
+
return True
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
class GitScanError(Exception):
|
|
140
|
+
"""Raised when a git command needed to enumerate files-to-scan fails.
|
|
141
|
+
Callers MUST treat this as fail-CLOSED (block / exit non-zero) -- it is
|
|
142
|
+
NOT equivalent to "the range/index genuinely contains zero changed
|
|
143
|
+
files" (wave-25 P2 fix: get_range_files previously returned [] on ANY
|
|
144
|
+
git-diff error, indistinguishable from a real empty range, so main()
|
|
145
|
+
printed CLEAN and exited 0 on an unresolvable range)."""
|
|
146
|
+
|
|
79
147
|
|
|
80
148
|
def has_pragma(filepath):
|
|
81
149
|
"""Check if file has 'secretscan: allow-pattern-docs' pragma in first 10 lines."""
|
|
@@ -163,22 +231,8 @@ def scan_file(filepath):
|
|
|
163
231
|
- Large files: scan first 1MB; emit SKIPPED-LARGE to stderr if file is larger
|
|
164
232
|
- Binary files: decode as latin-1; emit SKIPPED-BINARY to stderr if not fully scanned
|
|
165
233
|
"""
|
|
166
|
-
# Rules that CAN be softened by pragma (doc-shaped rules only)
|
|
167
|
-
SOFTENED_BY_PRAGMA = {"generic_secret_assignment", "env_access"}
|
|
168
|
-
|
|
169
|
-
# Rules that are ALWAYS fatal, pragma never applies
|
|
170
|
-
FATAL_RULES = {
|
|
171
|
-
"pem_private_key",
|
|
172
|
-
"aws_access_key",
|
|
173
|
-
"github_token",
|
|
174
|
-
"slack_token",
|
|
175
|
-
"openai_anthropic_key",
|
|
176
|
-
"connection_string",
|
|
177
|
-
}
|
|
178
|
-
|
|
179
234
|
SIZE_THRESHOLD = 1024 * 1024 # 1MB
|
|
180
235
|
MAX_READ_SIZE = 2 * 1024 * 1024 # 2MB max to read
|
|
181
|
-
SCAN_PREFIX_SIZE = 1024 * 1024 # Scan first 1MB of large files
|
|
182
236
|
|
|
183
237
|
findings = []
|
|
184
238
|
|
|
@@ -204,35 +258,8 @@ def scan_file(filepath):
|
|
|
204
258
|
is_binary = is_binary_file(filepath)
|
|
205
259
|
is_large = file_size > SIZE_THRESHOLD
|
|
206
260
|
|
|
207
|
-
if
|
|
208
|
-
#
|
|
209
|
-
with open(filepath, "rb") as f:
|
|
210
|
-
content = f.read(SCAN_PREFIX_SIZE)
|
|
211
|
-
|
|
212
|
-
# Decode as latin-1 (handles any binary content)
|
|
213
|
-
try:
|
|
214
|
-
content_str = content.decode("latin-1")
|
|
215
|
-
except Exception:
|
|
216
|
-
content_str = content.decode("utf-8", errors="ignore")
|
|
217
|
-
|
|
218
|
-
# Emit skip notice to stderr
|
|
219
|
-
print(f"SKIPPED-LARGE {filepath} (scanned first {SCAN_PREFIX_SIZE // 1024}KB)", file=sys.stderr)
|
|
220
|
-
|
|
221
|
-
# Scan content for FATAL_RULES only
|
|
222
|
-
for line_num, line in enumerate(content_str.split("\n"), start=1):
|
|
223
|
-
for rule_name in FATAL_RULES:
|
|
224
|
-
if rule_name not in PATTERNS:
|
|
225
|
-
continue
|
|
226
|
-
pattern, flags = PATTERNS[rule_name]
|
|
227
|
-
matches = re.finditer(pattern, line, flags)
|
|
228
|
-
for match in matches:
|
|
229
|
-
match_str = match.group(0)
|
|
230
|
-
if is_placeholder(match_str):
|
|
231
|
-
continue
|
|
232
|
-
findings.append((line_num, rule_name, match_str, True))
|
|
233
|
-
|
|
234
|
-
elif is_binary:
|
|
235
|
-
# Scan binary file as latin-1 for FATAL_RULES only
|
|
261
|
+
if is_binary:
|
|
262
|
+
# Binary file (any size): scan as-is for FATAL_RULES only
|
|
236
263
|
with open(filepath, "rb") as f:
|
|
237
264
|
content = f.read(MAX_READ_SIZE)
|
|
238
265
|
|
|
@@ -258,8 +285,38 @@ def scan_file(filepath):
|
|
|
258
285
|
continue
|
|
259
286
|
findings.append((line_num, rule_name, match_str, True))
|
|
260
287
|
|
|
288
|
+
elif is_large:
|
|
289
|
+
# Large text file: scan entire file in chunks for all rules
|
|
290
|
+
print(f"SKIPPED-LARGE {filepath} (scanned in chunks)", file=sys.stderr)
|
|
291
|
+
line_num = 0
|
|
292
|
+
try:
|
|
293
|
+
with open(filepath, "r", encoding="utf-8", errors="ignore") as f:
|
|
294
|
+
# Read in 1MB chunks to avoid loading entire large file into memory
|
|
295
|
+
for chunk in iter(lambda: f.read(1024 * 1024), ""):
|
|
296
|
+
for chunk_line in chunk.split("\n"):
|
|
297
|
+
line_num += 1
|
|
298
|
+
for rule_name, (pattern, flags) in PATTERNS.items():
|
|
299
|
+
# Skip env_assignment rule if not an .env-like file
|
|
300
|
+
if rule_name == "env_assignment" and not is_env_file(filepath):
|
|
301
|
+
continue
|
|
302
|
+
|
|
303
|
+
matches = re.finditer(pattern, chunk_line, flags)
|
|
304
|
+
for match in matches:
|
|
305
|
+
match_str = match.group(0)
|
|
306
|
+
|
|
307
|
+
# Skip if it's a placeholder
|
|
308
|
+
if is_placeholder(match_str):
|
|
309
|
+
continue
|
|
310
|
+
|
|
311
|
+
is_fatal = _classify_finding(rule_name, has_file_pragma)
|
|
312
|
+
findings.append((line_num, rule_name, match_str, is_fatal))
|
|
313
|
+
except (IOError, OSError) as e:
|
|
314
|
+
# FAIL CLOSED: if we cannot fully scan a large text file, exit with error
|
|
315
|
+
print(f"FATAL: Cannot fully scan large text file {filepath}: {e}", file=sys.stderr)
|
|
316
|
+
sys.exit(1)
|
|
317
|
+
|
|
261
318
|
else:
|
|
262
|
-
# Normal text file: scan all rules
|
|
319
|
+
# Normal small text file: scan all rules
|
|
263
320
|
with open(filepath, "r", encoding="utf-8", errors="ignore") as f:
|
|
264
321
|
for line_num, line in enumerate(f, start=1):
|
|
265
322
|
for rule_name, (pattern, flags) in PATTERNS.items():
|
|
@@ -275,17 +332,7 @@ def scan_file(filepath):
|
|
|
275
332
|
if is_placeholder(match_str):
|
|
276
333
|
continue
|
|
277
334
|
|
|
278
|
-
|
|
279
|
-
if rule_name in FATAL_RULES:
|
|
280
|
-
# These are always fatal, pragma never applies
|
|
281
|
-
is_fatal = True
|
|
282
|
-
elif has_file_pragma and rule_name in SOFTENED_BY_PRAGMA:
|
|
283
|
-
# Only these rules can be softened by pragma
|
|
284
|
-
is_fatal = False
|
|
285
|
-
else:
|
|
286
|
-
# Other rules are fatal unless pragma and softened
|
|
287
|
-
is_fatal = not has_file_pragma if rule_name in SOFTENED_BY_PRAGMA else True
|
|
288
|
-
|
|
335
|
+
is_fatal = _classify_finding(rule_name, has_file_pragma)
|
|
289
336
|
findings.append((line_num, rule_name, match_str, is_fatal))
|
|
290
337
|
|
|
291
338
|
except Exception:
|
|
@@ -294,25 +341,190 @@ def scan_file(filepath):
|
|
|
294
341
|
return findings
|
|
295
342
|
|
|
296
343
|
|
|
344
|
+
def scan_blob(label, content_bytes):
|
|
345
|
+
"""
|
|
346
|
+
Scan raw blob bytes for secrets, reporting findings against `label` as the
|
|
347
|
+
display path. Mirrors scan_file()'s rule/fatality logic, but reads content
|
|
348
|
+
already fetched from a git object (via get_git_blob) instead of opening the
|
|
349
|
+
working-tree disk file. Used by --staged (the STAGED INDEX blob) and
|
|
350
|
+
--range (the blob at the TIP of the range) so a secret present only in the
|
|
351
|
+
git object being pushed -- but absent from an out-of-sync working copy --
|
|
352
|
+
is not missed (wave-25 P2 fix; see module docstring "Scan contract").
|
|
353
|
+
|
|
354
|
+
Returns list of (line_num, rule, match_str, is_fatal), like scan_file().
|
|
355
|
+
"""
|
|
356
|
+
filepath = Path(label)
|
|
357
|
+
SIZE_THRESHOLD = 1024 * 1024 # 1MB
|
|
358
|
+
MAX_READ_SIZE = 2 * 1024 * 1024 # 2MB max to read (matches scan_file's binary cap)
|
|
359
|
+
|
|
360
|
+
findings = []
|
|
361
|
+
|
|
362
|
+
if should_skip_file(filepath):
|
|
363
|
+
return findings
|
|
364
|
+
|
|
365
|
+
filename = filepath.name.lower()
|
|
366
|
+
for pattern in CREDENTIAL_FILENAMES:
|
|
367
|
+
if re.match(pattern, filename, re.IGNORECASE):
|
|
368
|
+
findings.append(
|
|
369
|
+
(0, "credential_filename", f"File name matches credential pattern: {filepath.name}", True)
|
|
370
|
+
)
|
|
371
|
+
break
|
|
372
|
+
|
|
373
|
+
# Pragma check over the first 10 lines, mirroring has_pragma() but reading
|
|
374
|
+
# from the in-memory blob instead of opening a path from disk.
|
|
375
|
+
has_file_pragma = False
|
|
376
|
+
try:
|
|
377
|
+
probe = content_bytes.decode("utf-8", errors="ignore")
|
|
378
|
+
for i, line in enumerate(probe.split("\n")):
|
|
379
|
+
if i >= 10:
|
|
380
|
+
break
|
|
381
|
+
if "secretscan: allow-pattern-docs" in line:
|
|
382
|
+
has_file_pragma = True
|
|
383
|
+
break
|
|
384
|
+
except Exception:
|
|
385
|
+
pass
|
|
386
|
+
|
|
387
|
+
is_binary = b"\x00" in content_bytes[:8192]
|
|
388
|
+
is_large = len(content_bytes) > SIZE_THRESHOLD
|
|
389
|
+
|
|
390
|
+
if is_binary:
|
|
391
|
+
raw = content_bytes[:MAX_READ_SIZE]
|
|
392
|
+
try:
|
|
393
|
+
content_str = raw.decode("latin-1")
|
|
394
|
+
except Exception:
|
|
395
|
+
content_str = raw.decode("utf-8", errors="ignore")
|
|
396
|
+
|
|
397
|
+
print(f"SKIPPED-BINARY {filepath} (scanned via latin-1)", file=sys.stderr)
|
|
398
|
+
|
|
399
|
+
for line_num, line in enumerate(content_str.split("\n"), start=1):
|
|
400
|
+
for rule_name in FATAL_RULES:
|
|
401
|
+
pattern, flags = PATTERNS[rule_name]
|
|
402
|
+
for match in re.finditer(pattern, line, flags):
|
|
403
|
+
match_str = match.group(0)
|
|
404
|
+
if is_placeholder(match_str):
|
|
405
|
+
continue
|
|
406
|
+
findings.append((line_num, rule_name, match_str, True))
|
|
407
|
+
return findings
|
|
408
|
+
|
|
409
|
+
if is_large:
|
|
410
|
+
print(f"SKIPPED-LARGE {filepath} (scanned in full from git object)", file=sys.stderr)
|
|
411
|
+
|
|
412
|
+
try:
|
|
413
|
+
content_str = content_bytes.decode("utf-8", errors="ignore")
|
|
414
|
+
except Exception:
|
|
415
|
+
content_str = content_bytes.decode("latin-1", errors="ignore")
|
|
416
|
+
|
|
417
|
+
env_file = is_env_file(filepath)
|
|
418
|
+
for line_num, line in enumerate(content_str.split("\n"), start=1):
|
|
419
|
+
for rule_name, (pattern, flags) in PATTERNS.items():
|
|
420
|
+
if rule_name == "env_assignment" and not env_file:
|
|
421
|
+
continue
|
|
422
|
+
|
|
423
|
+
for match in re.finditer(pattern, line, flags):
|
|
424
|
+
match_str = match.group(0)
|
|
425
|
+
if is_placeholder(match_str):
|
|
426
|
+
continue
|
|
427
|
+
is_fatal = _classify_finding(rule_name, has_file_pragma)
|
|
428
|
+
findings.append((line_num, rule_name, match_str, is_fatal))
|
|
429
|
+
|
|
430
|
+
return findings
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def _range_tip_ref(commit_range):
|
|
434
|
+
"""Extract the right-hand (tip) ref from a two-dot or three-dot commit
|
|
435
|
+
range string, e.g. 'main..HEAD' -> 'HEAD', 'a...b' -> 'b'. Falls back to
|
|
436
|
+
the whole string if it isn't a recognizable range (single ref/sha)."""
|
|
437
|
+
if "..." in commit_range:
|
|
438
|
+
tip = commit_range.split("...", 1)[1]
|
|
439
|
+
elif ".." in commit_range:
|
|
440
|
+
tip = commit_range.split("..", 1)[1]
|
|
441
|
+
else:
|
|
442
|
+
tip = commit_range
|
|
443
|
+
return tip or "HEAD"
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
def get_git_blob(repo_path, ref_path):
|
|
447
|
+
"""Fetch raw blob bytes via `git show <ref_path>` -- e.g. ref_path=':foo.py'
|
|
448
|
+
reads foo.py from the STAGED INDEX, 'abc123:foo.py' reads foo.py as it
|
|
449
|
+
exists in commit abc123.
|
|
450
|
+
|
|
451
|
+
Raises GitScanError if the git show command fails (non-zero returncode or
|
|
452
|
+
exception). Since callers use --diff-filter=d to exclude deleted paths from
|
|
453
|
+
enumeration, a git-show failure for an enumerated path is a real error
|
|
454
|
+
(e.g., corruption, permissions, unreadable blob), not a legitimate 'file
|
|
455
|
+
absent' case. Failing closed (raising GitScanError) ensures such errors
|
|
456
|
+
block the push, consistent with how enumeration failures already fail closed.
|
|
457
|
+
"""
|
|
458
|
+
try:
|
|
459
|
+
result = subprocess.run(
|
|
460
|
+
["git", "show", ref_path],
|
|
461
|
+
cwd=repo_path,
|
|
462
|
+
capture_output=True,
|
|
463
|
+
timeout=15,
|
|
464
|
+
)
|
|
465
|
+
except Exception as e:
|
|
466
|
+
raise GitScanError(
|
|
467
|
+
f"git show {ref_path!r} raised {e!r}"
|
|
468
|
+
)
|
|
469
|
+
if result.returncode != 0:
|
|
470
|
+
raise GitScanError(
|
|
471
|
+
f"git show {ref_path!r} failed (rc={result.returncode}): "
|
|
472
|
+
f"{result.stderr.decode('utf-8', errors='ignore').strip()}"
|
|
473
|
+
)
|
|
474
|
+
return result.stdout
|
|
475
|
+
|
|
476
|
+
|
|
297
477
|
def get_staged_files(repo_path):
|
|
298
|
-
"""Get list of staged
|
|
478
|
+
"""Get list of staged file paths (relative to repo root) from git repo.
|
|
479
|
+
|
|
480
|
+
Raises GitScanError if the underlying `git diff --cached` invocation
|
|
481
|
+
fails; callers must fail CLOSED on that, not treat it as "nothing staged".
|
|
482
|
+
"""
|
|
299
483
|
try:
|
|
300
484
|
result = subprocess.run(
|
|
301
|
-
["git", "diff", "--cached", "--name-only"],
|
|
485
|
+
["git", "diff", "--cached", "--name-only", "--diff-filter=d"],
|
|
302
486
|
cwd=repo_path,
|
|
303
487
|
capture_output=True,
|
|
304
488
|
text=True,
|
|
305
489
|
timeout=10,
|
|
306
490
|
)
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
491
|
+
except Exception as e:
|
|
492
|
+
raise GitScanError(f"git diff --cached raised {e!r}")
|
|
493
|
+
|
|
494
|
+
if result.returncode != 0:
|
|
495
|
+
raise GitScanError(
|
|
496
|
+
f"git diff --cached failed (rc={result.returncode}): {result.stderr.strip()}"
|
|
497
|
+
)
|
|
498
|
+
|
|
499
|
+
return [f for f in result.stdout.strip().split("\n") if f.strip()]
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
def get_range_files(repo_path, commit_range):
|
|
503
|
+
"""Get list of file paths (relative to repo root) changed in commit range
|
|
504
|
+
(e.g. 'main..HEAD' or 'abc123..def456').
|
|
505
|
+
|
|
506
|
+
Raises GitScanError if the underlying `git diff` invocation fails (e.g.
|
|
507
|
+
an unresolvable ref); callers must fail CLOSED on that -- it is not the
|
|
508
|
+
same as a range that genuinely touches zero files.
|
|
509
|
+
"""
|
|
510
|
+
try:
|
|
511
|
+
result = subprocess.run(
|
|
512
|
+
["git", "diff", "--name-only", "--diff-filter=d", commit_range],
|
|
513
|
+
cwd=repo_path,
|
|
514
|
+
capture_output=True,
|
|
515
|
+
text=True,
|
|
516
|
+
timeout=10,
|
|
517
|
+
)
|
|
518
|
+
except Exception as e:
|
|
519
|
+
raise GitScanError(f"git diff --name-only {commit_range!r} raised {e!r}")
|
|
520
|
+
|
|
521
|
+
if result.returncode != 0:
|
|
522
|
+
raise GitScanError(
|
|
523
|
+
f"git diff --name-only {commit_range!r} failed (rc={result.returncode}): "
|
|
524
|
+
f"{result.stderr.strip()}"
|
|
525
|
+
)
|
|
526
|
+
|
|
527
|
+
return [f for f in result.stdout.strip().split("\n") if f.strip()]
|
|
316
528
|
|
|
317
529
|
|
|
318
530
|
def get_history_files(repo_path):
|
|
@@ -385,6 +597,11 @@ def main():
|
|
|
385
597
|
action="store_true",
|
|
386
598
|
help="Scan git staged files (requires --repo or uses cwd)",
|
|
387
599
|
)
|
|
600
|
+
parser.add_argument(
|
|
601
|
+
"--range",
|
|
602
|
+
metavar="COMMIT_RANGE",
|
|
603
|
+
help="Scan files changed in commit range (e.g., 'main..HEAD' or 'abc123..def456')",
|
|
604
|
+
)
|
|
388
605
|
parser.add_argument(
|
|
389
606
|
"--history",
|
|
390
607
|
action="store_true",
|
|
@@ -401,36 +618,90 @@ def main():
|
|
|
401
618
|
|
|
402
619
|
args = parser.parse_args()
|
|
403
620
|
|
|
404
|
-
# Validate usage
|
|
405
|
-
|
|
406
|
-
|
|
621
|
+
# Validate usage: exactly one of --staged, --range, --history, or paths
|
|
622
|
+
mode_count = sum([args.staged, bool(args.range), args.history, bool(args.paths)])
|
|
623
|
+
if mode_count != 1:
|
|
624
|
+
print("ERROR: Use exactly one of --staged, --range, --history, or path arguments", file=sys.stderr)
|
|
407
625
|
sys.exit(2)
|
|
408
626
|
|
|
409
|
-
# Collect files to scan
|
|
410
|
-
if args.staged:
|
|
411
|
-
files = get_staged_files(args.repo)
|
|
412
|
-
elif args.history:
|
|
413
|
-
# History mode: scan all files from git log
|
|
414
|
-
history_files = get_history_files(args.repo)
|
|
415
|
-
files = [] # We'll handle history differently
|
|
416
|
-
else:
|
|
417
|
-
files = scan_paths(args.paths)
|
|
418
|
-
|
|
419
|
-
# Scan files
|
|
420
627
|
all_findings = []
|
|
421
628
|
fatal_findings = []
|
|
422
629
|
allowed_doc_count = 0
|
|
630
|
+
file_count = 0
|
|
423
631
|
|
|
424
|
-
if args.
|
|
425
|
-
#
|
|
632
|
+
if args.staged:
|
|
633
|
+
# Scan the STAGED INDEX blob for each changed path, NOT the working-tree
|
|
634
|
+
# copy -- a worktree edit made after `git add` must not be able to hide
|
|
635
|
+
# a secret still sitting in the index that would actually get committed.
|
|
636
|
+
try:
|
|
637
|
+
relpaths = get_staged_files(args.repo)
|
|
638
|
+
except GitScanError as e:
|
|
639
|
+
print(f"FATAL: could not determine staged files: {e}", file=sys.stderr)
|
|
640
|
+
print("Failing CLOSED: refusing to report CLEAN when the staged-file listing could not be determined.", file=sys.stderr)
|
|
641
|
+
sys.exit(1)
|
|
642
|
+
|
|
643
|
+
file_count = len(relpaths)
|
|
644
|
+
for relpath in relpaths:
|
|
645
|
+
try:
|
|
646
|
+
content = get_git_blob(args.repo, f":{relpath}")
|
|
647
|
+
except GitScanError as e:
|
|
648
|
+
print(f"FATAL: could not read staged blob for {relpath}: {e}", file=sys.stderr)
|
|
649
|
+
print("Failing CLOSED: refusing to report CLEAN when a staged object cannot be read.", file=sys.stderr)
|
|
650
|
+
sys.exit(1)
|
|
651
|
+
label = str(Path(args.repo) / relpath)
|
|
652
|
+
findings = scan_blob(label, content)
|
|
653
|
+
for line_num, rule, match_str, is_fatal in findings:
|
|
654
|
+
all_findings.append((label, line_num, rule, match_str, is_fatal))
|
|
655
|
+
if is_fatal:
|
|
656
|
+
fatal_findings.append((label, line_num, rule, match_str))
|
|
657
|
+
else:
|
|
658
|
+
allowed_doc_count += 1
|
|
659
|
+
|
|
660
|
+
elif args.range:
|
|
661
|
+
# Scan the COMMITTED blob at the TIP of the range for each changed
|
|
662
|
+
# path, NOT the working-tree copy -- a secret committed then edited
|
|
663
|
+
# away in the worktree (without a new commit) must still be caught,
|
|
664
|
+
# since the pushed commit still carries the dirty blob.
|
|
665
|
+
try:
|
|
666
|
+
relpaths = get_range_files(args.repo, args.range)
|
|
667
|
+
except GitScanError as e:
|
|
668
|
+
print(f"FATAL: could not resolve commit range {args.range!r}: {e}", file=sys.stderr)
|
|
669
|
+
print("Failing CLOSED: refusing to report CLEAN on an unresolvable range.", file=sys.stderr)
|
|
670
|
+
sys.exit(1)
|
|
671
|
+
|
|
672
|
+
file_count = len(relpaths)
|
|
673
|
+
tip_ref = _range_tip_ref(args.range)
|
|
674
|
+
for relpath in relpaths:
|
|
675
|
+
try:
|
|
676
|
+
content = get_git_blob(args.repo, f"{tip_ref}:{relpath}")
|
|
677
|
+
except GitScanError as e:
|
|
678
|
+
print(f"FATAL: could not read blob at {tip_ref}:{relpath}: {e}", file=sys.stderr)
|
|
679
|
+
print("Failing CLOSED: refusing to report CLEAN when a committed object cannot be read.", file=sys.stderr)
|
|
680
|
+
sys.exit(1)
|
|
681
|
+
label = str(Path(args.repo) / relpath)
|
|
682
|
+
findings = scan_blob(label, content)
|
|
683
|
+
for line_num, rule, match_str, is_fatal in findings:
|
|
684
|
+
all_findings.append((label, line_num, rule, match_str, is_fatal))
|
|
685
|
+
if is_fatal:
|
|
686
|
+
fatal_findings.append((label, line_num, rule, match_str))
|
|
687
|
+
else:
|
|
688
|
+
allowed_doc_count += 1
|
|
689
|
+
|
|
690
|
+
elif args.history:
|
|
691
|
+
# History scanning mode: unaffected by the blob-scan fix above, since
|
|
692
|
+
# it already walks committed diff content via `git log -p`.
|
|
693
|
+
history_files = get_history_files(args.repo)
|
|
694
|
+
file_count = len(set(f for f, _ in history_files))
|
|
426
695
|
for filepath, content in history_files:
|
|
427
696
|
findings = scan_content(content)
|
|
428
697
|
for line_num, rule, match_str, is_fatal in findings:
|
|
429
698
|
all_findings.append((filepath, line_num, rule, match_str, is_fatal))
|
|
430
699
|
if is_fatal:
|
|
431
700
|
fatal_findings.append((filepath, line_num, rule, match_str))
|
|
701
|
+
|
|
432
702
|
else:
|
|
433
|
-
|
|
703
|
+
files = scan_paths(args.paths)
|
|
704
|
+
file_count = len(files)
|
|
434
705
|
for filepath in files:
|
|
435
706
|
findings = scan_file(filepath)
|
|
436
707
|
for line_num, rule, match_str, is_fatal in findings:
|
|
@@ -449,8 +720,6 @@ def main():
|
|
|
449
720
|
print(f"ALLOWED-DOC {filepath}:{line_num} {rule} ({masked})")
|
|
450
721
|
|
|
451
722
|
# Summary and exit
|
|
452
|
-
file_count = len(files) if not args.history else len(set(f for f, _, _, _, _ in all_findings))
|
|
453
|
-
|
|
454
723
|
if len(fatal_findings) == 0:
|
|
455
724
|
if allowed_doc_count == 0:
|
|
456
725
|
if args.history:
|