@openai/codex-security 0.1.9 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -18
- package/_bundled_plugin/.codex-plugin/plugin.json +1 -1
- package/_bundled_plugin/.mcp.json +3 -1
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/preflight/capability-profiles.toml +1 -1
- package/_bundled_plugin/references/scan-contract.md +1 -1
- package/_bundled_plugin/scripts/deep_scan_config.py +21 -1
- package/_bundled_plugin/scripts/deep_scan_workbench.py +66 -14
- package/_bundled_plugin/scripts/finalize_scan_contract.py +15 -207
- package/_bundled_plugin/scripts/generate_in_scope_files.py +127 -2
- package/_bundled_plugin/scripts/generate_rank_input.py +38 -5
- package/_bundled_plugin/scripts/normalize_candidates.py +20 -2
- package/_bundled_plugin/scripts/rank_preview.py +4 -0
- package/_bundled_plugin/scripts/report_projection.py +39 -1
- package/_bundled_plugin/scripts/workbench_cli.py +9 -0
- package/_bundled_plugin/scripts/workbench_db.py +295 -4
- package/_bundled_plugin/scripts/workbench_native_indexes.py +100 -43
- package/_bundled_plugin/scripts/workbench_schema.py +15 -0
- package/_bundled_plugin/scripts/workbench_source_excerpt.py +1 -5
- package/_bundled_plugin/scripts/workbench_target.py +82 -0
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +5 -3
- package/_bundled_plugin/skills/finding-discovery/SKILL.md +8 -4
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +21 -140
- package/dist/api.d.ts +43 -3
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +319 -135
- package/dist/api.js.map +1 -1
- package/dist/auth.d.ts +2 -3
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +59 -270
- package/dist/auth.js.map +1 -1
- package/dist/cli.d.ts +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +107 -43
- package/dist/cli.js.map +1 -1
- package/dist/contract.js +7 -5
- package/dist/contract.js.map +1 -1
- package/dist/cost.d.ts +2 -0
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +24 -2
- package/dist/cost.js.map +1 -1
- package/dist/errors.d.ts +4 -2
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +10 -46
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +2 -1
- package/dist/knowledge-base.js.map +1 -1
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +3 -2
- package/dist/multiscan.js.map +1 -1
- package/dist/result.d.ts +13 -1
- package/dist/result.d.ts.map +1 -1
- package/dist/result.js +3 -0
- package/dist/result.js.map +1 -1
- package/dist/runtime.d.ts +9 -6
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +130 -287
- package/dist/runtime.js.map +1 -1
- package/dist/scan-comparison.d.ts +14 -0
- package/dist/scan-comparison.d.ts.map +1 -1
- package/dist/scan-comparison.js +90 -1
- package/dist/scan-comparison.js.map +1 -1
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +1 -0
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-history-renderer.d.ts +1 -1
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +10 -0
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/scan-logs.d.ts +17 -0
- package/dist/scan-logs.d.ts.map +1 -0
- package/dist/scan-logs.js +90 -0
- package/dist/scan-logs.js.map +1 -0
- package/dist/targets.d.ts +1 -0
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +22 -0
- package/dist/targets.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/version.js.map +1 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -53,6 +53,10 @@ Use `security.preflight()` to validate local inputs, `onWorkerStatus` and
|
|
|
53
53
|
`onReconnect` to observe long-running scans, and an `AbortSignal` to cancel a
|
|
54
54
|
scan.
|
|
55
55
|
|
|
56
|
+
Successful results include open repository findings in `repositoryFindings`,
|
|
57
|
+
when available; `findings` remains the current scan. Matching earlier findings
|
|
58
|
+
can make one additional model call, including with a scan cost limit.
|
|
59
|
+
|
|
56
60
|
Results can contain source excerpts, vulnerability details, and reproduction
|
|
57
61
|
steps. Keep result directories and saved reports outside the repository and
|
|
58
62
|
limit access to authorized reviewers.
|
|
@@ -79,6 +83,7 @@ Pass scan configuration to `security.run(repository, options)` or
|
|
|
79
83
|
| `outputDir` | Choose an artifact directory outside the enclosing Git worktree. |
|
|
80
84
|
| `archiveExisting` | Archive results already in `outputDir` before starting a scan. |
|
|
81
85
|
| `maxCostUsd` | Stop after the estimated model cost exceeds a positive USD amount. |
|
|
86
|
+
| `maxTimeHours` | Limit deep-scan discovery to a positive number of hours, up to 96. |
|
|
82
87
|
| `failureSeverity` | Record a finding-severity policy in the saved scan recipe. |
|
|
83
88
|
| `parentScanId` | Link a rerun to an existing parent scan. |
|
|
84
89
|
| `expectedPluginVersion` | Require the original plugin version when replaying a scan. |
|
|
@@ -208,7 +213,7 @@ npx @openai/codex-security scan /path/to/repository --verbose
|
|
|
208
213
|
npx @openai/codex-security scan /path/to/repository --dry-run
|
|
209
214
|
npx @openai/codex-security scan /path/to/repository --fail-on-severity high
|
|
210
215
|
npx @openai/codex-security scan /path/to/repository --max-cost 5
|
|
211
|
-
npx @openai/codex-security scan /path/to/repository --mode deep --workers 2 --subagents 0 --stop-after-no-new 3 --max-discovery-runs 10
|
|
216
|
+
npx @openai/codex-security scan /path/to/repository --mode deep --workers 2 --subagents 0 --stop-after-no-new 3 --max-discovery-runs 10 --max-time-hours 1.5
|
|
212
217
|
npx @openai/codex-security install-hook
|
|
213
218
|
npx @openai/codex-security bulk-scan
|
|
214
219
|
npx @openai/codex-security bulk-scan --model gpt-5.6-terra --effort high
|
|
@@ -218,10 +223,12 @@ npx @openai/codex-security bulk-scan repositories.csv --output-dir /path/outside
|
|
|
218
223
|
npx @openai/codex-security scans list /path/to/repository
|
|
219
224
|
npx @openai/codex-security scans list --scan-root /path/outside/repository/results
|
|
220
225
|
npx @openai/codex-security scans show SCAN_ID
|
|
226
|
+
npx @openai/codex-security scans logs SCAN_ID
|
|
221
227
|
npx @openai/codex-security scans rerun SCAN_ID
|
|
222
228
|
npx @openai/codex-security scans match PREVIOUS_SCAN_ID CURRENT_SCAN_ID
|
|
223
229
|
npx @openai/codex-security scans match --all
|
|
224
230
|
npx @openai/codex-security scans compare PREVIOUS_SCAN_ID CURRENT_SCAN_ID
|
|
231
|
+
npx @openai/codex-security findings list /path/to/repository
|
|
225
232
|
npx @openai/codex-security findings false-positive OCCURRENCE_ID --reason "The route already checks permissions"
|
|
226
233
|
npx @openai/codex-security export /path/outside/repository/results --export-format sarif --output /path/outside/repository/results.sarif
|
|
227
234
|
npx @openai/codex-security export /path/outside/repository/results --export-format csv --output /path/outside/repository/findings.csv
|
|
@@ -257,8 +264,9 @@ Markdown, text, PDF, and Word (`.docx`) files.
|
|
|
257
264
|
|
|
258
265
|
For `scan --mode deep`, `--workers` limits concurrent discovery workers,
|
|
259
266
|
`--subagents` controls each worker's subagents, `--stop-after-no-new` stops after
|
|
260
|
-
that many runs find no new issues,
|
|
261
|
-
These options are also available
|
|
267
|
+
that many runs find no new issues, `--max-discovery-runs` limits total runs, and
|
|
268
|
+
`--max-time-hours` limits discovery duration. These options are also available
|
|
269
|
+
on SDK scans:
|
|
262
270
|
|
|
263
271
|
```ts
|
|
264
272
|
await security.run("/path/to/repository", {
|
|
@@ -267,6 +275,7 @@ await security.run("/path/to/repository", {
|
|
|
267
275
|
subagents: 0,
|
|
268
276
|
stopAfterNoNew: 3,
|
|
269
277
|
maxDiscoveryRuns: 10,
|
|
278
|
+
maxTimeHours: 1.5,
|
|
270
279
|
});
|
|
271
280
|
```
|
|
272
281
|
|
|
@@ -279,8 +288,15 @@ workers = 2
|
|
|
279
288
|
subagents = 0
|
|
280
289
|
stop_after_no_new = 3
|
|
281
290
|
max_discovery_runs = 10
|
|
291
|
+
max_time_hours = 1.5
|
|
282
292
|
```
|
|
283
293
|
|
|
294
|
+
The discovery deadline defaults to 96 hours. The configured value may be any
|
|
295
|
+
positive number, including fractional hours, up to 96. At the deadline,
|
|
296
|
+
in-flight discovery stops and completed findings are reduced and returned.
|
|
297
|
+
The 97-hour outer tool-call timeout reserves approximately one hour for final
|
|
298
|
+
reduction and result delivery, including at the 96-hour maximum.
|
|
299
|
+
|
|
284
300
|
`scan --workers` controls discovery workers within one deep scan;
|
|
285
301
|
`bulk-scan --workers` controls how many repositories are scanned concurrently.
|
|
286
302
|
|
|
@@ -376,22 +392,22 @@ workers = "auto"
|
|
|
376
392
|
subagents = 3
|
|
377
393
|
stop_after_no_new = 6
|
|
378
394
|
max_discovery_runs = 60
|
|
395
|
+
max_time_hours = 96
|
|
379
396
|
```
|
|
380
397
|
|
|
381
398
|
`workers = "auto"` uses half the available parallelism, with a minimum of one
|
|
382
399
|
and a maximum of six discovery workers. Set `workers` to a positive integer to
|
|
383
400
|
choose an explicit count. `subagents` must be a nonnegative integer;
|
|
384
|
-
`stop_after_no_new` and `max_discovery_runs` must be positive integers.
|
|
385
|
-
`
|
|
401
|
+
`stop_after_no_new` and `max_discovery_runs` must be positive integers.
|
|
402
|
+
`max_time_hours` must be a positive finite number no greater than 96; fractional
|
|
403
|
+
hours are supported. Unknown `[deep_scan]` keys are rejected.
|
|
386
404
|
|
|
387
405
|
These settings are separate from Codex's
|
|
388
406
|
`features.multi_agent_v2.max_concurrent_threads_per_session` and
|
|
389
|
-
`bulk-scan --workers`.
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
`--codex` to adjust the Codex session thread limit, not to set `[deep_scan]`
|
|
394
|
-
values.
|
|
407
|
+
`bulk-scan --workers`. Standalone CLI and SDK scans create an isolated
|
|
408
|
+
`CODEX_HOME`, import the ambient `[deep_scan]` configuration, and apply explicit
|
|
409
|
+
CLI or SDK options on top. Use `--codex` to adjust the Codex session thread
|
|
410
|
+
limit, not to set `[deep_scan]` values.
|
|
395
411
|
|
|
396
412
|
### Environment variables
|
|
397
413
|
|
|
@@ -400,10 +416,11 @@ The CLI and SDK recognize the following user-configurable environment:
|
|
|
400
416
|
| Variable | Effect |
|
|
401
417
|
| --------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------- |
|
|
402
418
|
| `OPENAI_API_KEY`, `CODEX_API_KEY` | Scan authentication; `OPENAI_API_KEY` wins when both are present. |
|
|
403
|
-
| `CODEX_SECURITY_LOG_LEVEL` | CLI-only; set to `debug` for
|
|
419
|
+
| `CODEX_SECURITY_LOG_LEVEL` | CLI-only; set to `debug` for verbose diagnostics. |
|
|
404
420
|
| `LOG_LEVEL` | CLI-only fallback when `CODEX_SECURITY_LOG_LEVEL` is unset. |
|
|
405
421
|
| `CODEX_SECURITY_STATE_DIR` | Override the private scan-history, workbench, and default artifact directory. |
|
|
406
422
|
| `CODEX_HOME` | Set the ambient Codex home for file-backed sign-in and default state; defaults to `~/.codex`. |
|
|
423
|
+
| `CODEX_CLI_PATH` | Use another Codex executable for authentication, plugin setup, scans, and nested workers. |
|
|
407
424
|
| `PYTHON` | Select a Python interpreter when `--python` or SDK `pythonPath` is not set. |
|
|
408
425
|
| `GH_HOST` | Select a GitHub Enterprise host during interactive `bulk-scan` discovery. |
|
|
409
426
|
| `CODEX_SECURITY_NO_UPDATE_NOTICE`, `NO_UPDATE_NOTIFIER` | Disable interactive update notices when either variable is defined. |
|
|
@@ -451,11 +468,12 @@ token and worker counts, estimated cost, the results directory, and the next
|
|
|
451
468
|
useful command.
|
|
452
469
|
Progress and summaries use stderr; structured scan results remain on stdout.
|
|
453
470
|
|
|
454
|
-
Add `--verbose` or set `CODEX_SECURITY_LOG_LEVEL=debug` to print
|
|
471
|
+
Add `--verbose` or set `CODEX_SECURITY_LOG_LEVEL=debug` to print
|
|
455
472
|
lifecycle, authentication, progress, and cost diagnostics to stderr.
|
|
456
473
|
`LOG_LEVEL=debug` is used only when `CODEX_SECURITY_LOG_LEVEL` is unset.
|
|
457
|
-
|
|
458
|
-
|
|
474
|
+
Structured JSON results remain on stdout. Verbose diagnostics may contain
|
|
475
|
+
sensitive data; review local logs before sharing them. The interactive
|
|
476
|
+
dashboard omits activity containing recognizable credentials.
|
|
459
477
|
|
|
460
478
|
Each scan records its model, tokens, and estimated cost in its JSON result,
|
|
461
479
|
scan history, and bulk-scan receipt. Estimates use
|
|
@@ -463,8 +481,11 @@ scan history, and bulk-scan receipt. Estimates use
|
|
|
463
481
|
including cached input and cache writes; fees and surcharges are not included.
|
|
464
482
|
|
|
465
483
|
Use `--max-cost USD` to stop a scan, including its delegated workers, when its
|
|
466
|
-
running cost exceeds the limit.
|
|
467
|
-
|
|
484
|
+
running cost exceeds the limit. If a Deep Scan has already finished discovery,
|
|
485
|
+
it returns a sealed partial report with any completed findings and lists
|
|
486
|
+
unvalidated candidates as follow-up work. Requests already in progress can
|
|
487
|
+
finish above the limit; preparing the partial report makes no additional model
|
|
488
|
+
requests. Incomplete coverage retains its existing exit code.
|
|
468
489
|
|
|
469
490
|
Run `npx @openai/codex-security scan --help` or `npx @openai/codex-security bulk-scan --help`
|
|
470
491
|
for the complete CLI references.
|
|
@@ -508,13 +529,17 @@ whose artifacts are under a particular root. `scans show SCAN_ID` includes the
|
|
|
508
529
|
scan configuration, results, coverage, and artifact locations. Add
|
|
509
530
|
`--show-linked-findings` to include finding links from previous scans.
|
|
510
531
|
|
|
532
|
+
`scans logs SCAN_ID` shows complete session events from the scan and its
|
|
533
|
+
workers, which can include source code and credentials.
|
|
534
|
+
|
|
511
535
|
Every scan history command accepts a full scan ID or a unique prefix of at
|
|
512
536
|
least eight characters.
|
|
513
537
|
|
|
514
538
|
Scan history uses the existing Codex Security workbench database at
|
|
515
539
|
`$CODEX_HOME/state/plugins/codex-security/workbench.sqlite3`. Set
|
|
516
540
|
`CODEX_SECURITY_STATE_DIR` to place the database elsewhere. Scan credentials
|
|
517
|
-
are never stored in the scan configuration.
|
|
541
|
+
are never stored in the scan configuration. Recorded failure summaries and
|
|
542
|
+
bulk-scan receipts omit messages that contain recognizable credentials.
|
|
518
543
|
|
|
519
544
|
The scan sandbox permits writes to the selected state directory so SQLite can
|
|
520
545
|
maintain its database and journal files. If the host itself cannot write to the
|
|
@@ -9,6 +9,8 @@
|
|
|
9
9
|
"CODEX_SQLITE_HOME",
|
|
10
10
|
"CODEX_API_KEY",
|
|
11
11
|
"CODEX_CLI_PATH",
|
|
12
|
+
"OPENROUTER_API_KEY",
|
|
13
|
+
"FIREWORKS_API_KEY",
|
|
12
14
|
"AWS_BEARER_TOKEN_BEDROCK",
|
|
13
15
|
"AWS_ACCESS_KEY_ID",
|
|
14
16
|
"AWS_SECRET_ACCESS_KEY",
|
|
@@ -37,7 +39,7 @@
|
|
|
37
39
|
"REQUESTS_CA_BUNDLE",
|
|
38
40
|
"NODE_EXTRA_CA_CERTS"
|
|
39
41
|
],
|
|
40
|
-
"tool_timeout_sec":
|
|
42
|
+
"tool_timeout_sec": 349200
|
|
41
43
|
}
|
|
42
44
|
}
|
|
43
45
|
}
|
|
Binary file
|
|
Binary file
|
|
@@ -40,7 +40,7 @@ description = "Capabilities for Git-backed Codex Security diff scans."
|
|
|
40
40
|
[[profiles.security_diff_scan.requirements]]
|
|
41
41
|
capability = "delegated_workers"
|
|
42
42
|
severity = "warn"
|
|
43
|
-
reason = "Large
|
|
43
|
+
reason = "Large diff scans may use discovery workers when available; otherwise the parent reviews every changed file."
|
|
44
44
|
|
|
45
45
|
[[profiles.security_diff_scan.requirements]]
|
|
46
46
|
capability = "goal_tools"
|
|
@@ -44,7 +44,7 @@ A dirty checkout has `allowedKinds: ["git_worktree"]`: copy `requiredSnapshotDig
|
|
|
44
44
|
|
|
45
45
|
`targetId` identifies the stable repository or workspace. Prefer a digest of a sanitized canonical absolute remote URL when one exists. Otherwise use a digest of a stable local workspace identity. Never persist remote URL credentials, query parameters, fragments, or tokens.
|
|
46
46
|
|
|
47
|
-
For dirty worktrees and diffs, calculate `snapshotDigest` from a deterministic representation of the reviewed content, including staged changes and reviewed untracked files where applicable. For directory snapshots, hash a sorted relative-path and file-hash inventory of the reviewed scope. Encode the result as `codex-security-snapshot/v1:sha256:<64 lowercase hex characters>`.
|
|
47
|
+
For dirty worktrees and working-tree diffs, calculate `snapshotDigest` from a deterministic representation of the reviewed content, including staged changes and reviewed untracked files where applicable. For committed or revision-range diffs, derive it from the exact authoritative diff kind and immutable base/head revisions. For directory snapshots, hash a sorted relative-path and file-hash inventory of the reviewed scope. Encode the result as `codex-security-snapshot/v1:sha256:<64 lowercase hex characters>`.
|
|
48
48
|
|
|
49
49
|
## Finding Identity
|
|
50
50
|
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import argparse
|
|
6
|
+
import math
|
|
6
7
|
import os
|
|
7
8
|
from pathlib import Path
|
|
8
9
|
from typing import Any
|
|
@@ -16,6 +17,8 @@ DEFAULT_SUBAGENTS = 3
|
|
|
16
17
|
DEFAULT_STOP_AFTER_NO_NEW = 6
|
|
17
18
|
DEFAULT_STOP_AFTER_CONSECUTIVE_ERRORS = 3
|
|
18
19
|
DEFAULT_MAX_DISCOVERY_RUNS = 60
|
|
20
|
+
DEFAULT_MAX_TIME_HOURS = 96
|
|
21
|
+
MAX_TIME_HOURS = 96
|
|
19
22
|
MAX_AUTOMATIC_WORKERS = 6
|
|
20
23
|
CONFIG_KEYS = {
|
|
21
24
|
"workers",
|
|
@@ -23,6 +26,7 @@ CONFIG_KEYS = {
|
|
|
23
26
|
"stop_after_no_new",
|
|
24
27
|
"stop_after_consecutive_errors",
|
|
25
28
|
"max_discovery_runs",
|
|
29
|
+
"max_time_hours",
|
|
26
30
|
}
|
|
27
31
|
|
|
28
32
|
|
|
@@ -34,7 +38,7 @@ def config_path() -> Path:
|
|
|
34
38
|
return codex_home() / "codex-security" / "config.toml"
|
|
35
39
|
|
|
36
40
|
|
|
37
|
-
def resolve_deep_scan_config(available_parallelism: int) -> dict[str, int]:
|
|
41
|
+
def resolve_deep_scan_config(available_parallelism: int) -> dict[str, int | float]:
|
|
38
42
|
if isinstance(available_parallelism, bool) or available_parallelism < 1:
|
|
39
43
|
raise SystemExit("Available parallelism must be a positive integer.")
|
|
40
44
|
path = config_path()
|
|
@@ -87,6 +91,10 @@ def resolve_deep_scan_config(available_parallelism: int) -> dict[str, int]:
|
|
|
87
91
|
"deep_scan.max_discovery_runs",
|
|
88
92
|
minimum=1,
|
|
89
93
|
),
|
|
94
|
+
"maxTimeHours": require_positive_number(
|
|
95
|
+
configured.get("max_time_hours", DEFAULT_MAX_TIME_HOURS),
|
|
96
|
+
"deep_scan.max_time_hours",
|
|
97
|
+
),
|
|
90
98
|
}
|
|
91
99
|
|
|
92
100
|
|
|
@@ -103,6 +111,18 @@ def require_integer(value: object, label: str, *, minimum: int) -> int:
|
|
|
103
111
|
return value
|
|
104
112
|
|
|
105
113
|
|
|
114
|
+
def require_positive_number(value: object, label: str) -> int | float:
|
|
115
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
116
|
+
raise SystemExit(f"{label} must be a positive finite number no greater than 96.")
|
|
117
|
+
try:
|
|
118
|
+
finite = math.isfinite(value)
|
|
119
|
+
except OverflowError:
|
|
120
|
+
finite = False
|
|
121
|
+
if not finite or value <= 0 or value > MAX_TIME_HOURS:
|
|
122
|
+
raise SystemExit(f"{label} must be a positive finite number no greater than 96.")
|
|
123
|
+
return value
|
|
124
|
+
|
|
125
|
+
|
|
106
126
|
def main() -> None:
|
|
107
127
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
108
128
|
parser.add_argument("--available-parallelism", type=int, required=True)
|
|
@@ -217,6 +217,11 @@ def require_deep_scan_run(connection: sqlite3.Connection, scan_id: str) -> sqlit
|
|
|
217
217
|
return row
|
|
218
218
|
|
|
219
219
|
|
|
220
|
+
def deep_scan_deadline_reached(run: sqlite3.Row) -> bool:
|
|
221
|
+
elapsed = datetime.fromisoformat(now()) - datetime.fromisoformat(str(run["created_at"]))
|
|
222
|
+
return elapsed.total_seconds() / 3600 >= run["max_time_hours"]
|
|
223
|
+
|
|
224
|
+
|
|
220
225
|
def require_deep_scan_ready_for_parent_completion(
|
|
221
226
|
connection: sqlite3.Connection, scan: sqlite3.Row
|
|
222
227
|
) -> None:
|
|
@@ -361,6 +366,23 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A
|
|
|
361
366
|
""",
|
|
362
367
|
(run["scan_id"],),
|
|
363
368
|
).fetchone()
|
|
369
|
+
canonical_artifacts = None
|
|
370
|
+
if run["canonical_inventory_path"] is None:
|
|
371
|
+
if successful_reducer is not None:
|
|
372
|
+
canonical_artifacts = canonical_discovery_artifacts(scan)
|
|
373
|
+
elif (
|
|
374
|
+
run["status"] == "succeeded"
|
|
375
|
+
and run["terminal_reason"] == "capped"
|
|
376
|
+
and run["completion_sequence"] == 0
|
|
377
|
+
and deep_scan_deadline_reached(run)
|
|
378
|
+
):
|
|
379
|
+
candidate_artifacts = canonical_discovery_artifacts(scan)
|
|
380
|
+
if Path(candidate_artifacts["candidateLedgerPath"]).stat().st_size != 0:
|
|
381
|
+
raise SystemExit(
|
|
382
|
+
"A capped Deep Scan without completed discoveries requires an empty "
|
|
383
|
+
"candidate ledger."
|
|
384
|
+
)
|
|
385
|
+
canonical_artifacts = candidate_artifacts
|
|
364
386
|
return {
|
|
365
387
|
"scanId": run["scan_id"],
|
|
366
388
|
"targetPath": scan["target_path"],
|
|
@@ -378,17 +400,14 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A
|
|
|
378
400
|
"stopAfterNoNew": run["stop_after_no_new"],
|
|
379
401
|
"stopAfterConsecutiveErrors": run["stop_after_consecutive_errors"],
|
|
380
402
|
"maxDiscoveryRuns": run["max_discovery_runs"],
|
|
403
|
+
"maxTimeHours": run["max_time_hours"],
|
|
381
404
|
},
|
|
382
405
|
"dispatchedCount": run["discovery_runs_dispatched"],
|
|
383
406
|
"completionSequence": run["completion_sequence"],
|
|
384
407
|
"noNewStreak": run["consecutive_no_new"],
|
|
385
408
|
"consecutiveErrors": run["consecutive_errors"],
|
|
386
409
|
"cancelRequested": bool(run["cancel_requested"]),
|
|
387
|
-
"canonicalArtifacts":
|
|
388
|
-
canonical_discovery_artifacts(scan)
|
|
389
|
-
if successful_reducer is not None and run["canonical_inventory_path"] is None
|
|
390
|
-
else None
|
|
391
|
-
),
|
|
410
|
+
"canonicalArtifacts": canonical_artifacts,
|
|
392
411
|
"manifestPath": run["manifest_path"],
|
|
393
412
|
"terminalReason": run["terminal_reason"],
|
|
394
413
|
"error": run["error_message"],
|
|
@@ -466,7 +485,7 @@ def deep_scan_result(
|
|
|
466
485
|
return result
|
|
467
486
|
|
|
468
487
|
|
|
469
|
-
def effective_deep_scan_config(args: argparse.Namespace) -> dict[str, int]:
|
|
488
|
+
def effective_deep_scan_config(args: argparse.Namespace) -> dict[str, int | float]:
|
|
470
489
|
available_parallelism = args.available_parallelism or os.cpu_count() or 1
|
|
471
490
|
return resolve_deep_scan_config(available_parallelism)
|
|
472
491
|
|
|
@@ -474,7 +493,7 @@ def effective_deep_scan_config(args: argparse.Namespace) -> dict[str, int]:
|
|
|
474
493
|
def ensure_deep_scan_run(
|
|
475
494
|
connection: sqlite3.Connection,
|
|
476
495
|
scan: sqlite3.Row,
|
|
477
|
-
config: dict[str, int],
|
|
496
|
+
config: dict[str, int | float],
|
|
478
497
|
workflow_version: str,
|
|
479
498
|
timestamp: str,
|
|
480
499
|
) -> sqlite3.Row:
|
|
@@ -492,9 +511,9 @@ def ensure_deep_scan_run(
|
|
|
492
511
|
INSERT INTO deep_scan_runs (
|
|
493
512
|
scan_id, schema_version, workflow_version, status, phase,
|
|
494
513
|
workers, subagents, stop_after_no_new, stop_after_consecutive_errors,
|
|
495
|
-
max_discovery_runs,
|
|
514
|
+
max_discovery_runs, max_time_hours,
|
|
496
515
|
created_at, updated_at
|
|
497
|
-
) VALUES (?, 1, ?, 'running', 'setup', ?, ?, ?, ?, ?, ?, ?)
|
|
516
|
+
) VALUES (?, 1, ?, 'running', 'setup', ?, ?, ?, ?, ?, ?, ?, ?)
|
|
498
517
|
""",
|
|
499
518
|
(
|
|
500
519
|
scan["id"],
|
|
@@ -504,6 +523,7 @@ def ensure_deep_scan_run(
|
|
|
504
523
|
config["stopAfterNoNew"],
|
|
505
524
|
config["stopAfterConsecutiveErrors"],
|
|
506
525
|
config["maxDiscoveryRuns"],
|
|
526
|
+
config["maxTimeHours"],
|
|
507
527
|
timestamp,
|
|
508
528
|
timestamp,
|
|
509
529
|
),
|
|
@@ -1274,6 +1294,28 @@ def upsert_deep_scan_worker(
|
|
|
1274
1294
|
""",
|
|
1275
1295
|
(timestamp, scan_id),
|
|
1276
1296
|
)
|
|
1297
|
+
elif args.kind == "dedup" and args.status == "failed" and existing["status"] == "running":
|
|
1298
|
+
connection.execute(
|
|
1299
|
+
"""
|
|
1300
|
+
UPDATE deep_scan_workers
|
|
1301
|
+
SET merge_state = 'buffered', updated_at = ?
|
|
1302
|
+
WHERE scan_id = ? AND kind = 'discovery' AND status = 'succeeded'
|
|
1303
|
+
AND merge_state = 'merging'
|
|
1304
|
+
AND id IN (
|
|
1305
|
+
SELECT discovery_worker_id FROM deep_scan_dedup_inputs
|
|
1306
|
+
WHERE scan_id = ? AND dedup_worker_id = ?
|
|
1307
|
+
)
|
|
1308
|
+
""",
|
|
1309
|
+
(timestamp, scan_id, scan_id, worker_id),
|
|
1310
|
+
)
|
|
1311
|
+
connection.execute(
|
|
1312
|
+
"""
|
|
1313
|
+
UPDATE deep_scan_runs
|
|
1314
|
+
SET phase = 'discovery', updated_at = ?
|
|
1315
|
+
WHERE scan_id = ?
|
|
1316
|
+
""",
|
|
1317
|
+
(timestamp, scan_id),
|
|
1318
|
+
)
|
|
1277
1319
|
completed_at = (
|
|
1278
1320
|
timestamp
|
|
1279
1321
|
if args.status in {"succeeded", "failed", "canceled"}
|
|
@@ -1382,9 +1424,12 @@ def claim_deep_scan_dedup(
|
|
|
1382
1424
|
"A Deep Scan dedup worker must claim an ordered prefix of buffered discovery "
|
|
1383
1425
|
"results in completion order."
|
|
1384
1426
|
)
|
|
1385
|
-
|
|
1427
|
+
capped_singleton = (
|
|
1386
1428
|
len(input_ids) == 1
|
|
1387
|
-
and
|
|
1429
|
+
and (
|
|
1430
|
+
run["discovery_runs_dispatched"] >= run["max_discovery_runs"]
|
|
1431
|
+
or deep_scan_deadline_reached(run)
|
|
1432
|
+
)
|
|
1388
1433
|
and connection.execute(
|
|
1389
1434
|
"""
|
|
1390
1435
|
SELECT 1 FROM deep_scan_workers
|
|
@@ -1403,7 +1448,7 @@ def claim_deep_scan_dedup(
|
|
|
1403
1448
|
""",
|
|
1404
1449
|
(scan_id,),
|
|
1405
1450
|
).fetchone()
|
|
1406
|
-
minimum_inputs = 1 if successful_reducer is not None or
|
|
1451
|
+
minimum_inputs = 1 if successful_reducer is not None or capped_singleton else 2
|
|
1407
1452
|
if len(input_ids) < minimum_inputs:
|
|
1408
1453
|
raise SystemExit(
|
|
1409
1454
|
"The first Deep Scan dedup requires two buffered discovery results."
|
|
@@ -1657,12 +1702,13 @@ def finish_deep_scan_locked(
|
|
|
1657
1702
|
if (
|
|
1658
1703
|
args.terminal_reason == "capped"
|
|
1659
1704
|
and run["discovery_runs_dispatched"] < run["max_discovery_runs"]
|
|
1705
|
+
and not deep_scan_deadline_reached(run)
|
|
1660
1706
|
):
|
|
1661
1707
|
raise SystemExit(
|
|
1662
1708
|
"Deep Scan cannot finish capped before reaching its configured maximum."
|
|
1663
1709
|
)
|
|
1664
1710
|
try:
|
|
1665
|
-
canonical_discovery_artifacts(scan)
|
|
1711
|
+
canonical_artifacts = canonical_discovery_artifacts(scan)
|
|
1666
1712
|
except SystemExit as exc:
|
|
1667
1713
|
raise SystemExit(
|
|
1668
1714
|
f"Deep Scan cannot finish without canonical discovery artifacts: {exc}"
|
|
@@ -1675,7 +1721,13 @@ def finish_deep_scan_locked(
|
|
|
1675
1721
|
""",
|
|
1676
1722
|
(scan_id,),
|
|
1677
1723
|
).fetchone()
|
|
1678
|
-
|
|
1724
|
+
zero_discovery_deadline = (
|
|
1725
|
+
args.terminal_reason == "capped"
|
|
1726
|
+
and deep_scan_deadline_reached(run)
|
|
1727
|
+
and run["completion_sequence"] == 0
|
|
1728
|
+
and Path(canonical_artifacts["candidateLedgerPath"]).stat().st_size == 0
|
|
1729
|
+
)
|
|
1730
|
+
if successful_reducer is None and not zero_discovery_deadline:
|
|
1679
1731
|
raise SystemExit("Deep Scan cannot finish without a successful dedup worker.")
|
|
1680
1732
|
failed_worker = connection.execute(
|
|
1681
1733
|
"""
|