@openai/codex-security 0.5.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/permission-profile-preflight.mjs +60 -9
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/schemas/definitions/artifact-common.schema.json +1 -1
- package/_bundled_plugin/scripts/finalize_scan_contract.py +12 -1
- package/_bundled_plugin/scripts/generate_in_scope_files.py +6 -6
- package/_bundled_plugin/scripts/generate_rank_input.py +12 -6
- package/_bundled_plugin/scripts/report_projection.py +2 -1
- package/_bundled_plugin/scripts/threat_model_projection.py +1 -3
- package/_bundled_plugin/scripts/workbench/saved_coverage.py +234 -0
- package/_bundled_plugin/scripts/workbench_cli.py +5 -7
- package/_bundled_plugin/scripts/workbench_db.py +12 -6
- package/_bundled_plugin/scripts/workbench_native_indexes.py +10 -7
- package/_bundled_plugin/scripts/workbench_publication.py +4 -3
- package/_bundled_plugin/scripts/workbench_remediation.py +5 -1
- package/_bundled_plugin/scripts/workbench_saved_results.py +542 -148
- package/_bundled_plugin/scripts/workbench_scan_history.py +79 -78
- package/_bundled_plugin/scripts/workbench_scan_usage.py +61 -13
- package/_bundled_plugin/scripts/workbench_target.py +21 -7
- package/_bundled_plugin/scripts/workbench_validation.py +21 -18
- package/_bundled_plugin/shared/workbench-migrations.json +10 -0
- package/dist/api.d.ts +1 -1
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +161 -57
- package/dist/api.js.map +1 -1
- package/dist/classify-scan-severity.d.ts.map +1 -1
- package/dist/classify-scan-severity.js +16 -18
- package/dist/classify-scan-severity.js.map +1 -1
- package/dist/cli-daybreak.d.ts +5 -0
- package/dist/cli-daybreak.d.ts.map +1 -0
- package/dist/cli-daybreak.js +89 -0
- package/dist/cli-daybreak.js.map +1 -0
- package/dist/cli.d.ts +5 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +821 -591
- package/dist/cli.js.map +1 -1
- package/dist/codeowners.js +2 -2
- package/dist/codeowners.js.map +1 -1
- package/dist/component-plan.d.ts.map +1 -1
- package/dist/component-plan.js +3 -2
- package/dist/component-plan.js.map +1 -1
- package/dist/component-scan.js +2 -2
- package/dist/component-scan.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +5 -1
- package/dist/config.js.map +1 -1
- package/dist/cost.d.ts +2 -1
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +17 -8
- package/dist/cost.js.map +1 -1
- package/dist/custom-validation.d.ts.map +1 -1
- package/dist/custom-validation.js +2 -5
- package/dist/custom-validation.js.map +1 -1
- package/dist/deduplication/codex-review.d.ts.map +1 -1
- package/dist/deduplication/codex-review.js +8 -6
- package/dist/deduplication/codex-review.js.map +1 -1
- package/dist/deduplication/local.d.ts.map +1 -1
- package/dist/deduplication/local.js +5 -2
- package/dist/deduplication/local.js.map +1 -1
- package/dist/deduplication/retry.d.ts.map +1 -1
- package/dist/deduplication/retry.js +5 -0
- package/dist/deduplication/retry.js.map +1 -1
- package/dist/deduplication/scan.js +5 -1
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/deep-scan.d.ts +6 -0
- package/dist/deep-scan.d.ts.map +1 -1
- package/dist/deep-scan.js +8 -6
- package/dist/deep-scan.js.map +1 -1
- package/dist/findings-import.d.ts.map +1 -1
- package/dist/findings-import.js +23 -3
- package/dist/findings-import.js.map +1 -1
- package/dist/host-protocol.d.ts +1 -0
- package/dist/host-protocol.d.ts.map +1 -1
- package/dist/host-protocol.js +1 -1
- package/dist/host-protocol.js.map +1 -1
- package/dist/hosted-remediation-protocol.d.ts +5 -0
- package/dist/hosted-remediation-protocol.d.ts.map +1 -0
- package/dist/hosted-remediation-protocol.js +26 -0
- package/dist/hosted-remediation-protocol.js.map +1 -0
- package/dist/hosted-remediation.d.ts +89 -0
- package/dist/hosted-remediation.d.ts.map +1 -0
- package/dist/hosted-remediation.js +122 -0
- package/dist/hosted-remediation.js.map +1 -0
- package/dist/import-scan.d.ts +1 -0
- package/dist/import-scan.d.ts.map +1 -1
- package/dist/import-scan.js +5 -1
- package/dist/import-scan.js.map +1 -1
- package/dist/index.d.ts +5 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +3 -1
- package/dist/multiscan.js.map +1 -1
- package/dist/project-config.d.ts.map +1 -1
- package/dist/project-config.js +13 -9
- package/dist/project-config.js.map +1 -1
- package/dist/publish.d.ts +2 -0
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +9 -0
- package/dist/publish.js.map +1 -1
- package/dist/remediation.d.ts +63 -0
- package/dist/remediation.d.ts.map +1 -0
- package/dist/remediation.js +110 -0
- package/dist/remediation.js.map +1 -0
- package/dist/request-metadata.d.ts +17 -7
- package/dist/request-metadata.d.ts.map +1 -1
- package/dist/request-metadata.js +43 -8
- package/dist/request-metadata.js.map +1 -1
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +21 -14
- package/dist/runtime.js.map +1 -1
- package/dist/saved-scan-bootstrap.d.ts +1 -0
- package/dist/saved-scan-bootstrap.d.ts.map +1 -1
- package/dist/saved-scan-bootstrap.js +16 -4
- package/dist/saved-scan-bootstrap.js.map +1 -1
- package/dist/scan-activity.js +8 -8
- package/dist/scan-activity.js.map +1 -1
- package/dist/scan-comparison.d.ts.map +1 -1
- package/dist/scan-comparison.js +15 -9
- package/dist/scan-comparison.js.map +1 -1
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +12 -6
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-executor.d.ts +5 -1
- package/dist/scan-executor.d.ts.map +1 -1
- package/dist/scan-executor.js +9 -8
- package/dist/scan-executor.js.map +1 -1
- package/dist/{deep-progress.d.ts → scan-progress.d.ts} +15 -7
- package/dist/scan-progress.d.ts.map +1 -0
- package/dist/scan-progress.js +138 -0
- package/dist/scan-progress.js.map +1 -0
- package/dist/security-policy.js +1 -1
- package/dist/security-policy.js.map +1 -1
- package/dist/server/embeddings.d.ts.map +1 -1
- package/dist/server/embeddings.js +4 -0
- package/dist/server/embeddings.js.map +1 -1
- package/dist/server/sqlite-store.d.ts.map +1 -1
- package/dist/server/sqlite-store.js +14 -2
- package/dist/server/sqlite-store.js.map +1 -1
- package/dist/severity-store.js +4 -1
- package/dist/severity-store.js.map +1 -1
- package/dist/timestamps.d.ts +3 -0
- package/dist/timestamps.d.ts.map +1 -0
- package/dist/timestamps.js +14 -0
- package/dist/timestamps.js.map +1 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +13 -2
- package/dist/version.js.map +1 -1
- package/dist/workbench-arguments.d.ts.map +1 -1
- package/dist/workbench-arguments.js +9 -3
- package/dist/workbench-arguments.js.map +1 -1
- package/dist/worker-progress.d.ts +11 -1
- package/dist/worker-progress.d.ts.map +1 -1
- package/dist/worker-progress.js +11 -1
- package/dist/worker-progress.js.map +1 -1
- package/docs/cli.md +86 -15
- package/docs/findings-service.md +7 -2
- package/docs/hosted-remediation.md +117 -0
- package/docs/hosted-scans.md +8 -3
- package/package.json +5 -4
- package/dist/deep-progress.d.ts.map +0 -1
- package/dist/deep-progress.js +0 -92
- package/dist/deep-progress.js.map +0 -1
package/README.md
CHANGED
|
@@ -265,6 +265,13 @@ Constructor options are `codexOverrides`, `pythonPath`, and `pluginPath`.
|
|
|
265
265
|
The bundled runtime and plugin are used by default. `pythonPath` overrides the
|
|
266
266
|
`PYTHON` environment variable. To choose a model, set `codexOverrides.model`.
|
|
267
267
|
|
|
268
|
+
To enable Daybreak on Codex 0.163.0 or later, set `codexOverrides.daybreak: true`.
|
|
269
|
+
The toggle defaults off; an explicit `false` keeps it off. The setting reaches
|
|
270
|
+
Deep Scan discovery and reducer workers, including resumed workers. Scans use
|
|
271
|
+
isolated configuration, so a desktop toggle or personal Codex configuration does
|
|
272
|
+
not configure an SDK scan. See [Daybreak opt-in](docs/cli.md#daybreak-opt-in) for
|
|
273
|
+
CLI, project-file, and direct plugin configuration.
|
|
274
|
+
|
|
268
275
|
New and resumed Deep Scans require a current plugin with
|
|
269
276
|
`directDeepScanEngine: 1`, `workerProviderSnapshot: 4`, and isolated worker
|
|
270
277
|
configuration support. Older custom plugins fail before scan execution; update
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -53,6 +53,50 @@ async function readDeepScanRuntimeConfig(options) {
|
|
|
53
53
|
return config;
|
|
54
54
|
});
|
|
55
55
|
}
|
|
56
|
+
async function readDeepScanRuntimeConfigWithOrigins(options) {
|
|
57
|
+
return withPreflightClient(
|
|
58
|
+
options,
|
|
59
|
+
(client) => client.request("config/read", { cwd: options.cwd, includeLayers: true })
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
async function readDeepScanParentCyberAccessProgram(options) {
|
|
63
|
+
return withPreflightClient(options, async (client) => {
|
|
64
|
+
let response;
|
|
65
|
+
try {
|
|
66
|
+
response = await client.request("thread/read", {
|
|
67
|
+
threadId: options.parentTurn.threadId,
|
|
68
|
+
includeTurns: false
|
|
69
|
+
});
|
|
70
|
+
} catch (error) {
|
|
71
|
+
const nativeError = asRecord(
|
|
72
|
+
error instanceof Error ? error.cause : void 0
|
|
73
|
+
);
|
|
74
|
+
if (nativeError?.code === -32600 && nativeError.message === `thread not loaded: ${options.parentTurn.threadId}`)
|
|
75
|
+
return void 0;
|
|
76
|
+
throw error;
|
|
77
|
+
}
|
|
78
|
+
const thread = asRecord(response.thread);
|
|
79
|
+
if (thread?.id !== options.parentTurn.threadId || typeof thread.path !== "string")
|
|
80
|
+
return void 0;
|
|
81
|
+
const input = createReadStream(thread.path, { encoding: "utf8" });
|
|
82
|
+
const lines = createInterface({ input, crlfDelay: Infinity });
|
|
83
|
+
try {
|
|
84
|
+
for await (const line of lines) {
|
|
85
|
+
if (!line.trim()) continue;
|
|
86
|
+
const entry = asRecord(JSON.parse(line));
|
|
87
|
+
if (entry?.type !== "turn_context") continue;
|
|
88
|
+
const context = asRecord(entry.payload);
|
|
89
|
+
if (context?.turn_id !== options.parentTurn.turnId) continue;
|
|
90
|
+
const program = context.cyber_access_program;
|
|
91
|
+
return typeof program === "string" ? program : void 0;
|
|
92
|
+
}
|
|
93
|
+
return void 0;
|
|
94
|
+
} finally {
|
|
95
|
+
lines.close();
|
|
96
|
+
input.destroy();
|
|
97
|
+
}
|
|
98
|
+
});
|
|
99
|
+
}
|
|
56
100
|
async function prepareCliDeepScanSession(options) {
|
|
57
101
|
return withPreflightClient(options, async (client) => {
|
|
58
102
|
const response = await client.request("thread/start", {
|
|
@@ -157,14 +201,18 @@ async function withPreflightClient(options, operation) {
|
|
|
157
201
|
var AppServerPreflightClient = class {
|
|
158
202
|
constructor(options) {
|
|
159
203
|
this.options = options;
|
|
160
|
-
const args = [
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
204
|
+
const args = [
|
|
205
|
+
...options.appServerArgs ?? options.commandArgs ?? []
|
|
206
|
+
];
|
|
207
|
+
if (options.appServerArgs === void 0) {
|
|
208
|
+
for (const override of options.configOverrides) {
|
|
209
|
+
args.push("--config", override);
|
|
210
|
+
}
|
|
211
|
+
for (const override of options.providerConfigOverrides ?? []) {
|
|
212
|
+
args.push("--config", override);
|
|
213
|
+
}
|
|
214
|
+
args.push("app-server", "--stdio");
|
|
166
215
|
}
|
|
167
|
-
args.push("app-server", "--stdio");
|
|
168
216
|
this.child = spawn(executablePathForSpawn(options.codexPath), args, {
|
|
169
217
|
cwd: options.cwd,
|
|
170
218
|
...options.env === void 0 ? {} : { env: options.env },
|
|
@@ -482,7 +530,8 @@ function jsonRpcPreflightError(codexPath, method, value, context) {
|
|
|
482
530
|
return failure;
|
|
483
531
|
}
|
|
484
532
|
return new Error(
|
|
485
|
-
subject(context) + " cannot safely verify its read-only worker permission profile because the selected Codex executable " + JSON.stringify(codexPath) + " returned an error for " + JSON.stringify(method) + (code === void 0 ? "" : " (JSON-RPC code " + code + ")") + ". Check the Codex configuration and retry. " + subject(context) + " did not run." + (typeof error?.message === "string" ? "\n" + error.message : "")
|
|
533
|
+
subject(context) + " cannot safely verify its read-only worker permission profile because the selected Codex executable " + JSON.stringify(codexPath) + " returned an error for " + JSON.stringify(method) + (code === void 0 ? "" : " (JSON-RPC code " + code + ")") + ". Check the Codex configuration and retry. " + subject(context) + " did not run." + (typeof error?.message === "string" ? "\n" + error.message : ""),
|
|
534
|
+
{ cause: value }
|
|
486
535
|
);
|
|
487
536
|
}
|
|
488
537
|
function codexExecutableStartError(codexPath, error, context) {
|
|
@@ -552,5 +601,7 @@ export {
|
|
|
552
601
|
deepScanPermissionProfileFallbackError,
|
|
553
602
|
preflightDeepScanWorkerPermissionProfile,
|
|
554
603
|
prepareCliDeepScanSession,
|
|
555
|
-
|
|
604
|
+
readDeepScanParentCyberAccessProgram,
|
|
605
|
+
readDeepScanRuntimeConfig,
|
|
606
|
+
readDeepScanRuntimeConfigWithOrigins
|
|
556
607
|
};
|
|
Binary file
|
|
Binary file
|
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
"type": "string",
|
|
13
13
|
"minLength": 1,
|
|
14
14
|
"maxLength": 4096,
|
|
15
|
-
"pattern": "^(?:\\./)?(?!/)(?![A-Za-z]:)(
|
|
15
|
+
"pattern": "^(?:\\./)?(?!/)(?![A-Za-z]:)(?![\\s\\S]*\\\\)(?![\\s\\S]*(?:^|/)\\.\\.?(/|$))(?![\\s\\S]*//)(?![\\s\\S]*[\\u0000-\\u001f\\u007f])(?![\\s\\S]*\\/$)[\\s\\S]+$"
|
|
16
16
|
},
|
|
17
17
|
"candidateId": {
|
|
18
18
|
"type": "string",
|
|
@@ -1256,6 +1256,7 @@ def _recover_unsealed_coverage(
|
|
|
1256
1256
|
warnings.append("Recovered malformed Deep Scan inventory strategy.")
|
|
1257
1257
|
|
|
1258
1258
|
surface_ids: set[str] = set()
|
|
1259
|
+
receipt_paths: dict[str, str] = {}
|
|
1259
1260
|
for field, label in (
|
|
1260
1261
|
("surfaces", "coverage surface"),
|
|
1261
1262
|
("explicitExclusions", "coverage exclusion"),
|
|
@@ -1276,6 +1277,7 @@ def _recover_unsealed_coverage(
|
|
|
1276
1277
|
if not isinstance(item, dict):
|
|
1277
1278
|
raise ContractError(f"{context}: expected an object")
|
|
1278
1279
|
if field == "surfaces":
|
|
1280
|
+
surface_receipt_paths = dict(receipt_paths)
|
|
1279
1281
|
surface_id = _require_str(item, "id", context)
|
|
1280
1282
|
if surface_id in surface_ids:
|
|
1281
1283
|
raise ContractError(f"{context}.id: duplicate surface id")
|
|
@@ -1310,6 +1312,11 @@ def _recover_unsealed_coverage(
|
|
|
1310
1312
|
f"{ref_context}: expected a file under artifacts/"
|
|
1311
1313
|
)
|
|
1312
1314
|
_require_scan_local_file(scan_dir, normalized_ref, ref_context)
|
|
1315
|
+
key = normalized_ref.lower()
|
|
1316
|
+
previous = surface_receipt_paths.get(key)
|
|
1317
|
+
if previous is not None and previous != normalized_ref:
|
|
1318
|
+
raise ContractError(f"{ref_context}: duplicate artifact path")
|
|
1319
|
+
surface_receipt_paths[key] = normalized_ref
|
|
1313
1320
|
except ContractError as exc:
|
|
1314
1321
|
warnings.append(
|
|
1315
1322
|
f"Skipped malformed coverage receipt "
|
|
@@ -1332,6 +1339,7 @@ def _recover_unsealed_coverage(
|
|
|
1332
1339
|
|
|
1333
1340
|
if field == "surfaces":
|
|
1334
1341
|
surface_ids.add(surface_id)
|
|
1342
|
+
receipt_paths = surface_receipt_paths
|
|
1335
1343
|
recovered.append(item)
|
|
1336
1344
|
|
|
1337
1345
|
coverage[field] = recovered
|
|
@@ -2787,7 +2795,10 @@ def write_sarif_projection(
|
|
|
2787
2795
|
scan_dir: Path, source_root: Path | None = None, schema_dir: Path | None = None
|
|
2788
2796
|
) -> None:
|
|
2789
2797
|
sarif = build_sarif_projection(scan_dir, source_root, schema_dir)
|
|
2790
|
-
|
|
2798
|
+
relative_path = EXPORT_PATHS["sarif"]
|
|
2799
|
+
write_export_output(
|
|
2800
|
+
scan_dir, scan_dir / relative_path, "sarif", _contract_json_bytes(relative_path, sarif)
|
|
2801
|
+
)
|
|
2791
2802
|
|
|
2792
2803
|
|
|
2793
2804
|
def csv_cell(value: Any) -> Any:
|
|
@@ -12,7 +12,7 @@ from pathlib import Path
|
|
|
12
12
|
|
|
13
13
|
# Some plugin hosts launch Python with safe-path isolation enabled.
|
|
14
14
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
15
|
-
from workbench_target import git_blob_samples, git_command
|
|
15
|
+
from workbench_target import SOURCE_METADATA_DIRECTORIES, git_blob_samples, git_command
|
|
16
16
|
|
|
17
17
|
|
|
18
18
|
class InventoryError(ValueError):
|
|
@@ -94,11 +94,11 @@ def generate_in_scope_files(repository: Path, scope: str, output: Path) -> int:
|
|
|
94
94
|
"--hidden",
|
|
95
95
|
"--path-separator",
|
|
96
96
|
"/",
|
|
97
|
-
# Prune
|
|
97
|
+
# Prune metadata and its contents even when the scope starts inside it.
|
|
98
98
|
"--glob",
|
|
99
|
-
"
|
|
99
|
+
"!**/{.git,.svn}",
|
|
100
100
|
"--glob",
|
|
101
|
-
"
|
|
101
|
+
"!**/{.git,.svn}/**",
|
|
102
102
|
"--",
|
|
103
103
|
scope,
|
|
104
104
|
]
|
|
@@ -121,7 +121,7 @@ def generate_in_scope_files(repository: Path, scope: str, output: Path) -> int:
|
|
|
121
121
|
message = f"{message}: {detail}"
|
|
122
122
|
raise InventoryError(message)
|
|
123
123
|
|
|
124
|
-
if (
|
|
124
|
+
if any((parent / ".git").exists() for parent in (repository, *repository.parents)):
|
|
125
125
|
try:
|
|
126
126
|
tracked = git_command(
|
|
127
127
|
repository,
|
|
@@ -146,7 +146,7 @@ def generate_in_scope_files(repository: Path, scope: str, output: Path) -> int:
|
|
|
146
146
|
inventory.seek(0)
|
|
147
147
|
rows: list[bytes] = []
|
|
148
148
|
for path in inventory.read().split(b"\0"):
|
|
149
|
-
if not path:
|
|
149
|
+
if not path or not SOURCE_METADATA_DIRECTORIES.isdisjoint(os.fsdecode(path).split("/")):
|
|
150
150
|
continue
|
|
151
151
|
if b"\n" in path or b"\r" in path:
|
|
152
152
|
raise InventoryError(
|
|
@@ -31,7 +31,12 @@ from rank_preview import (
|
|
|
31
31
|
preview_for,
|
|
32
32
|
preview_for_bytes,
|
|
33
33
|
)
|
|
34
|
-
from workbench_target import
|
|
34
|
+
from workbench_target import (
|
|
35
|
+
SOURCE_METADATA_DIRECTORIES,
|
|
36
|
+
git_blob_samples,
|
|
37
|
+
git_command,
|
|
38
|
+
git_directory_snapshot_paths,
|
|
39
|
+
)
|
|
35
40
|
|
|
36
41
|
JsonRow = dict[str, object]
|
|
37
42
|
|
|
@@ -155,11 +160,11 @@ def scope_candidates(repo: Path, scope_path: Path) -> Iterable[Path]:
|
|
|
155
160
|
"--hidden",
|
|
156
161
|
"--no-require-git",
|
|
157
162
|
"--null",
|
|
158
|
-
# Also exclude descendants when the scope starts inside .
|
|
163
|
+
# Also exclude descendants when the scope starts inside metadata.
|
|
159
164
|
"--glob",
|
|
160
|
-
"
|
|
165
|
+
"!**/{.git,.svn}",
|
|
161
166
|
"--glob",
|
|
162
|
-
"
|
|
167
|
+
"!**/{.git,.svn}/**",
|
|
163
168
|
"--",
|
|
164
169
|
str(scope_path.relative_to(repo)),
|
|
165
170
|
]
|
|
@@ -207,7 +212,7 @@ def make_repo_scope_input(args: argparse.Namespace) -> None:
|
|
|
207
212
|
relative = path.resolve(strict=True).relative_to(repo)
|
|
208
213
|
except (OSError, ValueError):
|
|
209
214
|
continue
|
|
210
|
-
if
|
|
215
|
+
if not SOURCE_METADATA_DIRECTORIES.isdisjoint(relative.parts):
|
|
211
216
|
continue
|
|
212
217
|
rows_by_path.setdefault(relative.as_posix(), {"path": relative.as_posix()})
|
|
213
218
|
|
|
@@ -276,7 +281,8 @@ def git_changed_paths(repo: Path, base: str, head: str, mode: str) -> list[tuple
|
|
|
276
281
|
return sorted(
|
|
277
282
|
(path, status)
|
|
278
283
|
for path, status in combined.items()
|
|
279
|
-
if
|
|
284
|
+
if SOURCE_METADATA_DIRECTORIES.isdisjoint(path.relative_to(repo).parts)
|
|
285
|
+
and (status == "D" or (not path.is_symlink() and path.is_file()))
|
|
280
286
|
)
|
|
281
287
|
raise SystemExit(f"Unknown diff mode: {mode}")
|
|
282
288
|
|
|
@@ -47,7 +47,8 @@ def _strings(value: Any) -> list[str]:
|
|
|
47
47
|
|
|
48
48
|
def _path_label(value: str) -> str:
|
|
49
49
|
if "," in value or value != " ".join(value.split()) or any(ord(c) < 32 for c in value):
|
|
50
|
-
|
|
50
|
+
quoted = json.dumps(value, ensure_ascii=False)
|
|
51
|
+
return "".join(c if c.isprintable() else json.dumps(c)[1:-1] for c in quoted)
|
|
51
52
|
return value
|
|
52
53
|
|
|
53
54
|
|
|
@@ -129,9 +129,7 @@ def render_threat_model(model: dict[str, Any], provenance: dict[str, Any] | None
|
|
|
129
129
|
):
|
|
130
130
|
if provenance.get(key):
|
|
131
131
|
footer.append(f"- {label}: {text(str(provenance[key]), 'not recorded')}")
|
|
132
|
-
footer.append(
|
|
133
|
-
f"- Model origin: {text(str(model.get('origin', 'not recorded')), 'not recorded')}"
|
|
134
|
-
)
|
|
132
|
+
footer.append(f"- Model origin: {text(model.get('origin'), 'not recorded')}")
|
|
135
133
|
footer.extend(_scope_lines(model.get("scope"), "Model scope", text))
|
|
136
134
|
scan_scope = provenance.get("scanScope")
|
|
137
135
|
if isinstance(scan_scope, dict):
|
|
@@ -0,0 +1,234 @@
|
|
|
1
|
+
"""Project saved worker coverage and reconcile its published parent copies."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import copy
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from finalize_scan_contract import (
|
|
12
|
+
ContractError,
|
|
13
|
+
_normalize_unsealed_open_questions,
|
|
14
|
+
_require_portable_relative_path,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _saved_coverage_id(item: dict[str, Any]) -> str:
|
|
19
|
+
if candidate_id := item.get("candidateId"):
|
|
20
|
+
return candidate_id
|
|
21
|
+
payload = json.dumps(
|
|
22
|
+
item, ensure_ascii=True, allow_nan=False, sort_keys=True, separators=(",", ":")
|
|
23
|
+
).encode()
|
|
24
|
+
return f"saved-{hashlib.sha256(payload).hexdigest()[:16]}"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _project_worker_coverage_row(item: Any, worker_id: str | None, field: str) -> Any:
|
|
28
|
+
row = copy.deepcopy(item)
|
|
29
|
+
if worker_id is None or not isinstance(row, dict):
|
|
30
|
+
return row
|
|
31
|
+
row["workerId"] = worker_id
|
|
32
|
+
if field == "openQuestions":
|
|
33
|
+
return row
|
|
34
|
+
# Reconcile raw checkpoint identities first. Anonymous output rows use the
|
|
35
|
+
# existing content identity, never their position in an earlier snapshot.
|
|
36
|
+
row.setdefault("id", _saved_coverage_id(row))
|
|
37
|
+
if isinstance(row.get("id"), str):
|
|
38
|
+
row["id"] = f"{worker_id}/{row['id']}"
|
|
39
|
+
if field == "deferred" and isinstance(row.get("surfaceIds"), list):
|
|
40
|
+
row["surfaceIds"] = [
|
|
41
|
+
f"{worker_id}/{identity}" if isinstance(identity, str) else identity
|
|
42
|
+
for identity in row["surfaceIds"]
|
|
43
|
+
]
|
|
44
|
+
return row
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _coverage_comparison_row(row: Any, field: str) -> Any:
|
|
48
|
+
if field == "surfaces" and isinstance(row, dict):
|
|
49
|
+
row = {"receiptRefs": [], **row}
|
|
50
|
+
if isinstance(row["receiptRefs"], list):
|
|
51
|
+
refs = []
|
|
52
|
+
for ref in row["receiptRefs"]:
|
|
53
|
+
if isinstance(ref, str):
|
|
54
|
+
try:
|
|
55
|
+
ref = _require_portable_relative_path(ref, "Saved coverage receipt")
|
|
56
|
+
except ContractError:
|
|
57
|
+
pass # Leave malformed refs to the existing per-record recovery.
|
|
58
|
+
refs.append(ref)
|
|
59
|
+
row["receiptRefs"] = refs
|
|
60
|
+
return row
|
|
61
|
+
if field == "openQuestions":
|
|
62
|
+
coverage = {"openQuestions": [copy.deepcopy(row)]}
|
|
63
|
+
_normalize_unsealed_open_questions(coverage)
|
|
64
|
+
return next(iter(coverage["openQuestions"]), row)
|
|
65
|
+
return row
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _aggregate_worker_coverage_rows(rows: list[Any], worker_id: str, field: str) -> list[Any]:
|
|
69
|
+
"""Match accepted aggregate IDs without changing raw recovery identities."""
|
|
70
|
+
reserved = {
|
|
71
|
+
row["id"] for row in rows if isinstance(row, dict) and isinstance(row.get("id"), str)
|
|
72
|
+
}
|
|
73
|
+
emitted: set[str] = set()
|
|
74
|
+
projected = []
|
|
75
|
+
for index, original in enumerate(rows):
|
|
76
|
+
row = copy.deepcopy(original)
|
|
77
|
+
if isinstance(row, dict) and field != "openQuestions":
|
|
78
|
+
generated = not isinstance(row.get("id"), str)
|
|
79
|
+
identity = str(index) if generated else row["id"]
|
|
80
|
+
base = identity
|
|
81
|
+
if generated or identity in emitted:
|
|
82
|
+
suffix = 2
|
|
83
|
+
while identity in reserved:
|
|
84
|
+
identity = f"{base}-{suffix}"
|
|
85
|
+
suffix += 1
|
|
86
|
+
reserved.add(identity)
|
|
87
|
+
emitted.add(identity)
|
|
88
|
+
row["id"] = identity
|
|
89
|
+
projected.append(
|
|
90
|
+
_coverage_comparison_row(_project_worker_coverage_row(row, worker_id, field), field)
|
|
91
|
+
)
|
|
92
|
+
return projected
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _prepare_worker_coverage_recovery(
|
|
96
|
+
sources: list[tuple[str, dict[str, Any], str | None]],
|
|
97
|
+
parent: dict[str, Any] | None,
|
|
98
|
+
source_order: dict[str, tuple[int, int]],
|
|
99
|
+
*,
|
|
100
|
+
enabled: bool,
|
|
101
|
+
) -> tuple[
|
|
102
|
+
bool, list[tuple[dict[str, Any], str, dict[str, Any], dict[str, Any], dict[str, Any] | None]]
|
|
103
|
+
]:
|
|
104
|
+
"""Put bound worker evidence in parent coordinates without changing source files."""
|
|
105
|
+
workers = {owner for _, _, owner in sources if owner is not None}
|
|
106
|
+
parents = [
|
|
107
|
+
draft
|
|
108
|
+
for _, draft, owner in sorted(
|
|
109
|
+
sources, key=lambda source: source_order[source[0]], reverse=True
|
|
110
|
+
)
|
|
111
|
+
if owner is None
|
|
112
|
+
]
|
|
113
|
+
if parent is not None:
|
|
114
|
+
# Prefer the selected parent over historical copies of a task's surface.
|
|
115
|
+
parents.insert(0, parent)
|
|
116
|
+
fields = ("surfaces", "explicitExclusions", "deferred", "openQuestions")
|
|
117
|
+
if not enabled:
|
|
118
|
+
enabled = any(
|
|
119
|
+
isinstance(row, dict)
|
|
120
|
+
and isinstance(row.get("workerId"), str)
|
|
121
|
+
and row["workerId"] in workers
|
|
122
|
+
and (
|
|
123
|
+
field == "openQuestions"
|
|
124
|
+
or (isinstance(row.get("id"), str) and row["id"].startswith(f"{row['workerId']}/"))
|
|
125
|
+
)
|
|
126
|
+
for draft in parents
|
|
127
|
+
for field in fields
|
|
128
|
+
for row in (
|
|
129
|
+
draft["coverage"].get(field, [])
|
|
130
|
+
if isinstance(draft["coverage"].get(field, []), list)
|
|
131
|
+
else []
|
|
132
|
+
)
|
|
133
|
+
)
|
|
134
|
+
if not enabled:
|
|
135
|
+
return False, []
|
|
136
|
+
projected: dict[str, list[Any]] = {field: [] for field in fields}
|
|
137
|
+
surface_sources: list[tuple[dict[str, Any], dict[str, Any]]] = []
|
|
138
|
+
task_sources: list[tuple[dict[str, Any], dict[str, Any]]] = []
|
|
139
|
+
surface_links: list[
|
|
140
|
+
tuple[dict[str, Any], str, dict[str, Any], dict[str, Any], dict[str, Any] | None]
|
|
141
|
+
] = []
|
|
142
|
+
for relative, draft, owner in sources:
|
|
143
|
+
if owner is None:
|
|
144
|
+
continue
|
|
145
|
+
directory = Path(relative).parent
|
|
146
|
+
if directory.name == "checkpoints":
|
|
147
|
+
directory = directory.parent
|
|
148
|
+
for field in fields:
|
|
149
|
+
rows = draft["coverage"].get(field, [])
|
|
150
|
+
if not isinstance(rows, list):
|
|
151
|
+
continue
|
|
152
|
+
for index, row in enumerate(rows):
|
|
153
|
+
if field == "openQuestions" and isinstance(row, str):
|
|
154
|
+
row = {"question": row}
|
|
155
|
+
rows[index] = row
|
|
156
|
+
if not isinstance(row, dict):
|
|
157
|
+
continue
|
|
158
|
+
row["workerId"] = owner
|
|
159
|
+
if field == "surfaces" and isinstance(row.get("receiptRefs"), list):
|
|
160
|
+
refs = []
|
|
161
|
+
for ref in row["receiptRefs"]:
|
|
162
|
+
if isinstance(ref, str):
|
|
163
|
+
try:
|
|
164
|
+
normalized = _require_portable_relative_path(
|
|
165
|
+
ref, "Worker coverage receipt"
|
|
166
|
+
)
|
|
167
|
+
except ContractError:
|
|
168
|
+
pass # Leave malformed refs to the existing per-record recovery.
|
|
169
|
+
else:
|
|
170
|
+
if normalized.startswith("artifacts/"):
|
|
171
|
+
ref = f"{directory.as_posix()}/{normalized}"
|
|
172
|
+
refs.append(ref)
|
|
173
|
+
row["receiptRefs"] = refs
|
|
174
|
+
if field == "surfaces":
|
|
175
|
+
row.setdefault("receiptRefs", [])
|
|
176
|
+
aggregate_rows = _aggregate_worker_coverage_rows(rows, owner, field)
|
|
177
|
+
projected[field].extend(aggregate_rows)
|
|
178
|
+
if field in {"surfaces", "deferred"}:
|
|
179
|
+
originals = surface_sources if field == "surfaces" else task_sources
|
|
180
|
+
originals.extend(
|
|
181
|
+
(aggregate, row)
|
|
182
|
+
for aggregate, row in zip(aggregate_rows, rows)
|
|
183
|
+
if isinstance(aggregate, dict) and isinstance(row, dict)
|
|
184
|
+
)
|
|
185
|
+
# A canonical aggregate contains copies of the same worker evidence. Let
|
|
186
|
+
# the existing per-worker chronology reconcile those originals exactly once.
|
|
187
|
+
# Keep parent-only evidence when its bound worker source is unavailable.
|
|
188
|
+
for draft in parents:
|
|
189
|
+
if not draft["coverage"]:
|
|
190
|
+
continue
|
|
191
|
+
for field in fields:
|
|
192
|
+
rows = draft["coverage"].get(field, [])
|
|
193
|
+
if isinstance(rows, list):
|
|
194
|
+
if field == "surfaces":
|
|
195
|
+
tasks = draft["coverage"].get("deferred", [])
|
|
196
|
+
for row in rows:
|
|
197
|
+
if not isinstance(row, dict) or not isinstance(row.get("id"), str):
|
|
198
|
+
continue
|
|
199
|
+
matching_sources = [
|
|
200
|
+
original
|
|
201
|
+
for aggregate, original in surface_sources
|
|
202
|
+
if aggregate == _coverage_comparison_row(row, field)
|
|
203
|
+
]
|
|
204
|
+
if not matching_sources:
|
|
205
|
+
continue
|
|
206
|
+
for task in tasks if isinstance(tasks, list) else []:
|
|
207
|
+
if (
|
|
208
|
+
isinstance(task, dict)
|
|
209
|
+
and isinstance(task.get("surfaceIds"), list)
|
|
210
|
+
and row["id"] in task["surfaceIds"]
|
|
211
|
+
):
|
|
212
|
+
surface_links.extend(
|
|
213
|
+
(
|
|
214
|
+
task,
|
|
215
|
+
row["id"],
|
|
216
|
+
source,
|
|
217
|
+
row,
|
|
218
|
+
next(
|
|
219
|
+
(
|
|
220
|
+
original
|
|
221
|
+
for aggregate, original in task_sources
|
|
222
|
+
if aggregate == task
|
|
223
|
+
),
|
|
224
|
+
None,
|
|
225
|
+
),
|
|
226
|
+
)
|
|
227
|
+
for source in matching_sources
|
|
228
|
+
)
|
|
229
|
+
draft["coverage"][field] = [
|
|
230
|
+
row
|
|
231
|
+
for row in rows
|
|
232
|
+
if _coverage_comparison_row(row, field) not in projected[field]
|
|
233
|
+
]
|
|
234
|
+
return True, surface_links
|
|
@@ -309,13 +309,11 @@ def parse_args(description: str) -> argparse.Namespace:
|
|
|
309
309
|
subparsers.add_parser("database-info")
|
|
310
310
|
subparsers.add_parser("finding-workflow")
|
|
311
311
|
subparsers.add_parser("local-dedupe")
|
|
312
|
-
|
|
313
|
-
if "
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
arguments[index] = "--user-context=" + sys.stdin.buffer.read().decode("utf-8")
|
|
318
|
-
return parser.parse_args(arguments)
|
|
312
|
+
args = parser.parse_args()
|
|
313
|
+
if getattr(args, "user_context_stdin", False):
|
|
314
|
+
args.user_context = sys.stdin.buffer.read().decode("utf-8")
|
|
315
|
+
args.user_context_stdin = False
|
|
316
|
+
return args
|
|
319
317
|
|
|
320
318
|
|
|
321
319
|
if __name__ == "__main__":
|
|
@@ -1186,12 +1186,18 @@ def finalize_scan_usage(connection: sqlite3.Connection, args: argparse.Namespace
|
|
|
1186
1186
|
completed_turn_id=args.turn_id,
|
|
1187
1187
|
)
|
|
1188
1188
|
if usage["coverage"] == "unavailable":
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1189
|
+
if not usage.get("accountRateLimits"):
|
|
1190
|
+
return scan_context(connection, scan_id)
|
|
1191
|
+
usage = {**stored["usage"], "accountRateLimits": usage["accountRateLimits"]}
|
|
1192
|
+
else:
|
|
1193
|
+
if args.turn_status != "completed":
|
|
1194
|
+
if usage["coverage"] == "complete":
|
|
1195
|
+
usage["coverage"] = "partial"
|
|
1196
|
+
usage["warnings"] = [
|
|
1197
|
+
*usage.get("warnings", []),
|
|
1198
|
+
f"scan_turn_{args.turn_status}",
|
|
1199
|
+
]
|
|
1200
|
+
usage["completion"] = {**completion, "finalized": True}
|
|
1195
1201
|
cost_json = parse_scan_cost(json.dumps({**stored, "usage": usage}))
|
|
1196
1202
|
with connection:
|
|
1197
1203
|
connection.execute(
|
|
@@ -203,15 +203,18 @@ def list_repositories(
|
|
|
203
203
|
connection: sqlite3.Connection,
|
|
204
204
|
args: argparse.Namespace | None = None,
|
|
205
205
|
) -> dict[str, Any]:
|
|
206
|
-
|
|
207
|
-
scans_by_id = {scan["scanId"]: scan for scan in scans}
|
|
208
|
-
scan_count_by_target = dict(Counter(scan["targetId"] for scan in scans))
|
|
209
|
-
|
|
206
|
+
scan_count_by_target = Counter()
|
|
210
207
|
latest_scan_by_target: dict[str, dict[str, Any]] = {}
|
|
211
|
-
for
|
|
212
|
-
|
|
208
|
+
for scan in (
|
|
209
|
+
scan_history.scan_summary(row)
|
|
210
|
+
for row in connection.execute(
|
|
211
|
+
f"""{scan_history.SCAN_SUMMARY_QUERY}
|
|
212
|
+
ORDER BY julianday(upper(scans.started_at)) DESC, scans.id DESC
|
|
213
|
+
"""
|
|
214
|
+
).fetchall()
|
|
213
215
|
):
|
|
214
|
-
|
|
216
|
+
scan_count_by_target[scan["targetId"]] += 1
|
|
217
|
+
latest_scan_by_target.setdefault(scan["targetId"], scan)
|
|
215
218
|
|
|
216
219
|
open_findings_by_target = Counter(
|
|
217
220
|
row["target_id"] for row in _indexed_findings(connection) if row["status"] == "open"
|
|
@@ -332,7 +332,7 @@ def export_findings(
|
|
|
332
332
|
"workspace": db.workspace_state(connection, scan["workspace_id"]),
|
|
333
333
|
}
|
|
334
334
|
try:
|
|
335
|
-
manifest,
|
|
335
|
+
manifest, findings_document, _ = finalize_scan(
|
|
336
336
|
scan_dir,
|
|
337
337
|
expected_coverage_mode=db.expected_coverage_mode(scan),
|
|
338
338
|
)
|
|
@@ -350,7 +350,8 @@ def export_findings(
|
|
|
350
350
|
raise SystemExit(str(exc)) from exc
|
|
351
351
|
path = db.artifact_path(scan_dir, "exports/results.sarif", required=True)
|
|
352
352
|
else:
|
|
353
|
-
path = write_csv_export(db, connection, scan)
|
|
353
|
+
path = write_csv_export(db, connection, scan, findings_document)
|
|
354
|
+
del findings_document
|
|
354
355
|
if path is None:
|
|
355
356
|
raise SystemExit(f"Could not export Codex Security findings as {args.format.upper()}.")
|
|
356
357
|
return {
|
|
@@ -364,6 +365,7 @@ def write_csv_export(
|
|
|
364
365
|
db: Any,
|
|
365
366
|
connection: sqlite3.Connection,
|
|
366
367
|
scan: sqlite3.Row,
|
|
368
|
+
findings_document: dict[str, Any],
|
|
367
369
|
) -> Path:
|
|
368
370
|
scan_dir = db.require_canonical_scan_directory(Path(scan["scan_dir"]))
|
|
369
371
|
output = io.StringIO(newline="")
|
|
@@ -371,7 +373,6 @@ def write_csv_export(
|
|
|
371
373
|
deep_scan = scan["mode"] == "deep"
|
|
372
374
|
candidate_ids_by_occurrence: dict[str, str] = {}
|
|
373
375
|
if deep_scan:
|
|
374
|
-
findings_document = db.read_json_object(scan_dir / db.ARTIFACTS["findings"])
|
|
375
376
|
findings = findings_document.get("findings")
|
|
376
377
|
if not isinstance(findings, list):
|
|
377
378
|
raise SystemExit("findings.json must contain a findings array.")
|
|
@@ -147,7 +147,11 @@ def cancel_finding_remediation_request(
|
|
|
147
147
|
connection.execute(
|
|
148
148
|
"""
|
|
149
149
|
UPDATE finding_remediation_attempts
|
|
150
|
-
SET
|
|
150
|
+
SET state = CASE
|
|
151
|
+
WHEN pending_action = 'verify' AND state = 'verifying' THEN 'applied'
|
|
152
|
+
ELSE state
|
|
153
|
+
END,
|
|
154
|
+
pending_action = NULL, pending_action_claimed_at = NULL,
|
|
151
155
|
pending_action_claim_token = NULL, pending_action_delivered_at = NULL,
|
|
152
156
|
version = version + 1, updated_at = ?
|
|
153
157
|
WHERE request_id = ? AND pending_action_claim_token = ?
|