@openai/codex-security 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +93 -2
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/permission-profile-preflight.mjs +60 -9
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/schemas/definitions/artifact-common.schema.json +1 -1
- package/_bundled_plugin/scripts/finalize_scan_contract.py +12 -1
- package/_bundled_plugin/scripts/generate_in_scope_files.py +6 -6
- package/_bundled_plugin/scripts/generate_rank_input.py +12 -6
- package/_bundled_plugin/scripts/report_projection.py +2 -1
- package/_bundled_plugin/scripts/threat_model_projection.py +1 -3
- package/_bundled_plugin/scripts/workbench/saved_coverage.py +234 -0
- package/_bundled_plugin/scripts/workbench_cli.py +5 -7
- package/_bundled_plugin/scripts/workbench_db.py +12 -6
- package/_bundled_plugin/scripts/workbench_finding_workflows.py +2 -0
- package/_bundled_plugin/scripts/workbench_native_indexes.py +10 -7
- package/_bundled_plugin/scripts/workbench_publication.py +4 -3
- package/_bundled_plugin/scripts/workbench_remediation.py +5 -1
- package/_bundled_plugin/scripts/workbench_saved_results.py +542 -148
- package/_bundled_plugin/scripts/workbench_scan_history.py +79 -78
- package/_bundled_plugin/scripts/workbench_scan_usage.py +61 -13
- package/_bundled_plugin/scripts/workbench_target.py +21 -7
- package/_bundled_plugin/scripts/workbench_validation.py +21 -18
- package/_bundled_plugin/shared/workbench-migrations.json +26 -0
- package/_bundled_plugin/skills/fix-finding/SKILL.md +16 -8
- package/_bundled_plugin/skills/publish-findings/SKILL.md +130 -0
- package/_bundled_plugin/skills/publish-findings/agents/openai.yaml +6 -0
- package/_bundled_plugin/skills/verify-fix/SKILL.md +4 -2
- package/dist/api.d.ts +4 -1
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +233 -91
- package/dist/api.js.map +1 -1
- package/dist/classify-scan-severity.d.ts.map +1 -1
- package/dist/classify-scan-severity.js +16 -18
- package/dist/classify-scan-severity.js.map +1 -1
- package/dist/classify-severity-records.d.ts +5 -2
- package/dist/classify-severity-records.d.ts.map +1 -1
- package/dist/classify-severity-records.js +40 -34
- package/dist/classify-severity-records.js.map +1 -1
- package/dist/classify-severity.d.ts +5 -1
- package/dist/classify-severity.d.ts.map +1 -1
- package/dist/classify-severity.js +20 -30
- package/dist/classify-severity.js.map +1 -1
- package/dist/cli-daybreak.d.ts +5 -0
- package/dist/cli-daybreak.d.ts.map +1 -0
- package/dist/cli-daybreak.js +89 -0
- package/dist/cli-daybreak.js.map +1 -0
- package/dist/cli-help.d.ts.map +1 -1
- package/dist/cli-help.js +1 -0
- package/dist/cli-help.js.map +1 -1
- package/dist/cli.d.ts +12 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +1224 -596
- package/dist/cli.js.map +1 -1
- package/dist/cloud-endpoint.d.ts +6 -0
- package/dist/cloud-endpoint.d.ts.map +1 -0
- package/dist/cloud-endpoint.js +51 -0
- package/dist/cloud-endpoint.js.map +1 -0
- package/dist/cloud-publish.d.ts.map +1 -1
- package/dist/cloud-publish.js +7 -5
- package/dist/cloud-publish.js.map +1 -1
- package/dist/codeowners.js +2 -2
- package/dist/codeowners.js.map +1 -1
- package/dist/component-plan.d.ts.map +1 -1
- package/dist/component-plan.js +3 -2
- package/dist/component-plan.js.map +1 -1
- package/dist/component-scan.js +2 -2
- package/dist/component-scan.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +5 -1
- package/dist/config.js.map +1 -1
- package/dist/cost.d.ts +2 -1
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +17 -8
- package/dist/cost.js.map +1 -1
- package/dist/custom-validation.d.ts.map +1 -1
- package/dist/custom-validation.js +2 -5
- package/dist/custom-validation.js.map +1 -1
- package/dist/deduplication/codex-review.d.ts.map +1 -1
- package/dist/deduplication/codex-review.js +8 -6
- package/dist/deduplication/codex-review.js.map +1 -1
- package/dist/deduplication/local.d.ts.map +1 -1
- package/dist/deduplication/local.js +5 -2
- package/dist/deduplication/local.js.map +1 -1
- package/dist/deduplication/retry.d.ts.map +1 -1
- package/dist/deduplication/retry.js +5 -0
- package/dist/deduplication/retry.js.map +1 -1
- package/dist/deduplication/scan.js +5 -1
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/deep-scan.d.ts +6 -0
- package/dist/deep-scan.d.ts.map +1 -1
- package/dist/deep-scan.js +8 -6
- package/dist/deep-scan.js.map +1 -1
- package/dist/default-severity-rubric.d.ts +3 -0
- package/dist/default-severity-rubric.d.ts.map +1 -0
- package/dist/default-severity-rubric.js +67 -0
- package/dist/default-severity-rubric.js.map +1 -0
- package/dist/endor-findings.d.ts +7 -0
- package/dist/endor-findings.d.ts.map +1 -0
- package/dist/endor-findings.js +233 -0
- package/dist/endor-findings.js.map +1 -0
- package/dist/external-findings-import.d.ts +5 -0
- package/dist/external-findings-import.d.ts.map +1 -0
- package/dist/external-findings-import.js +38 -0
- package/dist/external-findings-import.js.map +1 -0
- package/dist/external-findings-local.d.ts +63 -0
- package/dist/external-findings-local.d.ts.map +1 -0
- package/dist/external-findings-local.js +337 -0
- package/dist/external-findings-local.js.map +1 -0
- package/dist/external-findings-publish.d.ts +73 -0
- package/dist/external-findings-publish.d.ts.map +1 -0
- package/dist/external-findings-publish.js +713 -0
- package/dist/external-findings-publish.js.map +1 -0
- package/dist/external-findings-read.d.ts +6 -0
- package/dist/external-findings-read.d.ts.map +1 -0
- package/dist/external-findings-read.js +88 -0
- package/dist/external-findings-read.js.map +1 -0
- package/dist/external-findings.d.ts +21 -0
- package/dist/external-findings.d.ts.map +1 -0
- package/dist/external-findings.js +2 -0
- package/dist/external-findings.js.map +1 -0
- package/dist/external-import-contract.d.ts +13 -0
- package/dist/external-import-contract.d.ts.map +1 -0
- package/dist/external-import-contract.js +254 -0
- package/dist/external-import-contract.js.map +1 -0
- package/dist/external-import-models.d.ts +362 -0
- package/dist/external-import-models.d.ts.map +1 -0
- package/dist/external-import-models.js +3 -0
- package/dist/external-import-models.js.map +1 -0
- package/dist/finding-workflow.d.ts +1 -1
- package/dist/finding-workflow.d.ts.map +1 -1
- package/dist/finding-workflow.js +2 -1
- package/dist/finding-workflow.js.map +1 -1
- package/dist/findings-import.d.ts.map +1 -1
- package/dist/findings-import.js +23 -3
- package/dist/findings-import.js.map +1 -1
- package/dist/github-advisories.d.ts +386 -0
- package/dist/github-advisories.d.ts.map +1 -0
- package/dist/github-advisories.js +131 -0
- package/dist/github-advisories.js.map +1 -0
- package/dist/github-findings.d.ts +10 -0
- package/dist/github-findings.d.ts.map +1 -0
- package/dist/github-findings.js +256 -0
- package/dist/github-findings.js.map +1 -0
- package/dist/host-protocol.d.ts +1 -0
- package/dist/host-protocol.d.ts.map +1 -1
- package/dist/host-protocol.js +1 -1
- package/dist/host-protocol.js.map +1 -1
- package/dist/hosted-remediation-protocol.d.ts +6 -0
- package/dist/hosted-remediation-protocol.d.ts.map +1 -0
- package/dist/hosted-remediation-protocol.js +32 -0
- package/dist/hosted-remediation-protocol.js.map +1 -0
- package/dist/hosted-remediation.d.ts +93 -0
- package/dist/hosted-remediation.d.ts.map +1 -0
- package/dist/hosted-remediation.js +124 -0
- package/dist/hosted-remediation.js.map +1 -0
- package/dist/hosted-scan.d.ts +16 -0
- package/dist/hosted-scan.d.ts.map +1 -1
- package/dist/hosted-scan.js +14 -2
- package/dist/hosted-scan.js.map +1 -1
- package/dist/import-scan.d.ts +1 -0
- package/dist/import-scan.d.ts.map +1 -1
- package/dist/import-scan.js +5 -1
- package/dist/import-scan.js.map +1 -1
- package/dist/index.d.ts +18 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10 -0
- package/dist/index.js.map +1 -1
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +3 -1
- package/dist/multiscan.js.map +1 -1
- package/dist/project-config-schema.d.ts +2 -0
- package/dist/project-config-schema.d.ts.map +1 -1
- package/dist/project-config-schema.js +1 -0
- package/dist/project-config-schema.js.map +1 -1
- package/dist/project-config.d.ts +1 -0
- package/dist/project-config.d.ts.map +1 -1
- package/dist/project-config.js +15 -9
- package/dist/project-config.js.map +1 -1
- package/dist/publish.d.ts +2 -0
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +9 -0
- package/dist/publish.js.map +1 -1
- package/dist/remediation.d.ts +69 -0
- package/dist/remediation.d.ts.map +1 -0
- package/dist/remediation.js +121 -0
- package/dist/remediation.js.map +1 -0
- package/dist/request-metadata.d.ts +17 -7
- package/dist/request-metadata.d.ts.map +1 -1
- package/dist/request-metadata.js +43 -8
- package/dist/request-metadata.js.map +1 -1
- package/dist/result.d.ts +4 -0
- package/dist/result.d.ts.map +1 -1
- package/dist/result.js +10 -0
- package/dist/result.js.map +1 -1
- package/dist/runtime.d.ts +1 -0
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +24 -15
- package/dist/runtime.js.map +1 -1
- package/dist/saved-scan-bootstrap.d.ts +1 -0
- package/dist/saved-scan-bootstrap.d.ts.map +1 -1
- package/dist/saved-scan-bootstrap.js +16 -4
- package/dist/saved-scan-bootstrap.js.map +1 -1
- package/dist/scan-activity.js +8 -8
- package/dist/scan-activity.js.map +1 -1
- package/dist/scan-comparison.d.ts.map +1 -1
- package/dist/scan-comparison.js +15 -9
- package/dist/scan-comparison.js.map +1 -1
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +12 -6
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-executor.d.ts +5 -1
- package/dist/scan-executor.d.ts.map +1 -1
- package/dist/scan-executor.js +9 -8
- package/dist/scan-executor.js.map +1 -1
- package/dist/scan-owner-suggestions.d.ts +8 -0
- package/dist/scan-owner-suggestions.d.ts.map +1 -0
- package/dist/scan-owner-suggestions.js +33 -0
- package/dist/scan-owner-suggestions.js.map +1 -0
- package/dist/{deep-progress.d.ts → scan-progress.d.ts} +15 -7
- package/dist/scan-progress.d.ts.map +1 -0
- package/dist/scan-progress.js +138 -0
- package/dist/scan-progress.js.map +1 -0
- package/dist/scan-settings.d.ts +1 -0
- package/dist/scan-settings.d.ts.map +1 -1
- package/dist/scan-settings.js +4 -0
- package/dist/scan-settings.js.map +1 -1
- package/dist/scanner-import.d.ts +20 -0
- package/dist/scanner-import.d.ts.map +1 -0
- package/dist/scanner-import.js +204 -0
- package/dist/scanner-import.js.map +1 -0
- package/dist/security-policy-diff.d.ts +2 -0
- package/dist/security-policy-diff.d.ts.map +1 -0
- package/dist/security-policy-diff.js +197 -0
- package/dist/security-policy-diff.js.map +1 -0
- package/dist/security-policy.d.ts +7 -3
- package/dist/security-policy.d.ts.map +1 -1
- package/dist/security-policy.js +11 -57
- package/dist/security-policy.js.map +1 -1
- package/dist/semgrep-findings.d.ts +9 -0
- package/dist/semgrep-findings.d.ts.map +1 -0
- package/dist/semgrep-findings.js +293 -0
- package/dist/semgrep-findings.js.map +1 -0
- package/dist/server/embeddings.d.ts.map +1 -1
- package/dist/server/embeddings.js +4 -0
- package/dist/server/embeddings.js.map +1 -1
- package/dist/server/sqlite-store.d.ts.map +1 -1
- package/dist/server/sqlite-store.js +14 -2
- package/dist/server/sqlite-store.js.map +1 -1
- package/dist/severity-review.d.ts +3 -0
- package/dist/severity-review.d.ts.map +1 -1
- package/dist/severity-review.js +19 -0
- package/dist/severity-review.js.map +1 -1
- package/dist/severity-store.js +11 -3
- package/dist/severity-store.js.map +1 -1
- package/dist/snyk-findings.d.ts +9 -0
- package/dist/snyk-findings.d.ts.map +1 -0
- package/dist/snyk-findings.js +349 -0
- package/dist/snyk-findings.js.map +1 -0
- package/dist/targets.d.ts +2 -2
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +12 -2
- package/dist/targets.js.map +1 -1
- package/dist/timestamps.d.ts +3 -0
- package/dist/timestamps.d.ts.map +1 -0
- package/dist/timestamps.js +14 -0
- package/dist/timestamps.js.map +1 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +13 -2
- package/dist/version.js.map +1 -1
- package/dist/wiz-csv.d.ts +8 -0
- package/dist/wiz-csv.d.ts.map +1 -0
- package/dist/wiz-csv.js +116 -0
- package/dist/wiz-csv.js.map +1 -0
- package/dist/wiz-findings.d.ts +4 -0
- package/dist/wiz-findings.d.ts.map +1 -0
- package/dist/wiz-findings.js +460 -0
- package/dist/wiz-findings.js.map +1 -0
- package/dist/workbench-arguments.d.ts.map +1 -1
- package/dist/workbench-arguments.js +9 -3
- package/dist/workbench-arguments.js.map +1 -1
- package/dist/worker-progress.d.ts +11 -1
- package/dist/worker-progress.d.ts.map +1 -1
- package/dist/worker-progress.js +11 -1
- package/dist/worker-progress.js.map +1 -1
- package/docs/classify-severity-records.md +28 -10
- package/docs/cli.md +788 -46
- package/docs/findings-service.md +7 -2
- package/docs/hosted-remediation.md +134 -0
- package/docs/hosted-scans.md +20 -3
- package/package.json +9 -6
- package/schemas/external-findings.schema.json +1421 -0
- package/schemas/project-config.schema.json +4 -0
- package/dist/deep-progress.d.ts.map +0 -1
- package/dist/deep-progress.js +0 -92
- package/dist/deep-progress.js.map +0 -1
|
@@ -0,0 +1,713 @@
|
|
|
1
|
+
import { hash, randomUUID } from "node:crypto";
|
|
2
|
+
import { chmodSync } from "node:fs";
|
|
3
|
+
import { lstat, mkdir, open, readFile, readdir, rename, rm, } from "node:fs/promises";
|
|
4
|
+
import { createRequire } from "node:module";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
import { setTimeout as delay } from "node:timers/promises";
|
|
7
|
+
import { readCloudCredentials } from "./cloud-publish.js";
|
|
8
|
+
import { cloudBaseUrl, cloudFindingsUrl, DEFAULT_CLOUD_BASE_URL, } from "./cloud-endpoint.js";
|
|
9
|
+
import { CodexSecurityError } from "./errors.js";
|
|
10
|
+
import { codexSecurityStateDirectory, createWindowsOutputDirectory, validatePreparedOutputDir, } from "./runtime.js";
|
|
11
|
+
import { readExternalFindings, } from "./external-findings-read.js";
|
|
12
|
+
import { wizCsvRepositoryUrl } from "./wiz-csv.js";
|
|
13
|
+
import { snykRepositoryUrls } from "./snyk-findings.js";
|
|
14
|
+
import { endorRepositoryUrls } from "./endor-findings.js";
|
|
15
|
+
import { cloudRepositoryUrlKey, validateImportRequest, validateExternalEvidence, validateImportReceipt, validateRepositories, validateSourceReports, validateSourceReport, } from "./external-import-contract.js";
|
|
16
|
+
const MAX_REQUEST_BYTES = 5 * 1024 * 1024;
|
|
17
|
+
/** Preserve accepted work in CLI JSON output when publication cannot finish. */
|
|
18
|
+
export class ExternalPublicationError extends CodexSecurityError {
|
|
19
|
+
result;
|
|
20
|
+
constructor(message, result, cause) {
|
|
21
|
+
super(message, { cause });
|
|
22
|
+
this.result = result;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
// Limit read concurrency to avoid one network round trip per finding in series.
|
|
26
|
+
// Wait for the entire batch on failure so callers never outlive their read tasks.
|
|
27
|
+
async function readInBatches(items, read) {
|
|
28
|
+
const output = [];
|
|
29
|
+
for (let start = 0; start < items.length; start += 4) {
|
|
30
|
+
const batch = await Promise.allSettled(items.slice(start, start + 4).map(read));
|
|
31
|
+
for (const result of batch) {
|
|
32
|
+
if (result.status === "rejected")
|
|
33
|
+
throw result.reason;
|
|
34
|
+
output.push(result.value);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
return output;
|
|
38
|
+
}
|
|
39
|
+
function canonicalJson(value, compareKeys) {
|
|
40
|
+
return JSON.stringify(value, (_key, child) => {
|
|
41
|
+
if (child !== null && typeof child === "object" && !Array.isArray(child)) {
|
|
42
|
+
const record = child;
|
|
43
|
+
return Object.fromEntries(Object.keys(record)
|
|
44
|
+
.sort(compareKeys)
|
|
45
|
+
.map((key) => [key, record[key]]));
|
|
46
|
+
}
|
|
47
|
+
return child;
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
function sameSubmission(left, right) {
|
|
51
|
+
return (left.accountId === right.accountId &&
|
|
52
|
+
canonicalJson(left.requests) === canonicalJson(right.requests));
|
|
53
|
+
}
|
|
54
|
+
function submissionFindings(requests) {
|
|
55
|
+
return requests.flatMap((batch) => batch.items.map(({ source_finding_id, evidence }) => ({
|
|
56
|
+
source_finding_id,
|
|
57
|
+
evidence,
|
|
58
|
+
})));
|
|
59
|
+
}
|
|
60
|
+
function sameFindingSelection(left, right) {
|
|
61
|
+
return (left.length === right.length &&
|
|
62
|
+
left.every((finding, index) => finding.source_finding_id === right[index].source_finding_id));
|
|
63
|
+
}
|
|
64
|
+
async function readOptionalFile(path) {
|
|
65
|
+
return readFile(path, "utf8").catch((error) => {
|
|
66
|
+
if (error.code === "ENOENT")
|
|
67
|
+
return undefined;
|
|
68
|
+
throw error;
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
async function writeAtomicJson(path, value) {
|
|
72
|
+
const temporary = `${path}.${randomUUID()}.tmp`;
|
|
73
|
+
try {
|
|
74
|
+
const file = await open(temporary, "wx", 0o600);
|
|
75
|
+
try {
|
|
76
|
+
await file.writeFile(JSON.stringify(value));
|
|
77
|
+
await file.sync();
|
|
78
|
+
}
|
|
79
|
+
finally {
|
|
80
|
+
await file.close();
|
|
81
|
+
}
|
|
82
|
+
await rename(temporary, path);
|
|
83
|
+
}
|
|
84
|
+
finally {
|
|
85
|
+
await rm(temporary, { force: true });
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
function sourcePath(repository) {
|
|
89
|
+
return `/repository_connectors/${encodeURIComponent(repository.repo_connector_id)}/repositories/${encodeURIComponent(repository.id)}/source_reports`;
|
|
90
|
+
}
|
|
91
|
+
class CloudImportError extends CodexSecurityError {
|
|
92
|
+
status;
|
|
93
|
+
constructor(status, message) {
|
|
94
|
+
super(message);
|
|
95
|
+
this.status = status;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
function validatePublicationReceipt(value, batch) {
|
|
99
|
+
const receipt = validateImportReceipt(value);
|
|
100
|
+
const batchItems = new Map(batch.items.map((item) => [item.client_id, item]));
|
|
101
|
+
if (receipt.id !== batch.request_id ||
|
|
102
|
+
canonicalJson(receipt.repository) !== canonicalJson(batch.repository) ||
|
|
103
|
+
canonicalJson(receipt.source) !== canonicalJson(batch.source) ||
|
|
104
|
+
receipt.item_count !== batch.items.length ||
|
|
105
|
+
receipt.results.length !== batch.items.length ||
|
|
106
|
+
new Set(receipt.results.map((item) => item.client_id)).size !==
|
|
107
|
+
batch.items.length ||
|
|
108
|
+
receipt.results.some((item) => !batchItems.has(item.client_id)))
|
|
109
|
+
throw new CodexSecurityError("Cloud returned a receipt for a different publication.");
|
|
110
|
+
const actual = { created: 0, updated: 0, unchanged: 0, error: 0 };
|
|
111
|
+
for (const result of receipt.results) {
|
|
112
|
+
actual[result.outcome]++;
|
|
113
|
+
if (result.outcome === "error") {
|
|
114
|
+
if (!result.error ||
|
|
115
|
+
result.source_report_id !== null ||
|
|
116
|
+
result.observation_id !== null ||
|
|
117
|
+
result.canonical_finding_id !== null ||
|
|
118
|
+
result.version !== null)
|
|
119
|
+
throw new CodexSecurityError("Cloud returned inconsistent item error fields.");
|
|
120
|
+
}
|
|
121
|
+
else if (result.error !== null ||
|
|
122
|
+
!result.source_report_id ||
|
|
123
|
+
!result.observation_id ||
|
|
124
|
+
!result.canonical_finding_id ||
|
|
125
|
+
result.version === null)
|
|
126
|
+
throw new CodexSecurityError("Cloud returned an incomplete saved finding.");
|
|
127
|
+
}
|
|
128
|
+
if (canonicalJson(receipt.counts) !== canonicalJson(actual))
|
|
129
|
+
throw new CodexSecurityError("Cloud returned inconsistent publication counts.");
|
|
130
|
+
return receipt;
|
|
131
|
+
}
|
|
132
|
+
async function withImportLock(state, key, signal, operation) {
|
|
133
|
+
if (process.platform === "win32")
|
|
134
|
+
await createWindowsOutputDirectory(state, true);
|
|
135
|
+
else
|
|
136
|
+
await mkdir(state, { recursive: true, mode: 0o700 });
|
|
137
|
+
await validatePreparedOutputDir(state, undefined, true);
|
|
138
|
+
const path = join(state, `${key}.lock.sqlite`);
|
|
139
|
+
const metadata = await lstat(path).catch((error) => {
|
|
140
|
+
if (error.code === "ENOENT")
|
|
141
|
+
return undefined;
|
|
142
|
+
throw error;
|
|
143
|
+
});
|
|
144
|
+
if (metadata && !metadata.isFile())
|
|
145
|
+
throw new CodexSecurityError("The publication lock is not a regular file.");
|
|
146
|
+
const require = createRequire(import.meta.url);
|
|
147
|
+
const Database = (process.versions["bun"]
|
|
148
|
+
? require("bun:sqlite").Database
|
|
149
|
+
: require("node:sqlite").DatabaseSync);
|
|
150
|
+
const guard = new Database(path);
|
|
151
|
+
try {
|
|
152
|
+
if (process.platform !== "win32")
|
|
153
|
+
chmodSync(path, 0o600);
|
|
154
|
+
guard.exec("PRAGMA busy_timeout = 0");
|
|
155
|
+
for (;;) {
|
|
156
|
+
signal?.throwIfAborted();
|
|
157
|
+
try {
|
|
158
|
+
// Keep this inode across invocations. SQLite releases the transaction
|
|
159
|
+
// on process exit, so a paused publisher cannot lose its lock to a lease.
|
|
160
|
+
guard.exec("BEGIN EXCLUSIVE");
|
|
161
|
+
break;
|
|
162
|
+
}
|
|
163
|
+
catch (error) {
|
|
164
|
+
const sqliteError = error;
|
|
165
|
+
if (sqliteError.errcode !== 5 && sqliteError.code !== "SQLITE_BUSY")
|
|
166
|
+
throw error;
|
|
167
|
+
await delay(50, undefined, { signal });
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
return await operation();
|
|
171
|
+
}
|
|
172
|
+
finally {
|
|
173
|
+
guard.close();
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
export async function prepareExternalPublication(path, options, dependencies = {}) {
|
|
177
|
+
const parsed = await readExternalFindings(path, options);
|
|
178
|
+
return parsed.publication
|
|
179
|
+
? preparePublicationInput(parsed.publication, options, dependencies)
|
|
180
|
+
: prepareNormalizedExternalPublication(parsed.findings, options, dependencies);
|
|
181
|
+
}
|
|
182
|
+
/** Prepare normalized provider evidence without reading a vendor export. */
|
|
183
|
+
export async function prepareNormalizedExternalPublication(input, options, dependencies = {}) {
|
|
184
|
+
// Validation adds Cloud defaults; leave the caller's objects unchanged.
|
|
185
|
+
const current = structuredClone(input);
|
|
186
|
+
// structuredClone preserves aliases. Detach every raw record before adding
|
|
187
|
+
// typed defaults, including aliases shared with another finding's evidence.
|
|
188
|
+
for (const finding of current.findings)
|
|
189
|
+
if (finding.evidence.source_data !== undefined)
|
|
190
|
+
finding.evidence.source_data = structuredClone(finding.evidence.source_data);
|
|
191
|
+
const identities = new Set();
|
|
192
|
+
for (const finding of current.findings) {
|
|
193
|
+
const id = finding.source_finding_id;
|
|
194
|
+
if (!id.trim() || id.includes("\0") || Buffer.byteLength(id) > 512)
|
|
195
|
+
throw new CodexSecurityError("Source finding id must contain 1–512 UTF-8 bytes and no NUL characters.");
|
|
196
|
+
if (identities.has(id))
|
|
197
|
+
throw new CodexSecurityError(`Duplicate source finding id ${JSON.stringify(id)}.`);
|
|
198
|
+
identities.add(id);
|
|
199
|
+
finding.evidence = validateExternalEvidence(finding.evidence);
|
|
200
|
+
}
|
|
201
|
+
return preparePublicationInput({
|
|
202
|
+
current,
|
|
203
|
+
legacy: current,
|
|
204
|
+
currentIdentity: current.findings,
|
|
205
|
+
legacyIdentity: current.findings,
|
|
206
|
+
}, options, dependencies);
|
|
207
|
+
}
|
|
208
|
+
async function preparePublicationInput(input, options, dependencies) {
|
|
209
|
+
const progress = (event) => {
|
|
210
|
+
try {
|
|
211
|
+
dependencies.onProgress?.(event);
|
|
212
|
+
}
|
|
213
|
+
catch {
|
|
214
|
+
// Optional progress must not interrupt preparation or publication.
|
|
215
|
+
}
|
|
216
|
+
};
|
|
217
|
+
if (!options.sourceKey.trim() ||
|
|
218
|
+
options.sourceKey.includes("\0") ||
|
|
219
|
+
Buffer.byteLength(options.sourceKey) > 512) {
|
|
220
|
+
throw new CodexSecurityError("The source key must contain 1–512 UTF-8 bytes and no NUL characters. Use a stable scanner organization/project/branch namespace across imports.");
|
|
221
|
+
}
|
|
222
|
+
progress({ phase: "reading", completed: 0 });
|
|
223
|
+
const { current: parsed, legacy: legacyParsed, currentIdentity, legacyIdentity, } = input;
|
|
224
|
+
if (parsed.findings.length === 0) {
|
|
225
|
+
throw new CodexSecurityError(`No supported findings are ready to publish. ${parsed.excluded.map((item) => `Item ${item.position}: ${item.reason}`).join(" ")}`);
|
|
226
|
+
}
|
|
227
|
+
progress({ phase: "discovering", completed: 0 });
|
|
228
|
+
const environment = dependencies.environment ?? process.env;
|
|
229
|
+
const apiBaseUrl = cloudBaseUrl(environment);
|
|
230
|
+
const cloudApiUrl = `${apiBaseUrl}/external`;
|
|
231
|
+
// Validate the configured return destination before authentication or uploads.
|
|
232
|
+
cloudFindingsUrl(environment, options.repository);
|
|
233
|
+
const credentials = await (dependencies.credentials ?? (() => readCloudCredentials(environment)))();
|
|
234
|
+
async function request(endpoint, body) {
|
|
235
|
+
dependencies.signal?.throwIfAborted();
|
|
236
|
+
// Allow upload and response transport around the server's publication budget.
|
|
237
|
+
const timeout = AbortSignal.timeout(body ? 60_000 : 30_000);
|
|
238
|
+
const response = await (dependencies.fetch ?? globalThis.fetch)(`${cloudApiUrl}${endpoint}`, {
|
|
239
|
+
method: body ? "POST" : "GET",
|
|
240
|
+
headers: {
|
|
241
|
+
Authorization: `Bearer ${credentials.access_token}`,
|
|
242
|
+
"ChatGPT-Account-ID": credentials.account_id,
|
|
243
|
+
Accept: "application/json",
|
|
244
|
+
...(body ? { "Content-Type": "application/json" } : {}),
|
|
245
|
+
},
|
|
246
|
+
...(body ? { body: JSON.stringify(body) } : {}),
|
|
247
|
+
redirect: "error",
|
|
248
|
+
signal: dependencies.signal
|
|
249
|
+
? AbortSignal.any([dependencies.signal, timeout])
|
|
250
|
+
: timeout,
|
|
251
|
+
});
|
|
252
|
+
if (!response.ok) {
|
|
253
|
+
const payload = (await response.json().catch(() => null));
|
|
254
|
+
const retryAfter = response.headers.get("Retry-After");
|
|
255
|
+
throw new CloudImportError(response.status, `Cloud import failed (HTTP ${response.status}). ${payload?.error?.message ?? response.statusText}${retryAfter ? ` Retry-After: ${retryAfter}.` : ""}`);
|
|
256
|
+
}
|
|
257
|
+
return response.json();
|
|
258
|
+
}
|
|
259
|
+
const destinations = [];
|
|
260
|
+
let page = null;
|
|
261
|
+
const seenPages = new Set();
|
|
262
|
+
do {
|
|
263
|
+
const result = validateRepositories(await request(`/repositories?limit=20&repository=${encodeURIComponent(options.repository)}${page ? `&page=${encodeURIComponent(page)}` : ""}`));
|
|
264
|
+
destinations.push(...result.data.filter((item) => item.id === options.repository ||
|
|
265
|
+
cloudRepositoryUrlKey(item.url) ===
|
|
266
|
+
cloudRepositoryUrlKey(options.repository)));
|
|
267
|
+
page = result.next;
|
|
268
|
+
if (result.has_more !== (page !== null) ||
|
|
269
|
+
(page !== null && seenPages.has(page)))
|
|
270
|
+
throw new CodexSecurityError("Cloud returned invalid repository pagination.");
|
|
271
|
+
if (page !== null)
|
|
272
|
+
seenPages.add(page);
|
|
273
|
+
} while (page !== null);
|
|
274
|
+
if (destinations.length !== 1)
|
|
275
|
+
throw new CodexSecurityError(destinations.length
|
|
276
|
+
? "The repository matches multiple connectors. Select an unambiguous Cloud repository."
|
|
277
|
+
: "The destination repository is not available to this Cloud account. Copy its repository URL or ID from Codex Security Cloud and check that you are signed into the intended account.");
|
|
278
|
+
const destination = destinations[0];
|
|
279
|
+
const source = { provider: options.provider, source_key: options.sourceKey };
|
|
280
|
+
const state = join(codexSecurityStateDirectory(environment), "external-finding-publications");
|
|
281
|
+
const checkpointIdentity = [
|
|
282
|
+
// Keep existing production checkpoints readable; other deployments must
|
|
283
|
+
// never share a saved request or receipt with the production default.
|
|
284
|
+
...(apiBaseUrl === DEFAULT_CLOUD_BASE_URL ? [] : [apiBaseUrl]),
|
|
285
|
+
credentials.account_id,
|
|
286
|
+
destination.id,
|
|
287
|
+
destination.repo_connector_id,
|
|
288
|
+
source,
|
|
289
|
+
];
|
|
290
|
+
const inputDigest = hash("sha256", canonicalJson(currentIdentity));
|
|
291
|
+
const legacyInputDigest = hash("sha256", canonicalJson(legacyIdentity));
|
|
292
|
+
function validateSaved(serialized, legacy = false) {
|
|
293
|
+
const content = JSON.parse(serialized);
|
|
294
|
+
if (content.accountId !== credentials.account_id ||
|
|
295
|
+
!Array.isArray(content.requests) ||
|
|
296
|
+
content.requests.length === 0)
|
|
297
|
+
throw new Error("Saved import account or requests do not match.");
|
|
298
|
+
content.requests = content.requests.map(validateImportRequest);
|
|
299
|
+
const savedFindings = submissionFindings(content.requests);
|
|
300
|
+
const savedEvidence = canonicalJson(savedFindings);
|
|
301
|
+
const expected = legacy ? legacyParsed.findings : parsed.findings;
|
|
302
|
+
const matches = content.inputProjection !== undefined
|
|
303
|
+
? sameFindingSelection(savedFindings, expected) &&
|
|
304
|
+
content.inputProjection?.inputDigest ===
|
|
305
|
+
(legacy ? legacyInputDigest : inputDigest) &&
|
|
306
|
+
content.inputProjection?.savedEvidenceDigest ===
|
|
307
|
+
hash("sha256", savedEvidence)
|
|
308
|
+
: savedEvidence === canonicalJson(expected);
|
|
309
|
+
if (!matches)
|
|
310
|
+
throw new Error("Saved import evidence does not match the selected input.");
|
|
311
|
+
for (const submission of content.requests) {
|
|
312
|
+
if (submission.repository.id !== destination.id ||
|
|
313
|
+
submission.repository.repo_connector_id !==
|
|
314
|
+
destination.repo_connector_id ||
|
|
315
|
+
canonicalJson(submission.source) !== canonicalJson(source))
|
|
316
|
+
throw new Error("Saved import destination does not match.");
|
|
317
|
+
}
|
|
318
|
+
return content;
|
|
319
|
+
}
|
|
320
|
+
const key = hash("sha256", canonicalJson([...checkpointIdentity, currentIdentity]));
|
|
321
|
+
const pendingPath = join(state, `${key}.pending.json`);
|
|
322
|
+
const historicalKey = hash("sha256", canonicalJson([...checkpointIdentity, legacyParsed.findings], (left, right) => left.localeCompare(right)));
|
|
323
|
+
const legacyKeys = new Set([
|
|
324
|
+
hash("sha256", canonicalJson([...checkpointIdentity, legacyIdentity])),
|
|
325
|
+
historicalKey,
|
|
326
|
+
]);
|
|
327
|
+
async function rejectUnidentifiedLegacyCheckpoint() {
|
|
328
|
+
const entries = await readdir(state, { withFileTypes: true }).catch((error) => {
|
|
329
|
+
if (error.code === "ENOENT")
|
|
330
|
+
return [];
|
|
331
|
+
throw error;
|
|
332
|
+
});
|
|
333
|
+
if (entries.length)
|
|
334
|
+
await validatePreparedOutputDir(state, undefined, true);
|
|
335
|
+
for (const entry of entries) {
|
|
336
|
+
if (!entry.isFile() || !/^[a-f0-9]{64}\.pending\.json$/u.test(entry.name))
|
|
337
|
+
continue;
|
|
338
|
+
const candidateKey = entry.name.slice(0, -".pending.json".length);
|
|
339
|
+
if (candidateKey === key || legacyKeys.has(candidateKey))
|
|
340
|
+
continue;
|
|
341
|
+
const serialized = await readOptionalFile(join(state, entry.name));
|
|
342
|
+
if (serialized === undefined)
|
|
343
|
+
continue;
|
|
344
|
+
try {
|
|
345
|
+
// Locale-dependent filenames predate input projections. Later formats
|
|
346
|
+
// must remain isolated by their deployment-bound checkpoint keys.
|
|
347
|
+
const content = JSON.parse(serialized);
|
|
348
|
+
if (!content || content.inputProjection !== undefined)
|
|
349
|
+
continue;
|
|
350
|
+
validateSaved(serialized, true);
|
|
351
|
+
}
|
|
352
|
+
catch {
|
|
353
|
+
continue;
|
|
354
|
+
}
|
|
355
|
+
// These older bodies contain no deployment identity. Do not guess a
|
|
356
|
+
// destination or replace their immutable request with a fresh publication.
|
|
357
|
+
throw new CodexSecurityError("An older pending import matches this input, but its checkpoint identity could not be established. Resume it using the original deployment and locale before publishing this selection. The saved request was preserved without uploading.");
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
let recovered;
|
|
361
|
+
for (const legacyKey of legacyKeys) {
|
|
362
|
+
if (legacyKey === key)
|
|
363
|
+
continue;
|
|
364
|
+
const legacyPath = join(state, `${legacyKey}.pending.json`);
|
|
365
|
+
const captured = await readOptionalFile(legacyPath);
|
|
366
|
+
if (captured !== undefined) {
|
|
367
|
+
// Preserve requests staged by the earlier locale-based filename format.
|
|
368
|
+
// Hold both publication locks while moving the original checkpoint.
|
|
369
|
+
await withImportLock(state, key, dependencies.signal, async () => withImportLock(state, legacyKey, dependencies.signal, async () => {
|
|
370
|
+
const current = await readOptionalFile(pendingPath);
|
|
371
|
+
const serialized = await readOptionalFile(legacyPath);
|
|
372
|
+
if (current !== undefined) {
|
|
373
|
+
recovered = validateSaved(current);
|
|
374
|
+
if (serialized !== undefined) {
|
|
375
|
+
// A crash may leave both copies after installing the canonical file.
|
|
376
|
+
// Retire only an identical immutable submission, never another request.
|
|
377
|
+
let duplicate = null;
|
|
378
|
+
try {
|
|
379
|
+
duplicate = JSON.parse(serialized);
|
|
380
|
+
}
|
|
381
|
+
catch (error) {
|
|
382
|
+
if (!(error instanceof SyntaxError))
|
|
383
|
+
throw error;
|
|
384
|
+
}
|
|
385
|
+
if (duplicate !== null &&
|
|
386
|
+
typeof duplicate === "object" &&
|
|
387
|
+
sameSubmission(duplicate, recovered))
|
|
388
|
+
await rm(legacyPath);
|
|
389
|
+
}
|
|
390
|
+
return;
|
|
391
|
+
}
|
|
392
|
+
// A competing resumer may have completed and removed both paths while
|
|
393
|
+
// this caller waited. Retain its observed immutable request and receipts.
|
|
394
|
+
const content = validateSaved(serialized ?? captured, true);
|
|
395
|
+
if (!sameFindingSelection(legacyParsed.findings, parsed.findings))
|
|
396
|
+
throw new CodexSecurityError("The saved import selects different findings from this input. Resume the original selection before publishing this selection.");
|
|
397
|
+
if (content.requests.some((submission) => submission.repository.reset_marker !== destination.reset_marker)) {
|
|
398
|
+
if (serialized !== undefined)
|
|
399
|
+
await rm(legacyPath);
|
|
400
|
+
throw new CloudImportError(409, "The repository was reset after this submission. The saved request was retired without uploading. Review the destination and run the command again to approve a fresh publication.");
|
|
401
|
+
}
|
|
402
|
+
const savedEvidenceDigest = content.inputProjection?.savedEvidenceDigest ??
|
|
403
|
+
hash("sha256", canonicalJson(legacyParsed.findings));
|
|
404
|
+
const rebindProjection = content.inputProjection !== undefined ||
|
|
405
|
+
savedEvidenceDigest !== inputDigest;
|
|
406
|
+
if (rebindProjection)
|
|
407
|
+
content.inputProjection = { inputDigest, savedEvidenceDigest };
|
|
408
|
+
recovered = content;
|
|
409
|
+
if (serialized === undefined)
|
|
410
|
+
return;
|
|
411
|
+
if (!rebindProjection) {
|
|
412
|
+
await rename(legacyPath, pendingPath);
|
|
413
|
+
}
|
|
414
|
+
else {
|
|
415
|
+
// Bind the new input projection to the original immutable evidence,
|
|
416
|
+
// so later inventory reordering still resumes these exact requests.
|
|
417
|
+
await writeAtomicJson(pendingPath, content);
|
|
418
|
+
await rm(legacyPath);
|
|
419
|
+
}
|
|
420
|
+
}));
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
const serialized = await readOptionalFile(pendingPath);
|
|
424
|
+
const saved = serialized === undefined ? recovered : validateSaved(serialized);
|
|
425
|
+
if (!saved)
|
|
426
|
+
await rejectUnidentifiedLegacyCheckpoint();
|
|
427
|
+
if (saved) {
|
|
428
|
+
const content = saved;
|
|
429
|
+
for (const submission of content.requests) {
|
|
430
|
+
if (submission.repository.reset_marker !== destination.reset_marker) {
|
|
431
|
+
// The old request is never sent after reset. A subsequent explicit
|
|
432
|
+
// invocation prepares a fresh submission and asks for approval again.
|
|
433
|
+
await withImportLock(state, key, dependencies.signal, async () => {
|
|
434
|
+
const current = await readOptionalFile(pendingPath);
|
|
435
|
+
if (current !== undefined &&
|
|
436
|
+
sameSubmission(JSON.parse(current), content))
|
|
437
|
+
await rm(pendingPath);
|
|
438
|
+
});
|
|
439
|
+
throw new CloudImportError(409, "The repository was reset after this submission. The saved request was retired without uploading. Review the destination and run the command again to approve a fresh publication.");
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
let requests = saved?.requests;
|
|
444
|
+
if (!requests) {
|
|
445
|
+
for (const finding of parsed.findings) {
|
|
446
|
+
const sourceRepositories = [
|
|
447
|
+
finding.evidence.details?.repository?.url,
|
|
448
|
+
...(options.provider === "wiz"
|
|
449
|
+
? [wizCsvRepositoryUrl(finding.evidence)]
|
|
450
|
+
: []),
|
|
451
|
+
...(options.provider === "snyk"
|
|
452
|
+
? snykRepositoryUrls(finding.evidence.source_data, {
|
|
453
|
+
cloudRepositoryUrl: destination.url,
|
|
454
|
+
})
|
|
455
|
+
: []),
|
|
456
|
+
...(options.provider === "endor"
|
|
457
|
+
? endorRepositoryUrls(finding.evidence.source_data)
|
|
458
|
+
: []),
|
|
459
|
+
];
|
|
460
|
+
for (const sourceRepository of sourceRepositories)
|
|
461
|
+
if (sourceRepository &&
|
|
462
|
+
cloudRepositoryUrlKey(sourceRepository) !==
|
|
463
|
+
cloudRepositoryUrlKey(destination.url))
|
|
464
|
+
throw new CodexSecurityError(`Finding ${JSON.stringify(finding.source_finding_id)} belongs to ${JSON.stringify(sourceRepository)}, which does not match the selected Cloud repository ${JSON.stringify(destination.url)}. Verify the source repository mapping before publishing.`);
|
|
465
|
+
}
|
|
466
|
+
let preparedCount = 0;
|
|
467
|
+
progress({
|
|
468
|
+
phase: "preparing",
|
|
469
|
+
completed: 0,
|
|
470
|
+
total: parsed.findings.length,
|
|
471
|
+
});
|
|
472
|
+
const previousReports = await readInBatches(parsed.findings, async (finding) => {
|
|
473
|
+
const query = new URLSearchParams({
|
|
474
|
+
provider: source.provider,
|
|
475
|
+
source_key: source.source_key,
|
|
476
|
+
source_finding_id: finding.source_finding_id,
|
|
477
|
+
limit: "2",
|
|
478
|
+
});
|
|
479
|
+
const summaries = validateSourceReports(await request(`${sourcePath(destination)}?${query}`));
|
|
480
|
+
if (summaries.has_more || summaries.data.length > 1)
|
|
481
|
+
throw new CodexSecurityError("Cloud returned more than one report for a source identity.");
|
|
482
|
+
const previous = summaries.data[0];
|
|
483
|
+
if (previous &&
|
|
484
|
+
(previous.source_finding_id !== finding.source_finding_id ||
|
|
485
|
+
previous.source.provider !== source.provider ||
|
|
486
|
+
previous.source.source_key !== source.source_key ||
|
|
487
|
+
previous.repo_id !== destination.id ||
|
|
488
|
+
previous.repo_connector_id !== destination.repo_connector_id))
|
|
489
|
+
throw new CodexSecurityError("Cloud returned a different source identity.");
|
|
490
|
+
progress({
|
|
491
|
+
phase: "preparing",
|
|
492
|
+
completed: ++preparedCount,
|
|
493
|
+
total: parsed.findings.length,
|
|
494
|
+
});
|
|
495
|
+
return previous;
|
|
496
|
+
});
|
|
497
|
+
requests = [];
|
|
498
|
+
let current;
|
|
499
|
+
for (const [index, finding] of parsed.findings.entries()) {
|
|
500
|
+
const previous = previousReports[index];
|
|
501
|
+
const environmentId = previous?.environment_id ?? destination.import_environment_id;
|
|
502
|
+
if (environmentId == null)
|
|
503
|
+
throw new CodexSecurityError("Configure an authorized Cloud environment for this repository before importing findings.");
|
|
504
|
+
const repository = {
|
|
505
|
+
id: destination.id,
|
|
506
|
+
repo_connector_id: destination.repo_connector_id,
|
|
507
|
+
environment_id: environmentId,
|
|
508
|
+
reset_marker: destination.reset_marker,
|
|
509
|
+
};
|
|
510
|
+
const item = {
|
|
511
|
+
client_id: `item-${index + 1}`,
|
|
512
|
+
source_finding_id: finding.source_finding_id,
|
|
513
|
+
expected_version: previous?.version ?? 0,
|
|
514
|
+
evidence: finding.evidence,
|
|
515
|
+
};
|
|
516
|
+
if (current &&
|
|
517
|
+
(current.repository.environment_id !== environmentId ||
|
|
518
|
+
current.items.length === 100 ||
|
|
519
|
+
Buffer.byteLength(JSON.stringify({ ...current, items: [...current.items, item] })) > MAX_REQUEST_BYTES)) {
|
|
520
|
+
requests.push(validateImportRequest(current));
|
|
521
|
+
current = undefined;
|
|
522
|
+
}
|
|
523
|
+
current ??= { request_id: randomUUID(), repository, source, items: [] };
|
|
524
|
+
current.items.push(item);
|
|
525
|
+
}
|
|
526
|
+
if (current)
|
|
527
|
+
requests.push(validateImportRequest(current));
|
|
528
|
+
}
|
|
529
|
+
const submission = {
|
|
530
|
+
accountId: credentials.account_id,
|
|
531
|
+
requests,
|
|
532
|
+
receipts: saved?.receipts,
|
|
533
|
+
inputProjection: saved?.inputProjection ?? {
|
|
534
|
+
inputDigest,
|
|
535
|
+
savedEvidenceDigest: hash("sha256", canonicalJson(submissionFindings(requests))),
|
|
536
|
+
},
|
|
537
|
+
};
|
|
538
|
+
const preview = {
|
|
539
|
+
...parsed,
|
|
540
|
+
findings: submissionFindings(requests),
|
|
541
|
+
cloudApiUrl,
|
|
542
|
+
cloudUrl: cloudFindingsUrl(environment, destination.id),
|
|
543
|
+
accountId: credentials.account_id,
|
|
544
|
+
destination,
|
|
545
|
+
source,
|
|
546
|
+
resumed: saved !== undefined,
|
|
547
|
+
requests,
|
|
548
|
+
};
|
|
549
|
+
return {
|
|
550
|
+
preview,
|
|
551
|
+
async publish() {
|
|
552
|
+
progress({ phase: "waiting", completed: 0 });
|
|
553
|
+
const result = (acknowledged, verified) => {
|
|
554
|
+
const counts = { created: 0, updated: 0, unchanged: 0, error: 0 };
|
|
555
|
+
const failures = [];
|
|
556
|
+
for (const [index, receipt] of acknowledged.entries()) {
|
|
557
|
+
for (const outcome of Object.keys(counts))
|
|
558
|
+
counts[outcome] += receipt.counts[outcome] ?? 0;
|
|
559
|
+
const ids = new Map(requests[index].items.map((item) => [
|
|
560
|
+
item.client_id,
|
|
561
|
+
item.source_finding_id,
|
|
562
|
+
]));
|
|
563
|
+
for (const item of receipt.results)
|
|
564
|
+
if (item.error)
|
|
565
|
+
failures.push({
|
|
566
|
+
source_finding_id: ids.get(item.client_id),
|
|
567
|
+
...item.error,
|
|
568
|
+
});
|
|
569
|
+
}
|
|
570
|
+
return {
|
|
571
|
+
cloudApiUrl,
|
|
572
|
+
cloudUrl: preview.cloudUrl,
|
|
573
|
+
status: parsed.excluded.length || counts.error ? "partial" : "complete",
|
|
574
|
+
read: parsed.read,
|
|
575
|
+
ready: parsed.findings.length,
|
|
576
|
+
excluded: parsed.excluded,
|
|
577
|
+
receipts: acknowledged,
|
|
578
|
+
counts,
|
|
579
|
+
failures,
|
|
580
|
+
unacknowledged: parsed.findings.length -
|
|
581
|
+
Object.values(counts).reduce((sum, count) => sum + count, 0),
|
|
582
|
+
verified,
|
|
583
|
+
};
|
|
584
|
+
};
|
|
585
|
+
let acquired = false;
|
|
586
|
+
return await withImportLock(state, key, dependencies.signal, async () => {
|
|
587
|
+
acquired = true;
|
|
588
|
+
// Re-read under the lock to keep checkpoints from another publisher.
|
|
589
|
+
const pending = await readOptionalFile(pendingPath);
|
|
590
|
+
if (pending === undefined && !saved)
|
|
591
|
+
await rejectUnidentifiedLegacyCheckpoint();
|
|
592
|
+
const current = pending === undefined
|
|
593
|
+
? submission
|
|
594
|
+
: JSON.parse(pending);
|
|
595
|
+
if (!sameSubmission(current, submission))
|
|
596
|
+
throw new CodexSecurityError("Another publication prepared this input. Run the command again to review and resume that saved request.");
|
|
597
|
+
const receipts = (current.receipts ?? []).map(validateImportReceipt);
|
|
598
|
+
if (receipts.length > requests.length)
|
|
599
|
+
throw new CodexSecurityError("Saved publication has more receipts than requests.");
|
|
600
|
+
const acknowledged = [];
|
|
601
|
+
let verified = 0;
|
|
602
|
+
let persisted = pending !== undefined;
|
|
603
|
+
try {
|
|
604
|
+
if (pending === undefined) {
|
|
605
|
+
await writeAtomicJson(pendingPath, submission);
|
|
606
|
+
persisted = true;
|
|
607
|
+
}
|
|
608
|
+
progress({
|
|
609
|
+
phase: "uploading",
|
|
610
|
+
completed: 0,
|
|
611
|
+
total: parsed.findings.length,
|
|
612
|
+
});
|
|
613
|
+
let acknowledgedCount = 0;
|
|
614
|
+
for (const [batchIndex, batch] of requests.entries()) {
|
|
615
|
+
const receipt = validatePublicationReceipt(receipts[batchIndex] ??
|
|
616
|
+
(await request("/finding_imports", batch)), batch);
|
|
617
|
+
acknowledged.push(receipt);
|
|
618
|
+
acknowledgedCount += receipt.item_count;
|
|
619
|
+
progress({
|
|
620
|
+
phase: "uploading",
|
|
621
|
+
completed: acknowledgedCount,
|
|
622
|
+
total: parsed.findings.length,
|
|
623
|
+
});
|
|
624
|
+
if (batchIndex === receipts.length) {
|
|
625
|
+
receipts.push(receipt);
|
|
626
|
+
// Acknowledged batches must not consume another POST quota on retry.
|
|
627
|
+
// Readback can still resume after this checkpoint without replaying writes.
|
|
628
|
+
await writeAtomicJson(pendingPath, {
|
|
629
|
+
...submission,
|
|
630
|
+
receipts,
|
|
631
|
+
});
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
// Verify readable source records without mistaking a newer concurrent
|
|
635
|
+
// observation for failure of the original, immutable import receipt.
|
|
636
|
+
const readbacks = receipts.flatMap((receipt, batchIndex) => {
|
|
637
|
+
const batch = requests[batchIndex];
|
|
638
|
+
const batchItems = new Map(batch.items.map((item) => [item.client_id, item]));
|
|
639
|
+
return receipt.results
|
|
640
|
+
.filter((item) => item.outcome !== "error")
|
|
641
|
+
.map((item) => ({
|
|
642
|
+
item,
|
|
643
|
+
expected: batchItems.get(item.client_id),
|
|
644
|
+
batch,
|
|
645
|
+
}));
|
|
646
|
+
});
|
|
647
|
+
progress({
|
|
648
|
+
phase: "verifying",
|
|
649
|
+
completed: 0,
|
|
650
|
+
total: readbacks.length,
|
|
651
|
+
});
|
|
652
|
+
await readInBatches(readbacks, async ({ item, expected, batch }) => {
|
|
653
|
+
const report = validateSourceReport(await request(`${sourcePath(destination)}/${encodeURIComponent(item.source_report_id)}`));
|
|
654
|
+
if (report.id !== item.source_report_id ||
|
|
655
|
+
report.canonical_finding_id !== item.canonical_finding_id ||
|
|
656
|
+
report.version < item.version ||
|
|
657
|
+
(report.version === item.version &&
|
|
658
|
+
(report.observation_id !== item.observation_id ||
|
|
659
|
+
canonicalJson(report.evidence) !==
|
|
660
|
+
canonicalJson(expected.evidence))) ||
|
|
661
|
+
report.source_finding_id !== expected.source_finding_id ||
|
|
662
|
+
report.repo_id !== destination.id ||
|
|
663
|
+
report.repo_connector_id !== destination.repo_connector_id ||
|
|
664
|
+
report.environment_id !== batch.repository.environment_id ||
|
|
665
|
+
canonicalJson(report.source) !== canonicalJson(source))
|
|
666
|
+
throw new CodexSecurityError("Cloud readback did not match the saved finding identity.");
|
|
667
|
+
progress({
|
|
668
|
+
phase: "verifying",
|
|
669
|
+
completed: ++verified,
|
|
670
|
+
total: readbacks.length,
|
|
671
|
+
});
|
|
672
|
+
});
|
|
673
|
+
const completed = result(acknowledged, verified);
|
|
674
|
+
await writeAtomicJson(join(state, `${key}.result.json`), completed);
|
|
675
|
+
await rm(pendingPath, { force: true });
|
|
676
|
+
return completed;
|
|
677
|
+
}
|
|
678
|
+
catch (error) {
|
|
679
|
+
if (error instanceof CloudImportError && error.status === 409) {
|
|
680
|
+
await rm(pendingPath, { force: true });
|
|
681
|
+
persisted = false;
|
|
682
|
+
}
|
|
683
|
+
const message = `${error instanceof Error ? error.message : String(error)} ${error instanceof CloudImportError && error.status === 409 ? "The old submission was retired. Rediscover and approve a fresh publication." : persisted ? `Repeat the same command to resume the saved request. Saved submission: ${pendingPath}` : "No resumable request was saved. Correct the error and review the input before retrying."}`;
|
|
684
|
+
throw new ExternalPublicationError(message, {
|
|
685
|
+
...result(acknowledged, verified),
|
|
686
|
+
status: "interrupted",
|
|
687
|
+
error: message,
|
|
688
|
+
...(persisted ? { savedSubmission: pendingPath } : {}),
|
|
689
|
+
}, error);
|
|
690
|
+
}
|
|
691
|
+
}).catch((error) => {
|
|
692
|
+
if (acquired || !dependencies.signal?.aborted)
|
|
693
|
+
throw error;
|
|
694
|
+
// The active publisher owns the checkpoint. Report only acknowledgements
|
|
695
|
+
// already observed during preparation, without rewriting its files.
|
|
696
|
+
const known = submission.receipts ?? [];
|
|
697
|
+
if (known.length > requests.length)
|
|
698
|
+
throw new CodexSecurityError("Saved publication has more receipts than requests.");
|
|
699
|
+
const acknowledged = known.map((receipt, index) => validatePublicationReceipt(receipt, requests[index]));
|
|
700
|
+
const message = saved
|
|
701
|
+
? `Publication canceled while waiting. Repeat the same command to resume the saved request. Saved submission: ${pendingPath}`
|
|
702
|
+
: "Publication canceled while waiting. No resumable request was saved by this invocation.";
|
|
703
|
+
throw new ExternalPublicationError(message, {
|
|
704
|
+
...result(acknowledged, 0),
|
|
705
|
+
status: "interrupted",
|
|
706
|
+
error: message,
|
|
707
|
+
...(saved ? { savedSubmission: pendingPath } : {}),
|
|
708
|
+
}, error);
|
|
709
|
+
});
|
|
710
|
+
},
|
|
711
|
+
};
|
|
712
|
+
}
|
|
713
|
+
//# sourceMappingURL=external-findings-publish.js.map
|