@openai/codex-security 0.1.31 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +295 -32
- package/_bundled_plugin/.mcp.json +1 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +4 -4
- package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/references/artifact-storage.md +4 -4
- package/_bundled_plugin/references/config-preflight.md +1 -1
- package/_bundled_plugin/references/core-scan.md +14 -2
- package/_bundled_plugin/references/final-report.md +4 -2
- package/_bundled_plugin/references/scan-artifacts.md +2 -2
- package/_bundled_plugin/references/scan-contract.md +2 -2
- package/_bundled_plugin/references/threat-model.md +2 -2
- package/_bundled_plugin/schemas/coverage.schema.json +22 -0
- package/_bundled_plugin/schemas/scan-manifest.schema.json +100 -85
- package/_bundled_plugin/schemas/tools/scan-draft.schema.json +116 -102
- package/_bundled_plugin/scripts/config_preflight.py +4 -8
- package/_bundled_plugin/scripts/deep_scan_workbench.py +83 -217
- package/_bundled_plugin/scripts/filesystem_identity.py +1 -5
- package/_bundled_plugin/scripts/finalize_scan_contract.py +441 -166
- package/_bundled_plugin/scripts/finding_preview.py +42 -82
- package/_bundled_plugin/scripts/generate_in_scope_files.py +30 -34
- package/_bundled_plugin/scripts/generate_rank_input.py +58 -99
- package/_bundled_plugin/scripts/launch_codex_security_mcp +5 -1
- package/_bundled_plugin/scripts/rank_preview.py +85 -68
- package/_bundled_plugin/scripts/report_projection.py +37 -58
- package/_bundled_plugin/scripts/threat_model_projection.py +164 -0
- package/_bundled_plugin/scripts/windows_scan_local_files.py +11 -6
- package/_bundled_plugin/scripts/workbench/storage.py +11 -0
- package/_bundled_plugin/scripts/workbench_cli.py +11 -17
- package/_bundled_plugin/scripts/workbench_constants.py +6 -14
- package/_bundled_plugin/scripts/workbench_db.py +116 -228
- package/_bundled_plugin/scripts/workbench_finding_index.py +1 -6
- package/_bundled_plugin/scripts/workbench_native_indexes.py +1 -4
- package/_bundled_plugin/scripts/workbench_progress.py +12 -34
- package/_bundled_plugin/scripts/workbench_publication.py +47 -62
- package/_bundled_plugin/scripts/workbench_remediation.py +2 -20
- package/_bundled_plugin/scripts/workbench_saved_results.py +1246 -230
- package/_bundled_plugin/scripts/workbench_scan_start.py +4 -11
- package/_bundled_plugin/scripts/workbench_scan_usage.py +7 -15
- package/_bundled_plugin/scripts/workbench_schema.py +86 -271
- package/_bundled_plugin/scripts/workbench_severity.py +3 -0
- package/_bundled_plugin/scripts/workbench_source_excerpt.py +5 -6
- package/_bundled_plugin/scripts/workbench_target.py +98 -10
- package/_bundled_plugin/scripts/workbench_target_state.py +1 -5
- package/_bundled_plugin/scripts/workbench_validation.py +6 -20
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +3 -3
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +3 -3
- package/_bundled_plugin/skills/security-scan/SKILL.md +2 -2
- package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +2 -2
- package/_bundled_plugin/skills/threat-model/SKILL.md +2 -2
- package/dist/api.d.ts +3 -5
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +201 -207
- package/dist/api.js.map +1 -1
- package/dist/artifact-export.d.ts +70 -0
- package/dist/artifact-export.d.ts.map +1 -0
- package/dist/artifact-export.js +277 -0
- package/dist/artifact-export.js.map +1 -0
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +8 -3
- package/dist/auth.js.map +1 -1
- package/dist/bulk-scan-discovery.d.ts +1 -0
- package/dist/bulk-scan-discovery.d.ts.map +1 -1
- package/dist/bulk-scan-discovery.js +8 -9
- package/dist/bulk-scan-discovery.js.map +1 -1
- package/dist/classify-scan-severity.d.ts.map +1 -1
- package/dist/classify-scan-severity.js +2 -5
- package/dist/classify-scan-severity.js.map +1 -1
- package/dist/classify-severity.d.ts +2 -8
- package/dist/classify-severity.d.ts.map +1 -1
- package/dist/classify-severity.js +3 -5
- package/dist/classify-severity.js.map +1 -1
- package/dist/cli-help.d.ts +3 -0
- package/dist/cli-help.d.ts.map +1 -0
- package/dist/cli-help.js +281 -0
- package/dist/cli-help.js.map +1 -0
- package/dist/cli.d.ts +19 -22
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +631 -714
- package/dist/cli.js.map +1 -1
- package/dist/cloud-publish.d.ts.map +1 -1
- package/dist/cloud-publish.js +1 -8
- package/dist/cloud-publish.js.map +1 -1
- package/dist/codex-prompt.d.ts.map +1 -1
- package/dist/codex-prompt.js +1 -2
- package/dist/codex-prompt.js.map +1 -1
- package/dist/codex-sdk-environment.d.ts +2 -0
- package/dist/codex-sdk-environment.d.ts.map +1 -0
- package/dist/codex-sdk-environment.js +30 -0
- package/dist/codex-sdk-environment.js.map +1 -0
- package/dist/component-plan.d.ts.map +1 -1
- package/dist/component-plan.js +2 -6
- package/dist/component-plan.js.map +1 -1
- package/dist/component-scan.d.ts +1 -0
- package/dist/component-scan.d.ts.map +1 -1
- package/dist/component-scan.js +31 -9
- package/dist/component-scan.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +32 -33
- package/dist/config.js.map +1 -1
- package/dist/contract.d.ts +2 -1
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +63 -122
- package/dist/contract.js.map +1 -1
- package/dist/cost-model.d.ts.map +1 -1
- package/dist/cost-model.js +65 -34
- package/dist/cost-model.js.map +1 -1
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +15 -31
- package/dist/cost.js.map +1 -1
- package/dist/custom-validation-prompt.js +4 -4
- package/dist/custom-validation-prompt.js.map +1 -1
- package/dist/custom-validation.d.ts.map +1 -1
- package/dist/custom-validation.js +17 -16
- package/dist/custom-validation.js.map +1 -1
- package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
- package/dist/deduplication/checkpointed-review.js +10 -11
- package/dist/deduplication/checkpointed-review.js.map +1 -1
- package/dist/deduplication/codex-review.d.ts +2 -0
- package/dist/deduplication/codex-review.d.ts.map +1 -1
- package/dist/deduplication/codex-review.js +3 -3
- package/dist/deduplication/codex-review.js.map +1 -1
- package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -1
- package/dist/deduplication/deduplication-reviewer.js +2 -0
- package/dist/deduplication/deduplication-reviewer.js.map +1 -1
- package/dist/deduplication/deduplication.js +40 -62
- package/dist/deduplication/deduplication.js.map +1 -1
- package/dist/deduplication/records-protocol.d.ts.map +1 -1
- package/dist/deduplication/records-protocol.js +1 -4
- package/dist/deduplication/records-protocol.js.map +1 -1
- package/dist/deduplication/records.d.ts +2 -1
- package/dist/deduplication/records.d.ts.map +1 -1
- package/dist/deduplication/records.js +24 -6
- package/dist/deduplication/records.js.map +1 -1
- package/dist/deduplication/review.d.ts +7 -0
- package/dist/deduplication/review.d.ts.map +1 -1
- package/dist/deduplication/review.js.map +1 -1
- package/dist/deduplication/scan.js +3 -7
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/deep-config.d.ts.map +1 -1
- package/dist/deep-config.js +3 -19
- package/dist/deep-config.js.map +1 -1
- package/dist/deep-progress.d.ts +0 -1
- package/dist/deep-progress.d.ts.map +1 -1
- package/dist/deep-progress.js +22 -40
- package/dist/deep-progress.js.map +1 -1
- package/dist/errors.d.ts +3 -2
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +5 -7
- package/dist/errors.js.map +1 -1
- package/dist/finding-catalogue.d.ts.map +1 -1
- package/dist/finding-catalogue.js +22 -22
- package/dist/finding-catalogue.js.map +1 -1
- package/dist/finding-workflow.d.ts.map +1 -1
- package/dist/finding-workflow.js +4 -7
- package/dist/finding-workflow.js.map +1 -1
- package/dist/findings-import.d.ts.map +1 -1
- package/dist/findings-import.js +7 -17
- package/dist/findings-import.js.map +1 -1
- package/dist/github.d.ts.map +1 -1
- package/dist/github.js +2 -3
- package/dist/github.js.map +1 -1
- package/dist/import-scan.js +2 -2
- package/dist/import-scan.js.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/knowledge-base.js +2 -2
- package/dist/knowledge-base.js.map +1 -1
- package/dist/linear.d.ts.map +1 -1
- package/dist/linear.js +2 -3
- package/dist/linear.js.map +1 -1
- package/dist/models.d.ts +24 -0
- package/dist/models.d.ts.map +1 -1
- package/dist/models.js.map +1 -1
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +36 -31
- package/dist/multiscan.js.map +1 -1
- package/dist/owner-evidence.d.ts.map +1 -1
- package/dist/owner-evidence.js +2 -6
- package/dist/owner-evidence.js.map +1 -1
- package/dist/patch-tui.d.ts.map +1 -1
- package/dist/patch-tui.js +2 -4
- package/dist/patch-tui.js.map +1 -1
- package/dist/project-config-schema.d.ts +10 -0
- package/dist/project-config-schema.d.ts.map +1 -1
- package/dist/project-config-schema.js +1 -0
- package/dist/project-config-schema.js.map +1 -1
- package/dist/project-config.d.ts +1 -0
- package/dist/project-config.d.ts.map +1 -1
- package/dist/project-config.js +2 -0
- package/dist/project-config.js.map +1 -1
- package/dist/prompt-files.d.ts +0 -1
- package/dist/prompt-files.d.ts.map +1 -1
- package/dist/prompt-files.js +2 -4
- package/dist/prompt-files.js.map +1 -1
- package/dist/publication-events.d.ts.map +1 -1
- package/dist/publication-events.js +15 -29
- package/dist/publication-events.js.map +1 -1
- package/dist/publication-store.d.ts.map +1 -1
- package/dist/publication-store.js +9 -13
- package/dist/publication-store.js.map +1 -1
- package/dist/publication.d.ts.map +1 -1
- package/dist/publication.js +11 -14
- package/dist/publication.js.map +1 -1
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +31 -41
- package/dist/publish.js.map +1 -1
- package/dist/record.d.ts +2 -0
- package/dist/record.d.ts.map +1 -0
- package/dist/record.js +4 -0
- package/dist/record.js.map +1 -0
- package/dist/result.d.ts +4 -1
- package/dist/result.d.ts.map +1 -1
- package/dist/result.js +7 -0
- package/dist/result.js.map +1 -1
- package/dist/runtime.d.ts +10 -13
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +203 -131
- package/dist/runtime.js.map +1 -1
- package/dist/saved-scan.d.ts.map +1 -1
- package/dist/saved-scan.js +10 -20
- package/dist/saved-scan.js.map +1 -1
- package/dist/scan-activity.d.ts.map +1 -1
- package/dist/scan-activity.js +8 -22
- package/dist/scan-activity.js.map +1 -1
- package/dist/scan-comparison.d.ts +3 -3
- package/dist/scan-comparison.d.ts.map +1 -1
- package/dist/scan-comparison.js +38 -73
- package/dist/scan-comparison.js.map +1 -1
- package/dist/scan-dashboard.d.ts +0 -1
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +10 -19
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +1 -2
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/scan-logs.d.ts.map +1 -1
- package/dist/scan-logs.js +2 -6
- package/dist/scan-logs.js.map +1 -1
- package/dist/scan-sessions.d.ts.map +1 -1
- package/dist/scan-sessions.js +1 -3
- package/dist/scan-sessions.js.map +1 -1
- package/dist/scan-settings.d.ts +5 -0
- package/dist/scan-settings.d.ts.map +1 -1
- package/dist/scan-settings.js +4 -0
- package/dist/scan-settings.js.map +1 -1
- package/dist/security-policy-cli.d.ts.map +1 -1
- package/dist/security-policy-cli.js +8 -6
- package/dist/security-policy-cli.js.map +1 -1
- package/dist/security-policy.d.ts +8 -4
- package/dist/security-policy.d.ts.map +1 -1
- package/dist/security-policy.js +98 -84
- package/dist/security-policy.js.map +1 -1
- package/dist/server/dashboard/app.js +5 -5
- package/dist/server/embeddings.d.ts.map +1 -1
- package/dist/server/embeddings.js +1 -0
- package/dist/server/embeddings.js.map +1 -1
- package/dist/server/routes.d.ts +3 -2
- package/dist/server/routes.d.ts.map +1 -1
- package/dist/server/routes.js +19 -7
- package/dist/server/routes.js.map +1 -1
- package/dist/server/server.d.ts.map +1 -1
- package/dist/server/server.js +2 -3
- package/dist/server/server.js.map +1 -1
- package/dist/server/sqlite-store.d.ts.map +1 -1
- package/dist/server/sqlite-store.js +4 -14
- package/dist/server/sqlite-store.js.map +1 -1
- package/dist/severity-store.js +8 -15
- package/dist/severity-store.js.map +1 -1
- package/dist/suggest-owners.d.ts +2 -3
- package/dist/suggest-owners.d.ts.map +1 -1
- package/dist/suggest-owners.js +4 -5
- package/dist/suggest-owners.js.map +1 -1
- package/dist/targets.d.ts +1 -0
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +24 -21
- package/dist/targets.js.map +1 -1
- package/dist/trusted-executable.d.ts +5 -0
- package/dist/trusted-executable.d.ts.map +1 -1
- package/dist/trusted-executable.js +22 -5
- package/dist/trusted-executable.js.map +1 -1
- package/dist/value.d.ts +2 -0
- package/dist/value.d.ts.map +1 -0
- package/dist/value.js +20 -0
- package/dist/value.js.map +1 -0
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +3 -6
- package/dist/version.js.map +1 -1
- package/dist/windows-path.d.ts.map +1 -1
- package/dist/windows-path.js +1 -2
- package/dist/windows-path.js.map +1 -1
- package/dist/worker-progress.d.ts +1 -0
- package/dist/worker-progress.d.ts.map +1 -1
- package/dist/worker-progress.js +13 -34
- package/dist/worker-progress.js.map +1 -1
- package/docs/dedupe-records.md +24 -0
- package/package.json +11 -10
- package/schemas/project-config.schema.json +5 -0
- package/_bundled_plugin/scripts/normalize_candidates.py +0 -338
- package/dist/server/findings-service.d.ts +0 -20
- package/dist/server/findings-service.d.ts.map +0 -1
- package/dist/server/findings-service.js +0 -31
- package/dist/server/findings-service.js.map +0 -1
|
@@ -6,6 +6,8 @@ import argparse
|
|
|
6
6
|
import json
|
|
7
7
|
import re
|
|
8
8
|
import sys
|
|
9
|
+
from bisect import bisect_right
|
|
10
|
+
from itertools import chain
|
|
9
11
|
from pathlib import Path
|
|
10
12
|
from typing import Any
|
|
11
13
|
|
|
@@ -145,9 +147,9 @@ def bounded_finding_details(value: Any) -> dict[str, Any]:
|
|
|
145
147
|
if isinstance(writeup, dict) and isinstance(writeup.get("reportPath"), str):
|
|
146
148
|
prepared["writeup"] = {"reportPath": bounded_json_text(writeup["reportPath"], 512)[0]}
|
|
147
149
|
|
|
148
|
-
evidence_key, evidence =
|
|
150
|
+
evidence_key, evidence = merged_bounded_code_evidence(value)
|
|
149
151
|
if evidence_key is not None:
|
|
150
|
-
prepared[evidence_key] =
|
|
152
|
+
prepared[evidence_key] = evidence
|
|
151
153
|
|
|
152
154
|
for key in (
|
|
153
155
|
"confidence",
|
|
@@ -208,8 +210,9 @@ def bounded_finding_details(value: Any) -> dict[str, Any]:
|
|
|
208
210
|
}
|
|
209
211
|
projected_core = {}
|
|
210
212
|
for selected_guidance in (complete_guidance, minimum_guidance):
|
|
213
|
+
# Keep raw guidance inline to preserve its JSON-encoding recursion boundary.
|
|
211
214
|
reserved = (
|
|
212
|
-
len(json.dumps(selected_guidance, separators=(",", ":"))
|
|
215
|
+
len(json.dumps(selected_guidance, separators=(",", ":"))) - 1
|
|
213
216
|
if selected_guidance
|
|
214
217
|
else 0
|
|
215
218
|
)
|
|
@@ -247,53 +250,34 @@ def bounded_finding_section(
|
|
|
247
250
|
for key in (*priority_keys, *value):
|
|
248
251
|
if key in value and key not in ordered:
|
|
249
252
|
ordered[key] = value[key]
|
|
250
|
-
evidence_key, evidence =
|
|
253
|
+
evidence_key, evidence = merged_bounded_code_evidence(ordered)
|
|
251
254
|
if evidence_key is not None:
|
|
252
|
-
ordered[evidence_key] =
|
|
255
|
+
ordered[evidence_key] = evidence
|
|
253
256
|
ordered.pop("code_evidence" if evidence_key == "codeEvidence" else "codeEvidence", None)
|
|
254
257
|
return bounded_json_value(ordered, [maximum_bytes])
|
|
255
258
|
|
|
256
259
|
|
|
257
|
-
def
|
|
260
|
+
def merged_bounded_code_evidence(value: dict[str, Any]) -> tuple[str | None, Any]:
|
|
258
261
|
evidence_keys = [key for key in ("codeEvidence", "code_evidence") if key in value]
|
|
259
262
|
if not evidence_keys:
|
|
260
263
|
return None, None
|
|
261
264
|
catalogs = [value[key] for key in evidence_keys if isinstance(value[key], list)]
|
|
262
|
-
if catalogs:
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
return evidence_keys[0], value[evidence_keys[0]]
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
def _is_valid_code_evidence(item: Any) -> bool:
|
|
279
|
-
return (
|
|
280
|
-
isinstance(item, dict)
|
|
281
|
-
and isinstance(item.get("id"), str)
|
|
282
|
-
and bool(item["id"].strip())
|
|
283
|
-
and isinstance(item.get("code"), str)
|
|
284
|
-
and bool(item["code"].strip())
|
|
285
|
-
)
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
def bounded_code_evidence(value: Any) -> Any:
|
|
289
|
-
if not isinstance(value, list):
|
|
290
|
-
return value
|
|
291
|
-
bounded = []
|
|
292
|
-
for item in value:
|
|
293
|
-
if not _is_valid_code_evidence(item):
|
|
265
|
+
if not catalogs:
|
|
266
|
+
return evidence_keys[0], value[evidence_keys[0]]
|
|
267
|
+
bounded = {}
|
|
268
|
+
for item in chain.from_iterable(catalogs):
|
|
269
|
+
if not (
|
|
270
|
+
isinstance(item, dict)
|
|
271
|
+
and isinstance(item.get("id"), str)
|
|
272
|
+
and bool(item["id"].strip())
|
|
273
|
+
and isinstance(item.get("code"), str)
|
|
274
|
+
and bool(item["code"].strip())
|
|
275
|
+
):
|
|
276
|
+
continue
|
|
277
|
+
if item["id"] in bounded:
|
|
294
278
|
continue
|
|
295
279
|
if len(bounded) >= FINDING_CODE_EVIDENCE_LIMIT:
|
|
296
|
-
|
|
280
|
+
return evidence_keys[0], list(bounded.values())
|
|
297
281
|
evidence = dict(item)
|
|
298
282
|
for field in ("explanation", "label", "language", "path"):
|
|
299
283
|
if field in evidence and not isinstance(evidence[field], str):
|
|
@@ -316,14 +300,17 @@ def bounded_code_evidence(value: Any) -> Any:
|
|
|
316
300
|
and (not isinstance(end_line, int) or isinstance(end_line, bool) or end_line < 1)
|
|
317
301
|
):
|
|
318
302
|
evidence.pop("endLine")
|
|
319
|
-
code =
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
303
|
+
evidence["code"] = bounded_json_text(
|
|
304
|
+
evidence["code"],
|
|
305
|
+
FINDING_CODE_EVIDENCE_SNIPPET_BYTES,
|
|
306
|
+
)[0]
|
|
307
|
+
bounded[item["id"]] = evidence
|
|
308
|
+
return evidence_keys[0], list(bounded.values())
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def json_size(value: Any) -> int:
|
|
312
|
+
# ASCII output gives byte length; strings can use the cached default encoder.
|
|
313
|
+
return len(json.dumps(value, separators=None if isinstance(value, str) else (",", ":")))
|
|
327
314
|
|
|
328
315
|
|
|
329
316
|
def bounded_json_value(
|
|
@@ -343,7 +330,7 @@ def bounded_json_value(
|
|
|
343
330
|
consume_json_budget(budget, size)
|
|
344
331
|
return bounded
|
|
345
332
|
if value is None or isinstance(value, (bool, int, float)):
|
|
346
|
-
consume_json_budget(budget,
|
|
333
|
+
consume_json_budget(budget, json_size(value))
|
|
347
334
|
return value
|
|
348
335
|
if isinstance(value, list):
|
|
349
336
|
if not consume_json_budget(budget, 2):
|
|
@@ -360,7 +347,7 @@ def bounded_json_value(
|
|
|
360
347
|
depth=depth + 1,
|
|
361
348
|
max_depth=max_depth,
|
|
362
349
|
)
|
|
363
|
-
size =
|
|
350
|
+
size = json_size(bounded_item)
|
|
364
351
|
if separator + size > remaining or (
|
|
365
352
|
isinstance(item, str) and item and bounded_item == ""
|
|
366
353
|
):
|
|
@@ -398,19 +385,9 @@ def bounded_json_value(
|
|
|
398
385
|
and isinstance(controls[0], str)
|
|
399
386
|
and controls[0]
|
|
400
387
|
):
|
|
401
|
-
minimum_tests =
|
|
402
|
-
json.dumps([item[0][0]], separators=(",", ":")).encode("utf-8")
|
|
403
|
-
)
|
|
388
|
+
minimum_tests = json_size([item[0][0]])
|
|
404
389
|
for control in (controls[0], controls[0][0]):
|
|
405
|
-
reserved = (
|
|
406
|
-
len(
|
|
407
|
-
json.dumps(
|
|
408
|
-
{"preventiveControls": [control]},
|
|
409
|
-
separators=(",", ":"),
|
|
410
|
-
).encode("utf-8")
|
|
411
|
-
)
|
|
412
|
-
- 1
|
|
413
|
-
)
|
|
390
|
+
reserved = json_size({"preventiveControls": [control]}) - 1
|
|
414
391
|
if budget[0] >= minimum_tests + reserved:
|
|
415
392
|
item_budget = [budget[0] - reserved]
|
|
416
393
|
break
|
|
@@ -420,12 +397,7 @@ def bounded_json_value(
|
|
|
420
397
|
depth=depth + 1,
|
|
421
398
|
max_depth=max_depth,
|
|
422
399
|
)
|
|
423
|
-
size = (
|
|
424
|
-
separator
|
|
425
|
-
+ key_size
|
|
426
|
-
+ 1
|
|
427
|
-
+ len(json.dumps(bounded_item, separators=(",", ":")).encode("utf-8"))
|
|
428
|
-
)
|
|
400
|
+
size = separator + key_size + 1 + json_size(bounded_item)
|
|
429
401
|
if size > remaining or (isinstance(item, str) and item and bounded_item == ""):
|
|
430
402
|
budget[0] = remaining
|
|
431
403
|
break
|
|
@@ -445,21 +417,9 @@ def consume_json_budget(budget: list[int], size: int) -> bool:
|
|
|
445
417
|
|
|
446
418
|
|
|
447
419
|
def bounded_json_text(value: str, maximum_bytes: int) -> tuple[str, int]:
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
selected
|
|
451
|
-
selected_size = 2
|
|
452
|
-
while low <= high:
|
|
453
|
-
midpoint = (low + high) // 2
|
|
454
|
-
candidate = value[:midpoint]
|
|
455
|
-
size = len(json.dumps(candidate, separators=(",", ":")).encode("utf-8"))
|
|
456
|
-
if size <= maximum_bytes:
|
|
457
|
-
selected = candidate
|
|
458
|
-
selected_size = size
|
|
459
|
-
low = midpoint + 1
|
|
460
|
-
else:
|
|
461
|
-
high = midpoint - 1
|
|
462
|
-
return selected, selected_size
|
|
420
|
+
length = bisect_right(range(len(value) + 1), maximum_bytes, key=lambda n: json_size(value[:n]))
|
|
421
|
+
selected = value[: max(0, length - 1)]
|
|
422
|
+
return selected, json_size(selected)
|
|
463
423
|
|
|
464
424
|
|
|
465
425
|
if __name__ == "__main__":
|
|
@@ -10,6 +10,10 @@ import sys
|
|
|
10
10
|
import tempfile
|
|
11
11
|
from pathlib import Path
|
|
12
12
|
|
|
13
|
+
# Some plugin hosts launch Python with safe-path isolation enabled.
|
|
14
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
15
|
+
from workbench_target import git_blob_bytes, git_command
|
|
16
|
+
|
|
13
17
|
|
|
14
18
|
class InventoryError(ValueError):
|
|
15
19
|
"""Raised when the repository, scope, or inventory cannot be used safely."""
|
|
@@ -90,8 +94,11 @@ def generate_in_scope_files(repository: Path, scope: str, output: Path) -> int:
|
|
|
90
94
|
"--hidden",
|
|
91
95
|
"--path-separator",
|
|
92
96
|
"/",
|
|
97
|
+
# Prune Git metadata and its contents even when the scope starts inside .git.
|
|
98
|
+
"--glob",
|
|
99
|
+
"!**/.git",
|
|
93
100
|
"--glob",
|
|
94
|
-
"
|
|
101
|
+
"!**/.git/**",
|
|
95
102
|
"--",
|
|
96
103
|
scope,
|
|
97
104
|
]
|
|
@@ -116,21 +123,16 @@ def generate_in_scope_files(repository: Path, scope: str, output: Path) -> int:
|
|
|
116
123
|
|
|
117
124
|
if (repository / ".git").exists():
|
|
118
125
|
try:
|
|
119
|
-
tracked =
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
],
|
|
130
|
-
cwd=repository,
|
|
131
|
-
stdout=subprocess.PIPE,
|
|
132
|
-
stderr=subprocess.DEVNULL,
|
|
133
|
-
check=False,
|
|
126
|
+
tracked = git_command(
|
|
127
|
+
repository,
|
|
128
|
+
"ls-files",
|
|
129
|
+
"--cached",
|
|
130
|
+
"--ignored",
|
|
131
|
+
"--exclude-standard",
|
|
132
|
+
"-z",
|
|
133
|
+
"--",
|
|
134
|
+
scope,
|
|
135
|
+
text=False,
|
|
134
136
|
)
|
|
135
137
|
except OSError:
|
|
136
138
|
tracked = None
|
|
@@ -151,26 +153,21 @@ def generate_in_scope_files(repository: Path, scope: str, output: Path) -> int:
|
|
|
151
153
|
"Repository contains a path that cannot fit in the file inventory"
|
|
152
154
|
)
|
|
153
155
|
rows.append(path + b"\n")
|
|
154
|
-
rows.sort()
|
|
155
156
|
|
|
156
|
-
return write_inventory(output, rows)
|
|
157
|
+
return write_inventory(output, sorted(set(rows)))
|
|
157
158
|
|
|
158
159
|
|
|
159
160
|
def committed_changed_paths(repository: Path, base: str, head: str) -> list[tuple[Path, str]]:
|
|
160
|
-
result =
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
"--diff-filter=ACMRD",
|
|
169
|
-
f"{base}..{head}",
|
|
170
|
-
],
|
|
171
|
-
capture_output=True,
|
|
172
|
-
check=True,
|
|
161
|
+
result = git_command(
|
|
162
|
+
repository,
|
|
163
|
+
"diff",
|
|
164
|
+
"--raw",
|
|
165
|
+
"-z",
|
|
166
|
+
"--diff-filter=ACMRD",
|
|
167
|
+
f"{base}..{head}",
|
|
168
|
+
text=False,
|
|
173
169
|
)
|
|
170
|
+
result.check_returncode()
|
|
174
171
|
fields = result.stdout.split(b"\0")
|
|
175
172
|
changed: list[tuple[Path, str]] = []
|
|
176
173
|
index = 0
|
|
@@ -196,7 +193,6 @@ def generate_diff_in_scope_files(
|
|
|
196
193
|
output: Path,
|
|
197
194
|
) -> int:
|
|
198
195
|
"""Reuse the existing diff selection without generating previews or duplicate worklists."""
|
|
199
|
-
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
200
196
|
from generate_rank_input import git_changed_paths, path_is_diff_excluded
|
|
201
197
|
from rank_preview import (
|
|
202
198
|
DEFAULT_PREVIEW_BYTES,
|
|
@@ -204,7 +200,6 @@ def generate_diff_in_scope_files(
|
|
|
204
200
|
is_binary_sample,
|
|
205
201
|
preview_for,
|
|
206
202
|
)
|
|
207
|
-
from workbench_target import git_blob_bytes
|
|
208
203
|
|
|
209
204
|
rows: list[bytes] = []
|
|
210
205
|
try:
|
|
@@ -275,7 +270,8 @@ def write_inventory(output: Path, rows: list[bytes]) -> int:
|
|
|
275
270
|
with tempfile.NamedTemporaryFile(
|
|
276
271
|
mode="wb",
|
|
277
272
|
dir=output.parent,
|
|
278
|
-
|
|
273
|
+
# Including the output name can exceed the filesystem's 255-byte name limit.
|
|
274
|
+
prefix=".",
|
|
279
275
|
suffix=".tmp",
|
|
280
276
|
delete=False,
|
|
281
277
|
) as handle:
|
|
@@ -32,18 +32,19 @@ import re
|
|
|
32
32
|
import subprocess
|
|
33
33
|
import sys
|
|
34
34
|
from collections import Counter
|
|
35
|
-
from collections.abc import Callable
|
|
35
|
+
from collections.abc import Callable, KeysView
|
|
36
36
|
from pathlib import Path
|
|
37
37
|
|
|
38
38
|
# Some plugin hosts launch Python with safe-path isolation enabled.
|
|
39
39
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
40
|
+
from generate_in_scope_files import windows_stream_component
|
|
40
41
|
from rank_preview import (
|
|
41
42
|
DEFAULT_PREVIEW_BYTES,
|
|
42
43
|
TEXT_CODE_EXTENSIONS,
|
|
43
44
|
preview_for,
|
|
44
45
|
preview_for_bytes,
|
|
45
46
|
)
|
|
46
|
-
from workbench_target import git_blob_bytes, git_directory_snapshot_paths
|
|
47
|
+
from workbench_target import git_blob_bytes, git_command, git_directory_snapshot_paths
|
|
47
48
|
|
|
48
49
|
EXCLUDED_DIRS = {
|
|
49
50
|
".cache",
|
|
@@ -129,7 +130,6 @@ RANK_POOL_STRATEGY = "round_robin"
|
|
|
129
130
|
RANK_POOL_WORKER_CAP = 6
|
|
130
131
|
JsonRow = dict[str, object]
|
|
131
132
|
RowValidator = Callable[[JsonRow, Path, int], None]
|
|
132
|
-
RankWorkerAssignment = tuple[int, list[str], list[str]]
|
|
133
133
|
|
|
134
134
|
|
|
135
135
|
def parse_args() -> argparse.Namespace:
|
|
@@ -299,17 +299,6 @@ def path_is_diff_excluded(path: Path) -> bool:
|
|
|
299
299
|
return path_is_excluded(path)
|
|
300
300
|
|
|
301
301
|
|
|
302
|
-
def windows_stream_component(path: Path) -> str | None:
|
|
303
|
-
"""Return the first NTFS alternate-data-stream component."""
|
|
304
|
-
|
|
305
|
-
if os.name != "nt":
|
|
306
|
-
return None
|
|
307
|
-
return next(
|
|
308
|
-
(component for component in path.parts if component != path.anchor and ":" in component),
|
|
309
|
-
None,
|
|
310
|
-
)
|
|
311
|
-
|
|
312
|
-
|
|
313
302
|
def resolve_scope(
|
|
314
303
|
repo: Path,
|
|
315
304
|
scope: str,
|
|
@@ -394,12 +383,11 @@ def load_jsonl(path: Path, label: str, validator: RowValidator) -> list[JsonRow]
|
|
|
394
383
|
if not raw_line.strip():
|
|
395
384
|
raise SystemExit(f"{path}:{line_number}: blank JSONL rows are not allowed")
|
|
396
385
|
try:
|
|
397
|
-
|
|
386
|
+
row: object = json.loads(raw_line)
|
|
398
387
|
except json.JSONDecodeError as exc:
|
|
399
388
|
raise SystemExit(f"{path}:{line_number}: invalid JSON: {exc.msg}") from exc
|
|
400
|
-
if not isinstance(
|
|
389
|
+
if not isinstance(row, dict):
|
|
401
390
|
raise SystemExit(f"{path}:{line_number}: expected a JSON object")
|
|
402
|
-
row = {str(key): value for key, value in parsed.items()}
|
|
403
391
|
validator(row, path, line_number)
|
|
404
392
|
rows.append(row)
|
|
405
393
|
return rows
|
|
@@ -448,16 +436,12 @@ def validate_rank_output_row(row: JsonRow, path: Path, line_number: int) -> None
|
|
|
448
436
|
require_string(row, "reason", path, line_number, allow_empty=False)
|
|
449
437
|
|
|
450
438
|
|
|
451
|
-
def require_unique_paths(rows: list[JsonRow], label: str) ->
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
duplicates.add(path)
|
|
458
|
-
seen.add(path)
|
|
459
|
-
if duplicates:
|
|
460
|
-
raise SystemExit(f"{label} contains duplicate paths: {sorted(duplicates)}")
|
|
439
|
+
def require_unique_paths(rows: list[JsonRow], label: str) -> KeysView[str]:
|
|
440
|
+
counts = Counter(str(row["path"]) for row in rows)
|
|
441
|
+
if len(counts) < len(rows):
|
|
442
|
+
duplicates = sorted(path for path, count in counts.items() if count > 1)
|
|
443
|
+
raise SystemExit(f"{label} contains duplicate paths: {duplicates}")
|
|
444
|
+
return counts.keys()
|
|
461
445
|
|
|
462
446
|
|
|
463
447
|
def make_repo_rank_input(args: argparse.Namespace) -> None:
|
|
@@ -542,8 +526,11 @@ def make_repo_scope_input(args: argparse.Namespace) -> None:
|
|
|
542
526
|
"--hidden",
|
|
543
527
|
"--no-require-git",
|
|
544
528
|
"--null",
|
|
529
|
+
# Also exclude descendants when the scope starts inside .git.
|
|
530
|
+
"--glob",
|
|
531
|
+
"!**/.git",
|
|
545
532
|
"--glob",
|
|
546
|
-
"
|
|
533
|
+
"!**/.git/**",
|
|
547
534
|
"--",
|
|
548
535
|
str(scope_path.relative_to(repo)),
|
|
549
536
|
]
|
|
@@ -624,20 +611,16 @@ def bind_repo_scopes(args: argparse.Namespace) -> None:
|
|
|
624
611
|
|
|
625
612
|
|
|
626
613
|
def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[Path, str]]:
|
|
627
|
-
result =
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
"--diff-filter=ACMRD",
|
|
636
|
-
*diff_args,
|
|
637
|
-
],
|
|
638
|
-
check=True,
|
|
639
|
-
capture_output=True,
|
|
614
|
+
result = git_command(
|
|
615
|
+
repo,
|
|
616
|
+
"diff",
|
|
617
|
+
"--name-status",
|
|
618
|
+
"-z",
|
|
619
|
+
"--diff-filter=ACMRD",
|
|
620
|
+
*diff_args,
|
|
621
|
+
text=False,
|
|
640
622
|
)
|
|
623
|
+
result.check_returncode()
|
|
641
624
|
fields = result.stdout.split(b"\0")
|
|
642
625
|
if fields and not fields[-1]:
|
|
643
626
|
fields.pop()
|
|
@@ -661,11 +644,15 @@ def git_changed_paths(repo: Path, base: str, head: str, mode: str) -> list[tuple
|
|
|
661
644
|
if mode == "local-patch":
|
|
662
645
|
unstaged = run_git_changed_paths(repo, [base])
|
|
663
646
|
staged = run_git_changed_paths(repo, ["--cached", base])
|
|
664
|
-
untracked =
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
647
|
+
untracked = git_command(
|
|
648
|
+
repo,
|
|
649
|
+
"ls-files",
|
|
650
|
+
"--others",
|
|
651
|
+
"--exclude-standard",
|
|
652
|
+
"-z",
|
|
653
|
+
text=False,
|
|
668
654
|
)
|
|
655
|
+
untracked.check_returncode()
|
|
669
656
|
combined = dict(staged)
|
|
670
657
|
combined.update(unstaged)
|
|
671
658
|
combined.update(
|
|
@@ -707,9 +694,8 @@ def make_diff_rank_input(args: argparse.Namespace) -> None:
|
|
|
707
694
|
for path, status in changed:
|
|
708
695
|
rel = path.relative_to(repo)
|
|
709
696
|
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
elif args.mode == "revisions":
|
|
697
|
+
preview = ""
|
|
698
|
+
if status != "D" and args.mode == "revisions":
|
|
713
699
|
content = revision_blobs[rel]
|
|
714
700
|
if content is None:
|
|
715
701
|
raise SystemExit(
|
|
@@ -718,9 +704,7 @@ def make_diff_rank_input(args: argparse.Namespace) -> None:
|
|
|
718
704
|
preview, is_binary = preview_for_bytes(rel, content, args.preview_bytes)
|
|
719
705
|
if is_binary:
|
|
720
706
|
continue
|
|
721
|
-
elif path.is_symlink():
|
|
722
|
-
preview = ""
|
|
723
|
-
elif path.is_file():
|
|
707
|
+
elif status != "D" and not path.is_symlink() and path.is_file():
|
|
724
708
|
try:
|
|
725
709
|
path.resolve(strict=True).relative_to(repo)
|
|
726
710
|
except (OSError, ValueError):
|
|
@@ -729,8 +713,6 @@ def make_diff_rank_input(args: argparse.Namespace) -> None:
|
|
|
729
713
|
preview, is_binary = preview_for(path, args.preview_bytes)
|
|
730
714
|
if is_binary:
|
|
731
715
|
continue
|
|
732
|
-
else:
|
|
733
|
-
preview = ""
|
|
734
716
|
rows.append({"path": rel.as_posix(), "area": args.area, "preview": preview})
|
|
735
717
|
|
|
736
718
|
rows.sort(key=lambda row: str(row["path"]))
|
|
@@ -754,8 +736,7 @@ def make_rank_shards(args: argparse.Namespace) -> None:
|
|
|
754
736
|
raise SystemExit(f"Rank shard directory already contains shard files: {output_dir}")
|
|
755
737
|
|
|
756
738
|
shard_count = 0
|
|
757
|
-
for start in range(0, len(rows), args.max_rows):
|
|
758
|
-
shard_count += 1
|
|
739
|
+
for shard_count, start in enumerate(range(0, len(rows), args.max_rows), start=1):
|
|
759
740
|
shard_path = output_dir / f"rank-shard-{shard_count:04d}.input.jsonl"
|
|
760
741
|
write_jsonl(shard_path, rows[start : start + args.max_rows])
|
|
761
742
|
|
|
@@ -855,7 +836,7 @@ def load_rank_pool_plan(plan_path: Path) -> tuple[dict[str, object], bytes]:
|
|
|
855
836
|
raise SystemExit(f"{plan_path}: invalid JSON: {exc.msg}") from exc
|
|
856
837
|
if not isinstance(payload, dict):
|
|
857
838
|
raise SystemExit(f"{plan_path}: expected a JSON object")
|
|
858
|
-
return
|
|
839
|
+
return payload, plan_bytes
|
|
859
840
|
|
|
860
841
|
|
|
861
842
|
def require_integer(value: object, label: str, *, minimum: int) -> int:
|
|
@@ -872,24 +853,13 @@ def require_string_list(value: object, label: str) -> list[str]:
|
|
|
872
853
|
return [item for item in value if isinstance(item, str)]
|
|
873
854
|
|
|
874
855
|
|
|
875
|
-
def assignment_differences(
|
|
876
|
-
assigned_names: list[str], expected_names: list[str]
|
|
877
|
-
) -> tuple[list[str], list[str], list[str]]:
|
|
878
|
-
counts = Counter(assigned_names)
|
|
879
|
-
duplicates = sorted(name for name, count in counts.items() if count > 1)
|
|
880
|
-
assigned = set(assigned_names)
|
|
881
|
-
expected = set(expected_names)
|
|
882
|
-
return sorted(expected - assigned), duplicates, sorted(assigned - expected)
|
|
883
|
-
|
|
884
|
-
|
|
885
856
|
def validate_rank_pool_plan(
|
|
886
857
|
plan_path: Path, shard_dir: Path
|
|
887
|
-
) -> tuple[list[Path], list[str], list[
|
|
858
|
+
) -> tuple[list[Path], list[tuple[list[str], list[str]]], bytes]:
|
|
888
859
|
require_plan_shard_dir(plan_path, shard_dir)
|
|
889
860
|
require_no_misplaced_rank_shards(plan_path)
|
|
890
861
|
input_shards = discover_input_shards(shard_dir)
|
|
891
862
|
input_names = [path.name for path in input_shards]
|
|
892
|
-
output_names = [output_name_for(name) for name in input_names]
|
|
893
863
|
plan, plan_bytes = load_rank_pool_plan(plan_path)
|
|
894
864
|
expected_fields = {
|
|
895
865
|
"schema_version",
|
|
@@ -937,15 +907,13 @@ def validate_rank_pool_plan(
|
|
|
937
907
|
f"{plan_path}: workers must contain exactly {worker_count} worker assignments"
|
|
938
908
|
)
|
|
939
909
|
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
parsed_workers: list[RankWorkerAssignment] = []
|
|
910
|
+
counts: Counter[str] = Counter()
|
|
911
|
+
parsed_workers: list[tuple[list[str], list[str]]] = []
|
|
943
912
|
worker_fields = {"slot", "input_shards", "output_shards"}
|
|
944
|
-
for worker_index,
|
|
913
|
+
for worker_index, worker in enumerate(workers):
|
|
945
914
|
label = f"{plan_path}: workers[{worker_index}]"
|
|
946
|
-
if not isinstance(
|
|
915
|
+
if not isinstance(worker, dict):
|
|
947
916
|
raise SystemExit(f"{label} must be a JSON object")
|
|
948
|
-
worker = {str(key): value for key, value in raw_worker.items()}
|
|
949
917
|
if set(worker) != worker_fields:
|
|
950
918
|
raise SystemExit(
|
|
951
919
|
f"{label} fields do not match schema; "
|
|
@@ -962,47 +930,40 @@ def validate_rank_pool_plan(
|
|
|
962
930
|
expected_worker_outputs = [output_name_for(name) for name in worker_inputs]
|
|
963
931
|
if worker_outputs != expected_worker_outputs:
|
|
964
932
|
raise SystemExit(f"{label}.output_shards do not match its input_shards")
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
parsed_workers.append((slot, worker_inputs, worker_outputs))
|
|
933
|
+
counts.update(worker_inputs)
|
|
934
|
+
parsed_workers.append((worker_inputs, worker_outputs))
|
|
968
935
|
|
|
969
|
-
|
|
936
|
+
# Canonical input names map one-to-one to the per-worker outputs checked above.
|
|
937
|
+
duplicates = sorted(name for name, count in counts.items() if count > 1)
|
|
938
|
+
expected = set(input_names)
|
|
939
|
+
missing = sorted(expected - counts.keys())
|
|
940
|
+
unexpected = sorted(counts.keys() - expected)
|
|
970
941
|
if missing or duplicates or unexpected:
|
|
971
942
|
raise SystemExit(
|
|
972
943
|
f"{plan_path}: pool plan must assign each input shard exactly once; "
|
|
973
944
|
f"missing={missing}; duplicates={duplicates}; unexpected={unexpected}"
|
|
974
945
|
)
|
|
975
|
-
|
|
976
|
-
if missing or duplicates or unexpected:
|
|
977
|
-
raise SystemExit(
|
|
978
|
-
f"{plan_path}: pool plan must assign each output shard exactly once; "
|
|
979
|
-
f"missing={missing}; duplicates={duplicates}; unexpected={unexpected}"
|
|
980
|
-
)
|
|
981
|
-
|
|
982
|
-
for worker_index, (_, worker_inputs, worker_outputs) in enumerate(parsed_workers):
|
|
946
|
+
for worker_index, (worker_inputs, _) in enumerate(parsed_workers):
|
|
983
947
|
expected_inputs = input_names[worker_index::worker_count]
|
|
984
|
-
|
|
985
|
-
if worker_inputs != expected_inputs or worker_outputs != expected_outputs:
|
|
948
|
+
if worker_inputs != expected_inputs:
|
|
986
949
|
raise SystemExit(
|
|
987
950
|
f"{plan_path}: worker slot {worker_index + 1} does not match the deterministic "
|
|
988
951
|
f"{RANK_POOL_STRATEGY} assignment"
|
|
989
952
|
)
|
|
990
|
-
return input_shards,
|
|
953
|
+
return input_shards, parsed_workers, plan_bytes
|
|
991
954
|
|
|
992
955
|
|
|
993
956
|
def validate_rank_worker_command(args: argparse.Namespace) -> None:
|
|
994
957
|
plan_path = Path(args.plan).expanduser()
|
|
995
958
|
shard_dir = Path(args.shard_dir).expanduser()
|
|
996
|
-
_,
|
|
959
|
+
_, workers, plan_bytes = validate_rank_pool_plan(plan_path, shard_dir)
|
|
997
960
|
|
|
998
961
|
slot = require_integer(args.slot, "--slot", minimum=1)
|
|
999
962
|
worker_count = len(workers)
|
|
1000
963
|
if slot > worker_count:
|
|
1001
964
|
raise SystemExit(f"--slot must be at most {worker_count}")
|
|
1002
965
|
|
|
1003
|
-
|
|
1004
|
-
if assigned_slot != slot:
|
|
1005
|
-
raise SystemExit(f"{plan_path}: worker assignment for slot {slot} is inconsistent")
|
|
966
|
+
input_names, output_names = workers[slot - 1]
|
|
1006
967
|
|
|
1007
968
|
row_count = 0
|
|
1008
969
|
outputs_digest = hashlib.sha256()
|
|
@@ -1033,10 +994,10 @@ def validate_rank_worker_command(args: argparse.Namespace) -> None:
|
|
|
1033
994
|
def validate_rank_pool_command(args: argparse.Namespace) -> None:
|
|
1034
995
|
plan_path = Path(args.plan).expanduser()
|
|
1035
996
|
shard_dir = Path(args.shard_dir).expanduser()
|
|
1036
|
-
input_shards,
|
|
997
|
+
input_shards, workers, _ = validate_rank_pool_plan(plan_path, shard_dir)
|
|
1037
998
|
|
|
1038
999
|
actual_output_names = {path.name for path in shard_dir.glob(SHARD_OUTPUT_GLOB)}
|
|
1039
|
-
expected_outputs =
|
|
1000
|
+
expected_outputs = {output_name_for(path.name) for path in input_shards}
|
|
1040
1001
|
if actual_output_names != expected_outputs:
|
|
1041
1002
|
missing = sorted(expected_outputs - actual_output_names)
|
|
1042
1003
|
unexpected = sorted(actual_output_names - expected_outputs)
|
|
@@ -1060,12 +1021,10 @@ def validate_rank_shard(
|
|
|
1060
1021
|
input_shard: Path, output_shard: Path
|
|
1061
1022
|
) -> tuple[list[JsonRow], list[JsonRow]]:
|
|
1062
1023
|
shard_inputs = load_jsonl(input_shard, "Rank input shard", validate_rank_input_row)
|
|
1063
|
-
require_unique_paths(shard_inputs, f"Rank input shard {input_shard.name}")
|
|
1024
|
+
expected_paths = require_unique_paths(shard_inputs, f"Rank input shard {input_shard.name}")
|
|
1064
1025
|
shard_outputs = load_jsonl(output_shard, "Rank output shard", validate_rank_output_row)
|
|
1065
|
-
require_unique_paths(shard_outputs, f"Rank output shard {output_shard.name}")
|
|
1026
|
+
actual_paths = require_unique_paths(shard_outputs, f"Rank output shard {output_shard.name}")
|
|
1066
1027
|
|
|
1067
|
-
expected_paths = {str(row["path"]) for row in shard_inputs}
|
|
1068
|
-
actual_paths = {str(row["path"]) for row in shard_outputs}
|
|
1069
1028
|
if expected_paths != actual_paths:
|
|
1070
1029
|
missing = sorted(expected_paths - actual_paths)
|
|
1071
1030
|
unknown = sorted(actual_paths - expected_paths)
|
|
@@ -10,7 +10,11 @@ if [ "${1:-}" = "--helper" ]; then
|
|
|
10
10
|
server_path=$launcher_dir/../mcp/helpers.mjs
|
|
11
11
|
shift
|
|
12
12
|
# Node decodes argv and HOME as UTF-8; preserve POSIX bytes first.
|
|
13
|
-
|
|
13
|
+
# A separate descriptor preserves stdin and avoids a single argument limit.
|
|
14
|
+
exec 3<<EOF
|
|
15
|
+
$(printf '%s\000' "${HOME+x}" "${HOME-}" "$@" | od -An -v -tx1 | tr -d ' \n')
|
|
16
|
+
EOF
|
|
17
|
+
set -- --helper
|
|
14
18
|
fi
|
|
15
19
|
cache_root=${XDG_CACHE_HOME:-${HOME:-}/.cache}
|
|
16
20
|
codex_resources=
|