@openai/codex-security 0.8.1 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -57
- package/_bundled_plugin/.app.json +4 -0
- package/_bundled_plugin/.mcp.json +1 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +4 -4
- package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/permission-profile-preflight.mjs +46 -12
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-002 +0 -0
- package/_bundled_plugin/references/artifact-storage.md +1 -1
- package/_bundled_plugin/references/core-scan.md +14 -5
- package/_bundled_plugin/references/default-severity-rubric.md +28 -0
- package/_bundled_plugin/references/finding-detail-fields.md +1 -0
- package/_bundled_plugin/references/threat-model.md +1 -0
- package/_bundled_plugin/scripts/deep_scan_workbench.py +23 -112
- package/_bundled_plugin/scripts/finalize_scan_contract.py +167 -153
- package/_bundled_plugin/scripts/finding_preview.py +5 -29
- package/_bundled_plugin/scripts/posix_scan_local_files.py +187 -0
- package/_bundled_plugin/scripts/report_projection.py +87 -136
- package/_bundled_plugin/scripts/validate_scan_contract.py +2 -6
- package/_bundled_plugin/scripts/windows_scan_local_files.py +212 -55
- package/_bundled_plugin/scripts/workbench/retained_coverage.py +8 -24
- package/_bundled_plugin/scripts/workbench/saved_coverage.py +143 -231
- package/_bundled_plugin/scripts/workbench_cli.py +102 -13
- package/_bundled_plugin/scripts/workbench_constants.py +9 -0
- package/_bundled_plugin/scripts/workbench_coverage_receipts.py +0 -17
- package/_bundled_plugin/scripts/workbench_db.py +187 -363
- package/_bundled_plugin/scripts/workbench_finding_workflows.py +86 -4
- package/_bundled_plugin/scripts/workbench_native_indexes.py +17 -21
- package/_bundled_plugin/scripts/workbench_remediation.py +495 -11
- package/_bundled_plugin/scripts/workbench_saved_results.py +0 -66
- package/_bundled_plugin/scripts/workbench_scan_history.py +22 -25
- package/_bundled_plugin/scripts/workbench_scan_start.py +1 -1
- package/_bundled_plugin/scripts/workbench_scan_usage.py +25 -52
- package/_bundled_plugin/scripts/workbench_schema.py +1 -1
- package/_bundled_plugin/scripts/workbench_setup.py +239 -0
- package/_bundled_plugin/scripts/workbench_source_scopes.py +15 -23
- package/_bundled_plugin/scripts/workbench_target.py +120 -35
- package/_bundled_plugin/scripts/workbench_validation.py +1 -9
- package/_bundled_plugin/shared/workbench-history-repairs.json +7 -0
- package/_bundled_plugin/shared/workbench-migrations.json +19 -0
- package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +1 -1
- package/_bundled_plugin/skills/attack-path-analysis/references/severity-policy.md +4 -68
- package/_bundled_plugin/skills/check-skills/SKILL.md +43 -0
- package/_bundled_plugin/skills/check-skills/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/cloud-findings/SKILL.md +2 -0
- package/_bundled_plugin/skills/fix-finding/SKILL.md +18 -1
- package/_bundled_plugin/skills/publish-findings/SKILL.md +68 -35
- package/_bundled_plugin/skills/publish-findings/references/wiz-import.md +2 -21
- package/_bundled_plugin/skills/resolve-dependency-alert/SKILL.md +18 -0
- package/_bundled_plugin/skills/resolve-dependency-alert/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/threat-model-visualize/SKILL.md +83 -0
- package/_bundled_plugin/skills/threat-model-visualize/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/threat-model-visualize/assets/context-reference-overview.svg +366 -0
- package/_bundled_plugin/skills/threat-model-visualize/assets/context-reference-path.svg +558 -0
- package/_bundled_plugin/skills/threat-model-visualize/assets/control-effects.js +85 -0
- package/_bundled_plugin/skills/threat-model-visualize/assets/explorer.css +17 -0
- package/_bundled_plugin/skills/threat-model-visualize/assets/explorer.html +38 -0
- package/_bundled_plugin/skills/threat-model-visualize/assets/explorer.js +372 -0
- package/_bundled_plugin/skills/threat-model-visualize/assets/route-paths.js +71 -0
- package/_bundled_plugin/skills/threat-model-visualize/assets/source-roles.js +47 -0
- package/_bundled_plugin/skills/threat-model-visualize/assets/svg-layout.js +149 -0
- package/_bundled_plugin/skills/threat-model-visualize/references/context-explorer.md +21 -0
- package/_bundled_plugin/skills/threat-model-visualize/references/view-model.md +49 -0
- package/_bundled_plugin/skills/threat-model-visualize/scripts/build_explorer.py +569 -0
- package/_bundled_plugin/skills/track-findings/SKILL.md +5 -3
- package/_bundled_plugin/skills/triage-finding/references/github-rest-intake.md +22 -68
- package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +6 -4
- package/dist/api.d.ts +6 -1
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +414 -163
- package/dist/api.js.map +1 -1
- package/dist/auth.js +7 -2
- package/dist/auth.js.map +1 -1
- package/dist/blackduck-findings.d.ts +8 -0
- package/dist/blackduck-findings.d.ts.map +1 -0
- package/dist/blackduck-findings.js +305 -0
- package/dist/blackduck-findings.js.map +1 -0
- package/dist/bulk-scan-discovery.d.ts.map +1 -1
- package/dist/bulk-scan-discovery.js +51 -36
- package/dist/bulk-scan-discovery.js.map +1 -1
- package/dist/classify-severity-records.d.ts +5 -0
- package/dist/classify-severity-records.d.ts.map +1 -1
- package/dist/classify-severity-records.js +6 -8
- package/dist/classify-severity-records.js.map +1 -1
- package/dist/classify-severity.d.ts +4 -1
- package/dist/classify-severity.d.ts.map +1 -1
- package/dist/classify-severity.js +28 -9
- package/dist/classify-severity.js.map +1 -1
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +635 -266
- package/dist/cli.js.map +1 -1
- package/dist/codex-prompt.d.ts +2 -0
- package/dist/codex-prompt.d.ts.map +1 -1
- package/dist/codex-prompt.js +6 -0
- package/dist/codex-prompt.js.map +1 -1
- package/dist/component-scan.d.ts.map +1 -1
- package/dist/component-scan.js +6 -5
- package/dist/component-scan.js.map +1 -1
- package/dist/config.d.ts +2 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +162 -40
- package/dist/config.js.map +1 -1
- package/dist/continuum-findings.d.ts +7 -0
- package/dist/continuum-findings.d.ts.map +1 -0
- package/dist/continuum-findings.js +157 -0
- package/dist/continuum-findings.js.map +1 -0
- package/dist/contract-values.d.ts +28 -0
- package/dist/contract-values.d.ts.map +1 -0
- package/dist/contract-values.js +75 -0
- package/dist/contract-values.js.map +1 -0
- package/dist/contract.d.ts +2 -20
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +58 -669
- package/dist/contract.js.map +1 -1
- package/dist/cost-model.d.ts.map +1 -1
- package/dist/cost-model.js +1 -4
- package/dist/cost-model.js.map +1 -1
- package/dist/cost.d.ts +19 -1
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +2120 -39
- package/dist/cost.js.map +1 -1
- package/dist/custom-validation-prompt.d.ts +1 -1
- package/dist/custom-validation-prompt.d.ts.map +1 -1
- package/dist/custom-validation-prompt.js +5 -0
- package/dist/custom-validation-prompt.js.map +1 -1
- package/dist/custom-validation.d.ts +2 -0
- package/dist/custom-validation.d.ts.map +1 -1
- package/dist/custom-validation.js +6 -2
- package/dist/custom-validation.js.map +1 -1
- package/dist/deduplication/deduplication-prompts.d.ts +4 -4
- package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
- package/dist/deduplication/deduplication-prompts.js +2 -2
- package/dist/deduplication/deduplication-prompts.js.map +1 -1
- package/dist/deduplication/deduplication-reviewer.d.ts +14 -11
- package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -1
- package/dist/deduplication/deduplication-reviewer.js +12 -5
- package/dist/deduplication/deduplication-reviewer.js.map +1 -1
- package/dist/deduplication/deduplication.js.map +1 -1
- package/dist/deduplication/finding-schema.d.ts +19 -1
- package/dist/deduplication/finding-schema.d.ts.map +1 -1
- package/dist/deduplication/finding-schema.js +25 -0
- package/dist/deduplication/finding-schema.js.map +1 -1
- package/dist/deduplication/local.d.ts.map +1 -1
- package/dist/deduplication/local.js +3 -8
- package/dist/deduplication/local.js.map +1 -1
- package/dist/deduplication/records.d.ts +11 -3
- package/dist/deduplication/records.d.ts.map +1 -1
- package/dist/deduplication/records.js +48 -16
- package/dist/deduplication/records.js.map +1 -1
- package/dist/deep-scan.d.ts +11 -0
- package/dist/deep-scan.d.ts.map +1 -1
- package/dist/deep-scan.js +235 -4
- package/dist/deep-scan.js.map +1 -1
- package/dist/default-severity-rubric.d.ts +1 -2
- package/dist/default-severity-rubric.d.ts.map +1 -1
- package/dist/default-severity-rubric.js +2 -66
- package/dist/default-severity-rubric.js.map +1 -1
- package/dist/endor-findings.d.ts.map +1 -1
- package/dist/endor-findings.js +44 -89
- package/dist/endor-findings.js.map +1 -1
- package/dist/errors.js +1 -1
- package/dist/errors.js.map +1 -1
- package/dist/external-findings-import.d.ts.map +1 -1
- package/dist/external-findings-import.js +11 -15
- package/dist/external-findings-import.js.map +1 -1
- package/dist/external-findings-local.d.ts +12 -1
- package/dist/external-findings-local.d.ts.map +1 -1
- package/dist/external-findings-local.js +411 -184
- package/dist/external-findings-local.js.map +1 -1
- package/dist/external-findings-publish.d.ts +15 -1
- package/dist/external-findings-publish.d.ts.map +1 -1
- package/dist/external-findings-publish.js +1238 -226
- package/dist/external-findings-publish.js.map +1 -1
- package/dist/external-findings-read.d.ts +10 -0
- package/dist/external-findings-read.d.ts.map +1 -1
- package/dist/external-findings-read.js +71 -51
- package/dist/external-findings-read.js.map +1 -1
- package/dist/external-findings.d.ts +13 -0
- package/dist/external-findings.d.ts.map +1 -1
- package/dist/external-findings.js +124 -1
- package/dist/external-findings.js.map +1 -1
- package/dist/external-import-contract.d.ts +29 -1
- package/dist/external-import-contract.d.ts.map +1 -1
- package/dist/external-import-contract.js +314 -44
- package/dist/external-import-contract.js.map +1 -1
- package/dist/external-import-models.d.ts +1 -1
- package/dist/external-import-models.d.ts.map +1 -1
- package/dist/external-import-models.js.map +1 -1
- package/dist/finding-normalization.d.ts +1 -1
- package/dist/finding-normalization.d.ts.map +1 -1
- package/dist/finding-normalization.js +46 -11
- package/dist/finding-normalization.js.map +1 -1
- package/dist/finding-retrieval.d.ts +3 -3
- package/dist/finding-retrieval.d.ts.map +1 -1
- package/dist/finding-retrieval.js.map +1 -1
- package/dist/finding-workflow.d.ts +6 -1
- package/dist/finding-workflow.d.ts.map +1 -1
- package/dist/finding-workflow.js +2 -2
- package/dist/finding-workflow.js.map +1 -1
- package/dist/git-remote.d.ts +4 -0
- package/dist/git-remote.d.ts.map +1 -1
- package/dist/git-remote.js +20 -0
- package/dist/git-remote.js.map +1 -1
- package/dist/github-advisories.d.ts +3 -12
- package/dist/github-advisories.d.ts.map +1 -1
- package/dist/github-advisories.js +9 -29
- package/dist/github-advisories.js.map +1 -1
- package/dist/github-dependabot-findings.d.ts.map +1 -1
- package/dist/github-dependabot-findings.js +7 -16
- package/dist/github-dependabot-findings.js.map +1 -1
- package/dist/github-findings.d.ts.map +1 -1
- package/dist/github-findings.js +344 -56
- package/dist/github-findings.js.map +1 -1
- package/dist/github.d.ts +17 -7
- package/dist/github.d.ts.map +1 -1
- package/dist/github.js +43 -30
- package/dist/github.js.map +1 -1
- package/dist/host-protocol.d.ts +7 -2
- package/dist/host-protocol.d.ts.map +1 -1
- package/dist/host-protocol.js +27 -3
- package/dist/host-protocol.js.map +1 -1
- package/dist/host-records-protocol.d.ts +10 -11
- package/dist/host-records-protocol.d.ts.map +1 -1
- package/dist/host-records-protocol.js +69 -9
- package/dist/host-records-protocol.js.map +1 -1
- package/dist/hosted-scan.d.ts +15 -0
- package/dist/hosted-scan.d.ts.map +1 -1
- package/dist/hosted-scan.js +55 -33
- package/dist/hosted-scan.js.map +1 -1
- package/dist/index.d.ts +8 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -0
- package/dist/index.js.map +1 -1
- package/dist/inspector-findings.d.ts +9 -0
- package/dist/inspector-findings.d.ts.map +1 -0
- package/dist/inspector-findings.js +512 -0
- package/dist/inspector-findings.js.map +1 -0
- package/dist/knowledge-base.js +18 -5
- package/dist/knowledge-base.js.map +1 -1
- package/dist/local-git-metadata.d.ts +5 -0
- package/dist/local-git-metadata.d.ts.map +1 -0
- package/dist/local-git-metadata.js +131 -0
- package/dist/local-git-metadata.js.map +1 -0
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +22 -12
- package/dist/multiscan.js.map +1 -1
- package/dist/opengrep-findings.d.ts.map +1 -1
- package/dist/opengrep-findings.js +34 -20
- package/dist/opengrep-findings.js.map +1 -1
- package/dist/patch-tui.d.ts +1 -1
- package/dist/patch-tui.d.ts.map +1 -1
- package/dist/patch-tui.js +94 -36
- package/dist/patch-tui.js.map +1 -1
- package/dist/private-finding-import.d.ts +246 -0
- package/dist/private-finding-import.d.ts.map +1 -0
- package/dist/private-finding-import.js +387 -0
- package/dist/private-finding-import.js.map +1 -0
- package/dist/project-config.d.ts.map +1 -1
- package/dist/project-config.js +2 -2
- package/dist/project-config.js.map +1 -1
- package/dist/provider-profile.d.ts +10 -2
- package/dist/provider-profile.d.ts.map +1 -1
- package/dist/provider-profile.js +72 -10
- package/dist/provider-profile.js.map +1 -1
- package/dist/publication.js +7 -1
- package/dist/publication.js.map +1 -1
- package/dist/reader-selection.d.ts +13 -0
- package/dist/reader-selection.d.ts.map +1 -0
- package/dist/reader-selection.js +81 -0
- package/dist/reader-selection.js.map +1 -0
- package/dist/record.d.ts +6 -1
- package/dist/record.d.ts.map +1 -1
- package/dist/record.js +53 -2
- package/dist/record.js.map +1 -1
- package/dist/remediation.d.ts +5 -0
- package/dist/remediation.d.ts.map +1 -1
- package/dist/remediation.js +30 -0
- package/dist/remediation.js.map +1 -1
- package/dist/result.d.ts +2 -0
- package/dist/result.d.ts.map +1 -1
- package/dist/result.js +3 -1
- package/dist/result.js.map +1 -1
- package/dist/runtime.d.ts +5 -3
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +110 -57
- package/dist/runtime.js.map +1 -1
- package/dist/scan-activity.js +40 -17
- package/dist/scan-activity.js.map +1 -1
- package/dist/scan-comparison.d.ts +1 -1
- package/dist/scan-comparison.d.ts.map +1 -1
- package/dist/scan-comparison.js +11 -13
- package/dist/scan-comparison.js.map +1 -1
- package/dist/scan-dashboard.d.ts +1 -0
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +37 -48
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-executor.d.ts +2 -0
- package/dist/scan-executor.d.ts.map +1 -1
- package/dist/scan-executor.js.map +1 -1
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +22 -9
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/scan-logs.d.ts.map +1 -1
- package/dist/scan-logs.js +5 -5
- package/dist/scan-logs.js.map +1 -1
- package/dist/scanner-import.d.ts +48 -3
- package/dist/scanner-import.d.ts.map +1 -1
- package/dist/scanner-import.js +214 -39
- package/dist/scanner-import.js.map +1 -1
- package/dist/scanner-repository-url.d.ts +14 -0
- package/dist/scanner-repository-url.d.ts.map +1 -0
- package/dist/scanner-repository-url.js +67 -0
- package/dist/scanner-repository-url.js.map +1 -0
- package/dist/semgrep-findings.d.ts.map +1 -1
- package/dist/semgrep-findings.js +50 -88
- package/dist/semgrep-findings.js.map +1 -1
- package/dist/semgrep-platform-findings.d.ts +12 -0
- package/dist/semgrep-platform-findings.d.ts.map +1 -0
- package/dist/semgrep-platform-findings.js +198 -0
- package/dist/semgrep-platform-findings.js.map +1 -0
- package/dist/severity-rubric.d.ts +20 -0
- package/dist/severity-rubric.d.ts.map +1 -0
- package/dist/severity-rubric.js +73 -0
- package/dist/severity-rubric.js.map +1 -0
- package/dist/severity-store.js +15 -5
- package/dist/severity-store.js.map +1 -1
- package/dist/snyk-findings.d.ts.map +1 -1
- package/dist/snyk-findings.js +97 -106
- package/dist/snyk-findings.js.map +1 -1
- package/dist/sonar-collector.d.ts +16 -0
- package/dist/sonar-collector.d.ts.map +1 -0
- package/dist/sonar-collector.js +383 -0
- package/dist/sonar-collector.js.map +1 -0
- package/dist/sonar-findings.d.ts +49 -0
- package/dist/sonar-findings.d.ts.map +1 -0
- package/dist/sonar-findings.js +288 -0
- package/dist/sonar-findings.js.map +1 -0
- package/dist/sonar-import.d.ts +111 -0
- package/dist/sonar-import.d.ts.map +1 -0
- package/dist/sonar-import.js +279 -0
- package/dist/sonar-import.js.map +1 -0
- package/dist/suggest-owners.d.ts.map +1 -1
- package/dist/suggest-owners.js +2 -1
- package/dist/suggest-owners.js.map +1 -1
- package/dist/targets.d.ts +3 -0
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +18 -12
- package/dist/targets.js.map +1 -1
- package/dist/terminal-columns.d.ts +4 -0
- package/dist/terminal-columns.d.ts.map +1 -0
- package/dist/terminal-columns.js +19 -0
- package/dist/terminal-columns.js.map +1 -0
- package/dist/validation-records.d.ts +0 -3
- package/dist/validation-records.d.ts.map +1 -1
- package/dist/validation-records.js +0 -16
- package/dist/validation-records.js.map +1 -1
- package/dist/veracode-findings.d.ts +7 -0
- package/dist/veracode-findings.d.ts.map +1 -0
- package/dist/veracode-findings.js +371 -0
- package/dist/veracode-findings.js.map +1 -0
- package/dist/windows-path.d.ts.map +1 -1
- package/dist/windows-path.js +1 -0
- package/dist/windows-path.js.map +1 -1
- package/dist/wiz-csv.d.ts +2 -2
- package/dist/wiz-csv.d.ts.map +1 -1
- package/dist/wiz-csv.js +1 -3
- package/dist/wiz-csv.js.map +1 -1
- package/dist/wiz-findings.d.ts +10 -2
- package/dist/wiz-findings.d.ts.map +1 -1
- package/dist/wiz-findings.js +88 -49
- package/dist/wiz-findings.js.map +1 -1
- package/docs/classify-severity-records.md +1 -1
- package/docs/cli.md +320 -481
- package/docs/dedupe-records.md +27 -4
- package/docs/findings-service.md +21 -8
- package/docs/hosted-scans.md +16 -1
- package/package.json +9 -10
- package/schemas/external-findings.schema.json +12 -1
- package/schemas/hosted-scan-v2.fixture.json +16 -1
- package/_bundled_plugin/scripts/validate_tracking_source.py +0 -82
- package/_bundled_plugin/scripts/workbench_local_dedupe.py +0 -136
- package/dist/deduplication/records-protocol.d.ts +0 -4
- package/dist/deduplication/records-protocol.d.ts.map +0 -1
- package/dist/deduplication/records-protocol.js +0 -35
- package/dist/deduplication/records-protocol.js.map +0 -1
|
@@ -29,7 +29,7 @@ import importlib
|
|
|
29
29
|
import ntpath
|
|
30
30
|
import os
|
|
31
31
|
import secrets
|
|
32
|
-
from collections.abc import Iterator
|
|
32
|
+
from collections.abc import Callable, Iterator
|
|
33
33
|
from pathlib import Path, PurePosixPath
|
|
34
34
|
from typing import BinaryIO
|
|
35
35
|
|
|
@@ -100,6 +100,16 @@ _RESERVED_DEVICE_NAMES = {
|
|
|
100
100
|
}
|
|
101
101
|
|
|
102
102
|
|
|
103
|
+
class _FileBasicInfo(ctypes.Structure):
|
|
104
|
+
_fields_ = [
|
|
105
|
+
("CreationTime", ctypes.c_int64),
|
|
106
|
+
("LastAccessTime", ctypes.c_int64),
|
|
107
|
+
("LastWriteTime", ctypes.c_int64),
|
|
108
|
+
("ChangeTime", ctypes.c_int64),
|
|
109
|
+
("FileAttributes", wintypes.DWORD),
|
|
110
|
+
]
|
|
111
|
+
|
|
112
|
+
|
|
103
113
|
class _FileAttributeTagInfo(ctypes.Structure):
|
|
104
114
|
_fields_ = [
|
|
105
115
|
("FileAttributes", wintypes.DWORD),
|
|
@@ -182,6 +192,10 @@ if os.name == "nt":
|
|
|
182
192
|
]
|
|
183
193
|
_CreateFileW.restype = wintypes.HANDLE
|
|
184
194
|
|
|
195
|
+
_ReOpenFile = _kernel32.ReOpenFile
|
|
196
|
+
_ReOpenFile.argtypes = [wintypes.HANDLE, wintypes.DWORD, wintypes.DWORD, wintypes.DWORD]
|
|
197
|
+
_ReOpenFile.restype = wintypes.HANDLE
|
|
198
|
+
|
|
185
199
|
_CreateDirectoryW = _kernel32.CreateDirectoryW
|
|
186
200
|
_CreateDirectoryW.argtypes = [wintypes.LPCWSTR, wintypes.LPVOID]
|
|
187
201
|
_CreateDirectoryW.restype = wintypes.BOOL
|
|
@@ -443,13 +457,17 @@ def _canonical_scan_directory(scan_dir: Path) -> tuple[Path, tuple[int, int]]:
|
|
|
443
457
|
|
|
444
458
|
|
|
445
459
|
def _open_directory(
|
|
446
|
-
path: Path,
|
|
460
|
+
path: Path,
|
|
461
|
+
*,
|
|
462
|
+
missing_ok: bool = False,
|
|
463
|
+
missing_is_path_error: bool = True,
|
|
464
|
+
movable: bool = False,
|
|
447
465
|
) -> _OwnedHandle | None:
|
|
448
466
|
try:
|
|
449
467
|
handle = _create_file(
|
|
450
468
|
path,
|
|
451
469
|
access=_FILE_READ_ATTRIBUTES,
|
|
452
|
-
share=_FILE_SHARE_READ | _FILE_SHARE_WRITE,
|
|
470
|
+
share=_FILE_SHARE_READ | _FILE_SHARE_WRITE | (_FILE_SHARE_DELETE if movable else 0),
|
|
453
471
|
disposition=_OPEN_EXISTING,
|
|
454
472
|
flags=_FILE_FLAG_BACKUP_SEMANTICS | _FILE_FLAG_OPEN_REPARSE_POINT,
|
|
455
473
|
missing_ok=missing_ok,
|
|
@@ -606,6 +624,17 @@ def open_read_fd_with_path(scan_dir: Path, relative_path: str, context: str) ->
|
|
|
606
624
|
) from exc
|
|
607
625
|
|
|
608
626
|
|
|
627
|
+
def file_change_time(descriptor: int) -> int:
|
|
628
|
+
"""Return the filesystem change time. Python's Windows st_ctime is creation time."""
|
|
629
|
+
assert _msvcrt is not None
|
|
630
|
+
info = _FileBasicInfo()
|
|
631
|
+
if not _GetFileInformationByHandleEx(
|
|
632
|
+
_msvcrt.get_osfhandle(descriptor), 0, ctypes.byref(info), ctypes.sizeof(info)
|
|
633
|
+
):
|
|
634
|
+
_raise_last_error("GetFileInformationByHandleEx(FileBasicInfo)")
|
|
635
|
+
return info.ChangeTime
|
|
636
|
+
|
|
637
|
+
|
|
609
638
|
def stream_matches_payload(stream: BinaryIO, payload: bytes) -> bool:
|
|
610
639
|
offset = 0
|
|
611
640
|
while offset < len(payload):
|
|
@@ -632,7 +661,7 @@ def _write_all(handle: int, payload: bytes) -> None:
|
|
|
632
661
|
_raise_last_error("FlushFileBuffers")
|
|
633
662
|
|
|
634
663
|
|
|
635
|
-
def _rename_handle(handle: int, destination_path: Path) -> None:
|
|
664
|
+
def _rename_handle(handle: int, destination_path: Path, replace_existing: bool) -> None:
|
|
636
665
|
encoded_name = _extended_path(destination_path).encode("utf-16-le")
|
|
637
666
|
file_name_offset = _FileRenameInfo.FileName.offset
|
|
638
667
|
buffer_size = max(
|
|
@@ -641,7 +670,7 @@ def _rename_handle(handle: int, destination_path: Path) -> None:
|
|
|
641
670
|
)
|
|
642
671
|
buffer = ctypes.create_string_buffer(buffer_size)
|
|
643
672
|
info = ctypes.cast(buffer, ctypes.POINTER(_FileRenameInfo)).contents
|
|
644
|
-
info.FlagsOrReplaceIfExists =
|
|
673
|
+
info.FlagsOrReplaceIfExists = int(replace_existing)
|
|
645
674
|
# The fully qualified destination uses a null RootDirectory. Verified
|
|
646
675
|
# directory handles remain open to prevent ancestor replacement.
|
|
647
676
|
info.RootDirectory = None
|
|
@@ -653,6 +682,8 @@ def _rename_handle(handle: int, destination_path: Path) -> None:
|
|
|
653
682
|
buffer,
|
|
654
683
|
buffer_size,
|
|
655
684
|
):
|
|
685
|
+
if not replace_existing and ctypes.get_last_error() in _COLLISION_ERRORS:
|
|
686
|
+
raise FileExistsError(errno.EEXIST, os.strerror(errno.EEXIST), str(destination_path))
|
|
656
687
|
_raise_last_error("SetFileInformationByHandle(FileRenameInfo)")
|
|
657
688
|
|
|
658
689
|
|
|
@@ -667,7 +698,7 @@ def _mark_handle_for_deletion(handle: int) -> None:
|
|
|
667
698
|
_raise_last_error("SetFileInformationByHandle(FileDispositionInfo)")
|
|
668
699
|
|
|
669
700
|
|
|
670
|
-
def _validate_existing_output(path: Path) ->
|
|
701
|
+
def _validate_existing_output(path: Path) -> bool:
|
|
671
702
|
handle = _create_file(
|
|
672
703
|
path,
|
|
673
704
|
access=_FILE_READ_ATTRIBUTES,
|
|
@@ -677,10 +708,11 @@ def _validate_existing_output(path: Path) -> None:
|
|
|
677
708
|
missing_ok=True,
|
|
678
709
|
)
|
|
679
710
|
if handle is None:
|
|
680
|
-
return
|
|
711
|
+
return False
|
|
681
712
|
with handle:
|
|
682
713
|
assert handle.value is not None
|
|
683
714
|
_verify_regular_file(handle.value, path)
|
|
715
|
+
return True
|
|
684
716
|
|
|
685
717
|
|
|
686
718
|
def _existing_output_matches(path: Path, payload: bytes) -> bool:
|
|
@@ -723,64 +755,189 @@ def _existing_output_matches(path: Path, payload: bytes) -> bool:
|
|
|
723
755
|
return False
|
|
724
756
|
|
|
725
757
|
|
|
758
|
+
@contextlib.contextmanager
|
|
759
|
+
def _written_temp(destination: Path, payload: bytes) -> Iterator[tuple[Path, _OwnedHandle]]:
|
|
760
|
+
for _ in range(16):
|
|
761
|
+
path = destination.with_name(f".{destination.name}.{secrets.token_hex(8)}.tmp")
|
|
762
|
+
try:
|
|
763
|
+
handle = _create_file(
|
|
764
|
+
path,
|
|
765
|
+
access=_GENERIC_WRITE | _DELETE | _FILE_READ_ATTRIBUTES,
|
|
766
|
+
share=0,
|
|
767
|
+
disposition=_CREATE_NEW,
|
|
768
|
+
flags=_FILE_ATTRIBUTE_NORMAL,
|
|
769
|
+
)
|
|
770
|
+
except WindowsScanLocalFileError as exc:
|
|
771
|
+
if exc.errno in _COLLISION_ERRORS:
|
|
772
|
+
continue
|
|
773
|
+
raise
|
|
774
|
+
assert handle is not None and handle.value is not None
|
|
775
|
+
with handle:
|
|
776
|
+
try:
|
|
777
|
+
_verify_regular_file(handle.value, path)
|
|
778
|
+
_write_all(handle.value, payload)
|
|
779
|
+
yield path, handle
|
|
780
|
+
except BaseException:
|
|
781
|
+
if handle.value is not None:
|
|
782
|
+
with contextlib.suppress(OSError):
|
|
783
|
+
_mark_handle_for_deletion(handle.value)
|
|
784
|
+
raise
|
|
785
|
+
return
|
|
786
|
+
raise WindowsScanLocalFileError(errno.EEXIST, "could not allocate a unique temp file")
|
|
787
|
+
|
|
788
|
+
|
|
789
|
+
def _stage_metadata(
|
|
790
|
+
handle: _OwnedHandle, *, close_writer: bool = False
|
|
791
|
+
) -> tuple[os.stat_result, int]:
|
|
792
|
+
# Windows can update LastWriteTime on last writer close. Observe the same
|
|
793
|
+
# file across that close, independently of whatever is now at the name.
|
|
794
|
+
assert handle.value is not None and _msvcrt is not None
|
|
795
|
+
raw = _ReOpenFile(
|
|
796
|
+
handle.value,
|
|
797
|
+
_FILE_READ_ATTRIBUTES,
|
|
798
|
+
_FILE_SHARE_READ | _FILE_SHARE_WRITE | _FILE_SHARE_DELETE,
|
|
799
|
+
0,
|
|
800
|
+
)
|
|
801
|
+
if raw is None or raw == _INVALID_HANDLE_VALUE:
|
|
802
|
+
_raise_last_error("ReOpenFile")
|
|
803
|
+
with _OwnedHandle(raw) as observer:
|
|
804
|
+
descriptor = _msvcrt.open_osfhandle(raw, os.O_RDONLY | os.O_BINARY)
|
|
805
|
+
observer.detach()
|
|
806
|
+
try:
|
|
807
|
+
if close_writer:
|
|
808
|
+
handle.close()
|
|
809
|
+
try:
|
|
810
|
+
return os.fstat(descriptor), file_change_time(descriptor)
|
|
811
|
+
except BaseException:
|
|
812
|
+
if close_writer:
|
|
813
|
+
# The observer still identifies the staged file after the writer closed.
|
|
814
|
+
cleanup = _ReOpenFile(
|
|
815
|
+
raw, _DELETE, _FILE_SHARE_READ | _FILE_SHARE_WRITE | _FILE_SHARE_DELETE, 0
|
|
816
|
+
)
|
|
817
|
+
if cleanup and cleanup != _INVALID_HANDLE_VALUE:
|
|
818
|
+
with _OwnedHandle(cleanup), contextlib.suppress(OSError):
|
|
819
|
+
_mark_handle_for_deletion(cleanup)
|
|
820
|
+
raise
|
|
821
|
+
finally:
|
|
822
|
+
os.close(descriptor)
|
|
823
|
+
|
|
824
|
+
|
|
825
|
+
@contextlib.contextmanager
|
|
826
|
+
def stage_write(
|
|
827
|
+
scan_dir: Path,
|
|
828
|
+
relative_path: str,
|
|
829
|
+
payload: bytes,
|
|
830
|
+
*,
|
|
831
|
+
expected_root_identity: tuple[int, int] | None = None,
|
|
832
|
+
replace_existing: bool = False,
|
|
833
|
+
) -> Iterator[Callable[..., tuple[os.stat_result, int]]]:
|
|
834
|
+
"""Flush a patch and keep only a movable handle to its root across admission."""
|
|
835
|
+
if expected_root_identity is None:
|
|
836
|
+
root = scan_dir.lstat()
|
|
837
|
+
expected_root_identity = (root.st_dev, root.st_ino)
|
|
838
|
+
with _locked_parent(
|
|
839
|
+
scan_dir, relative_path, create=True, expected_root_identity=expected_root_identity
|
|
840
|
+
) as (parent, leaf):
|
|
841
|
+
destination = parent / leaf
|
|
842
|
+
if _validate_existing_output(destination) and not replace_existing:
|
|
843
|
+
raise FileExistsError(errno.EEXIST, os.strerror(errno.EEXIST), str(destination))
|
|
844
|
+
root_handle = _open_directory(scan_dir, movable=True)
|
|
845
|
+
assert root_handle is not None
|
|
846
|
+
try:
|
|
847
|
+
with _written_temp(destination, payload) as (temp, writer):
|
|
848
|
+
staged, change_time = _stage_metadata(writer, close_writer=True)
|
|
849
|
+
except BaseException:
|
|
850
|
+
root_handle.close()
|
|
851
|
+
raise
|
|
852
|
+
|
|
853
|
+
location = scan_dir
|
|
854
|
+
published = False
|
|
855
|
+
|
|
856
|
+
@contextlib.contextmanager
|
|
857
|
+
def open_stage() -> Iterator[tuple[_OwnedHandle, Path]]:
|
|
858
|
+
with _locked_parent(
|
|
859
|
+
location, relative_path, create=False, expected_root_identity=expected_root_identity
|
|
860
|
+
) as (parent, leaf):
|
|
861
|
+
path = parent / temp.name
|
|
862
|
+
handle = _create_file(
|
|
863
|
+
path,
|
|
864
|
+
access=_DELETE | _FILE_READ_ATTRIBUTES,
|
|
865
|
+
share=0,
|
|
866
|
+
disposition=_OPEN_EXISTING,
|
|
867
|
+
flags=_FILE_FLAG_OPEN_REPARSE_POINT | _FILE_FLAG_BACKUP_SEMANTICS,
|
|
868
|
+
)
|
|
869
|
+
assert handle is not None and handle.value is not None
|
|
870
|
+
with handle:
|
|
871
|
+
_verify_regular_file(handle.value, path)
|
|
872
|
+
current, changed = _stage_metadata(handle)
|
|
873
|
+
if not (
|
|
874
|
+
os.path.samestat(staged, current)
|
|
875
|
+
and staged.st_size == current.st_size
|
|
876
|
+
and staged.st_mtime_ns == current.st_mtime_ns
|
|
877
|
+
and change_time == changed
|
|
878
|
+
):
|
|
879
|
+
raise _invalid_path(path, "staged patch changed before publication")
|
|
880
|
+
yield handle, parent / leaf
|
|
881
|
+
|
|
882
|
+
def publish(directory: Path | None = None) -> tuple[os.stat_result, int]:
|
|
883
|
+
nonlocal location, published
|
|
884
|
+
if directory is not None:
|
|
885
|
+
location = directory
|
|
886
|
+
with open_stage() as (handle, destination):
|
|
887
|
+
assert handle.value is not None
|
|
888
|
+
_rename_handle(handle.value, destination, replace_existing)
|
|
889
|
+
published = True
|
|
890
|
+
_verify_regular_file(handle.value, destination)
|
|
891
|
+
return _stage_metadata(handle)
|
|
892
|
+
|
|
893
|
+
with root_handle:
|
|
894
|
+
try:
|
|
895
|
+
yield publish
|
|
896
|
+
finally:
|
|
897
|
+
if not published:
|
|
898
|
+
# A different workbench database may own the archive move.
|
|
899
|
+
assert root_handle.value is not None
|
|
900
|
+
with contextlib.suppress(OSError):
|
|
901
|
+
location = Path(_final_path(root_handle.value))
|
|
902
|
+
while True:
|
|
903
|
+
try:
|
|
904
|
+
with open_stage() as (handle, _):
|
|
905
|
+
assert handle.value is not None
|
|
906
|
+
_mark_handle_for_deletion(handle.value)
|
|
907
|
+
break
|
|
908
|
+
except OSError:
|
|
909
|
+
# A move can land between locating the root and fixing its parents.
|
|
910
|
+
moved_to = Path(_final_path(root_handle.value))
|
|
911
|
+
if moved_to == location:
|
|
912
|
+
break
|
|
913
|
+
location = moved_to
|
|
914
|
+
|
|
915
|
+
|
|
726
916
|
def atomic_write(
|
|
727
917
|
scan_dir: Path,
|
|
728
918
|
relative_path: str,
|
|
729
919
|
payload: bytes,
|
|
730
920
|
*,
|
|
731
921
|
expected_root_identity: tuple[int, int] | None = None,
|
|
922
|
+
replace_existing: bool = True,
|
|
732
923
|
) -> None:
|
|
733
|
-
"""Atomically
|
|
734
|
-
|
|
924
|
+
"""Atomically publish ``payload`` to a scan-local regular file."""
|
|
735
925
|
with _locked_parent(
|
|
736
|
-
scan_dir,
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
926
|
+
scan_dir, relative_path, create=True, expected_root_identity=expected_root_identity
|
|
927
|
+
) as (parent, leaf):
|
|
928
|
+
destination = parent / leaf
|
|
929
|
+
if _validate_existing_output(destination) and not replace_existing:
|
|
930
|
+
raise FileExistsError(errno.EEXIST, os.strerror(errno.EEXIST), str(destination))
|
|
931
|
+
if (
|
|
932
|
+
replace_existing
|
|
933
|
+
and expected_root_identity is not None
|
|
934
|
+
and _existing_output_matches(destination, payload)
|
|
745
935
|
):
|
|
746
936
|
return
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
temp_path = parent_path / f".{leaf_name}.{secrets.token_hex(8)}.tmp"
|
|
752
|
-
try:
|
|
753
|
-
temp_handle = _create_file(
|
|
754
|
-
temp_path,
|
|
755
|
-
access=_GENERIC_WRITE | _DELETE | _FILE_READ_ATTRIBUTES,
|
|
756
|
-
share=0,
|
|
757
|
-
disposition=_CREATE_NEW,
|
|
758
|
-
flags=_FILE_ATTRIBUTE_NORMAL,
|
|
759
|
-
)
|
|
760
|
-
except WindowsScanLocalFileError as exc:
|
|
761
|
-
if exc.errno in _COLLISION_ERRORS:
|
|
762
|
-
continue
|
|
763
|
-
raise
|
|
764
|
-
break
|
|
765
|
-
if temp_handle is None or temp_path is None:
|
|
766
|
-
raise WindowsScanLocalFileError(errno.EEXIST, "could not allocate a unique temp file")
|
|
767
|
-
|
|
768
|
-
with temp_handle:
|
|
769
|
-
assert temp_handle.value is not None
|
|
770
|
-
try:
|
|
771
|
-
_verify_regular_file(temp_handle.value, temp_path)
|
|
772
|
-
_write_all(temp_handle.value, payload)
|
|
773
|
-
_rename_handle(temp_handle.value, destination_path)
|
|
774
|
-
_verify_regular_file(temp_handle.value, destination_path)
|
|
775
|
-
except BaseException:
|
|
776
|
-
# Deleting by handle removes the exact temp/output file and cannot
|
|
777
|
-
# be redirected through a swapped path or reparse point.
|
|
778
|
-
try:
|
|
779
|
-
_mark_handle_for_deletion(temp_handle.value)
|
|
780
|
-
except OSError:
|
|
781
|
-
# Cleanup must not replace the original write or rename failure.
|
|
782
|
-
pass
|
|
783
|
-
raise
|
|
937
|
+
with _written_temp(destination, payload) as (_, handle):
|
|
938
|
+
assert handle.value is not None
|
|
939
|
+
_rename_handle(handle.value, destination, replace_existing)
|
|
940
|
+
_verify_regular_file(handle.value, destination)
|
|
784
941
|
|
|
785
942
|
|
|
786
943
|
def unlink_if_exists(scan_dir: Path, relative_path: str) -> None:
|
|
@@ -47,6 +47,7 @@ from workbench.retained_records import (
|
|
|
47
47
|
same_raw_value,
|
|
48
48
|
)
|
|
49
49
|
from workbench.saved_coverage import (
|
|
50
|
+
SavedCoverageProjection,
|
|
50
51
|
_coverage_rows,
|
|
51
52
|
_has_candidate_metadata,
|
|
52
53
|
_inherit_coverage_sources,
|
|
@@ -730,14 +731,6 @@ def merge_saved_results(
|
|
|
730
731
|
parent_model_source = latest_reducer
|
|
731
732
|
|
|
732
733
|
projected_coverage = saved_parent["reviewState"] if saved_parent else {}
|
|
733
|
-
reviews = projected_coverage.get("reviews")
|
|
734
|
-
reviewed_attempts = {
|
|
735
|
-
(review.get("workerId"), review.get("attempt"))
|
|
736
|
-
for review in (reviews if isinstance(reviews, list) else [])
|
|
737
|
-
if isinstance(review, dict)
|
|
738
|
-
and isinstance(review.get("workerId"), str)
|
|
739
|
-
and isinstance(review.get("attempt"), int)
|
|
740
|
-
}
|
|
741
734
|
# Reconcile raw source IDs and timestamps before projecting accepted outputs.
|
|
742
735
|
accepted_sources = {
|
|
743
736
|
relative
|
|
@@ -747,12 +740,7 @@ def merge_saved_results(
|
|
|
747
740
|
and worker["result_manifest_path"] == str(scan_dir / relative)
|
|
748
741
|
and draft.get("complete") is not False
|
|
749
742
|
}
|
|
750
|
-
|
|
751
|
-
(worker_id, workers_by_id[worker_id]["attempt"])
|
|
752
|
-
for relative, _, worker_id in sources
|
|
753
|
-
if relative in accepted_sources
|
|
754
|
-
)
|
|
755
|
-
projection = prepare_saved_coverage_projection(
|
|
743
|
+
project_worker_coverage, legacy_surface_links = prepare_saved_coverage_projection(
|
|
756
744
|
sources,
|
|
757
745
|
parent,
|
|
758
746
|
source_order,
|
|
@@ -762,7 +750,6 @@ def merge_saved_results(
|
|
|
762
750
|
scan_dir, row, workers_by_id[owner], relative
|
|
763
751
|
),
|
|
764
752
|
)
|
|
765
|
-
project_worker_coverage = projection.project_worker_coverage
|
|
766
753
|
|
|
767
754
|
def retain_coverage_row(output, item, owner, field, original=None, qualified=False):
|
|
768
755
|
retained = projection.retain_coverage_row(output, item, owner, field, original, qualified)
|
|
@@ -800,15 +787,14 @@ def merge_saved_results(
|
|
|
800
787
|
id(item): (worker_id, relative, field, index, draft["reviewState"])
|
|
801
788
|
for relative, draft, worker_id in sources
|
|
802
789
|
for field in ("surfaces", "explicitExclusions", "deferred", "openQuestions")
|
|
803
|
-
for index, item in enumerate(
|
|
804
|
-
draft["reviewState"].get(field, [])
|
|
805
|
-
if isinstance(draft["reviewState"].get(field, []), list)
|
|
806
|
-
else [],
|
|
807
|
-
1,
|
|
808
|
-
)
|
|
790
|
+
for index, item in enumerate(_coverage_rows(draft["reviewState"], field), 1)
|
|
809
791
|
if isinstance(item, dict)
|
|
810
792
|
}
|
|
811
793
|
|
|
794
|
+
projection = SavedCoverageProjection(
|
|
795
|
+
project_worker_coverage, legacy_surface_links, projection_origins
|
|
796
|
+
)
|
|
797
|
+
|
|
812
798
|
def project_record(
|
|
813
799
|
field: str,
|
|
814
800
|
item: dict[str, Any],
|
|
@@ -861,7 +847,6 @@ def merge_saved_results(
|
|
|
861
847
|
return result
|
|
862
848
|
|
|
863
849
|
all_sources = ([("parent", parent, None)] if parent else []) + sources
|
|
864
|
-
source_owners = {relative: owner for relative, _, owner in all_sources}
|
|
865
850
|
inferred_coverage_ids: set[int] = set()
|
|
866
851
|
# Older checkpoints can omit IDs already assigned in their published output.
|
|
867
852
|
for field in ("deferred", "surfaces"):
|
|
@@ -2675,7 +2660,7 @@ def merge_saved_results(
|
|
|
2675
2660
|
projection = type(initial_projection)(
|
|
2676
2661
|
initial_projection.project_worker_coverage,
|
|
2677
2662
|
initial_projection.legacy_surface_links,
|
|
2678
|
-
|
|
2663
|
+
projection_origins,
|
|
2679
2664
|
)
|
|
2680
2665
|
# Apply reconciled outcomes to coverage already copied from the parent too.
|
|
2681
2666
|
for field in ("surfaces", "deferred"):
|
|
@@ -2964,7 +2949,6 @@ def merge_saved_results(
|
|
|
2964
2949
|
if coverage_superseded and not selected_candidates and not retain_pending:
|
|
2965
2950
|
continue
|
|
2966
2951
|
if project_coverage:
|
|
2967
|
-
reviewed_attempts.add((worker_id, worker["attempt"]))
|
|
2968
2952
|
reviews = coverage.setdefault("reviews", [])
|
|
2969
2953
|
if isinstance(reviews, list) and not any(
|
|
2970
2954
|
isinstance(review, dict)
|