@open-mercato/cli 0.6.7 → 0.6.8-develop.6875.1.871a4afc94
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +4 -4
- package/build.mjs +24 -7
- package/dist/agentic/claude-code/hooks/entity-migration-check.ts +32 -88
- package/dist/agentic/codex/enforcement-rules.md +2 -16
- package/dist/agentic/cursor/hooks/entity-migration-check.mjs +19 -50
- package/dist/agentic/cursor/rules/entity-guard.mdc +9 -21
- package/dist/agentic/cursor/rules/generated-guard.mdc +3 -5
- package/dist/agentic/cursor/rules/open-mercato.mdc +3 -29
- package/dist/agentic/guides/ai-workflows.md +76 -0
- package/dist/agentic/guides/architecture.md +107 -0
- package/dist/agentic/guides/backend-ui.md +80 -0
- package/dist/agentic/guides/contracts.md +92 -0
- package/dist/agentic/guides/extensions.md +121 -0
- package/dist/agentic/guides/framework-extension-points.md +36 -0
- package/dist/agentic/guides/integrations.md +81 -0
- package/dist/agentic/guides/module-facts.json +115537 -3951
- package/dist/agentic/guides/module-system.md +12 -127
- package/dist/agentic/guides/modules/ai_assistant.md +227 -28
- package/dist/agentic/guides/modules/api_docs.md +58 -2
- package/dist/agentic/guides/modules/api_keys.md +92 -7
- package/dist/agentic/guides/modules/attachments.md +119 -14
- package/dist/agentic/guides/modules/audit_logs.md +131 -10
- package/dist/agentic/guides/modules/auth.md +276 -33
- package/dist/agentic/guides/modules/business_rules.md +149 -14
- package/dist/agentic/guides/modules/catalog.md +475 -43
- package/dist/agentic/guides/modules/channel_gmail.md +82 -3
- package/dist/agentic/guides/modules/channel_imap.md +82 -3
- package/dist/agentic/guides/modules/checkout.md +299 -27
- package/dist/agentic/guides/modules/communication_channels.md +315 -33
- package/dist/agentic/guides/modules/configs.md +104 -8
- package/dist/agentic/guides/modules/content.md +57 -2
- package/dist/agentic/guides/modules/currencies.md +152 -17
- package/dist/agentic/guides/modules/customer_accounts.md +387 -45
- package/dist/agentic/guides/modules/customers.md +746 -85
- package/dist/agentic/guides/modules/dashboards.md +155 -10
- package/dist/agentic/guides/modules/data_sync.md +149 -16
- package/dist/agentic/guides/modules/design_system.md +102 -0
- package/dist/agentic/guides/modules/dictionaries.md +107 -12
- package/dist/agentic/guides/modules/directory.md +148 -15
- package/dist/agentic/guides/modules/entities.md +145 -13
- package/dist/agentic/guides/modules/events.md +59 -2
- package/dist/agentic/guides/modules/feature_toggles.md +119 -9
- package/dist/agentic/guides/modules/gateway_stripe.md +92 -4
- package/dist/agentic/guides/modules/generators.md +45 -2
- package/dist/agentic/guides/modules/inbox_ops.md +207 -27
- package/dist/agentic/guides/modules/integrations.md +146 -16
- package/dist/agentic/guides/modules/messages.md +223 -24
- package/dist/agentic/guides/modules/notifications.md +96 -8
- package/dist/agentic/guides/modules/onboarding.md +77 -6
- package/dist/agentic/guides/modules/payment_gateways.md +164 -21
- package/dist/agentic/guides/modules/perspectives.md +77 -7
- package/dist/agentic/guides/modules/planner.md +136 -18
- package/dist/agentic/guides/modules/portal.md +93 -2
- package/dist/agentic/guides/modules/progress.md +104 -14
- package/dist/agentic/guides/modules/query_index.md +154 -15
- package/dist/agentic/guides/modules/record_locks.md +206 -21
- package/dist/agentic/guides/modules/resources.md +197 -30
- package/dist/agentic/guides/modules/sales.md +678 -84
- package/dist/agentic/guides/modules/scheduler.md +133 -14
- package/dist/agentic/guides/modules/search.md +170 -6
- package/dist/agentic/guides/modules/security.md +294 -35
- package/dist/agentic/guides/modules/shipping_carriers.md +134 -18
- package/dist/agentic/guides/modules/sso.md +151 -26
- package/dist/agentic/guides/modules/staff.md +369 -57
- package/dist/agentic/guides/modules/storage_s3.md +95 -5
- package/dist/agentic/guides/modules/sync_akeneo.md +85 -3
- package/dist/agentic/guides/modules/sync_excel.md +75 -6
- package/dist/agentic/guides/modules/system_status_overlays.md +60 -2
- package/dist/agentic/guides/modules/translations.md +104 -10
- package/dist/agentic/guides/modules/webhooks.md +203 -24
- package/dist/agentic/guides/modules/wms.md +441 -46
- package/dist/agentic/guides/modules/workflows.md +302 -43
- package/dist/agentic/guides/spec-delivery.md +22 -0
- package/dist/agentic/guides/testing-debugging.md +80 -0
- package/dist/agentic/guides/upstream/AGENTS.md +299 -0
- package/dist/agentic/guides/upstream/BACKWARD_COMPATIBILITY.md +316 -0
- package/dist/agentic/guides/upstream/manifest.json +8 -0
- package/dist/agentic/shared/AGENTS.md.template +124 -311
- package/dist/agentic/shared/ai/agentic.config.json +2 -0
- package/dist/agentic/shared/ai/harness/README.md +50 -0
- package/dist/agentic/shared/ai/harness/RELEASE.md +57 -0
- package/dist/agentic/shared/ai/harness/cases.json +13680 -0
- package/dist/agentic/shared/ai/harness/cases.schema.json +133 -0
- package/dist/agentic/shared/ai/harness/fixtures/index.json +49 -0
- package/dist/agentic/shared/ai/harness/fixtures/seeds.json +162 -0
- package/dist/agentic/shared/ai/harness/generated-code-review-policy.md +21 -0
- package/dist/agentic/shared/ai/harness/generated-code-review-response.schema.json +96 -0
- package/dist/agentic/shared/ai/harness/generated-code-review-result.schema.json +172 -0
- package/dist/agentic/shared/ai/harness/release-matrix.json +158 -0
- package/dist/agentic/shared/ai/harness/release-result.schema.json +219 -0
- package/dist/agentic/shared/ai/harness/result.schema.json +76 -0
- package/dist/agentic/shared/ai/harness/routing-response.schema.json +12 -0
- package/dist/agentic/shared/ai/harness/target-validation-result.schema.json +73 -0
- package/dist/agentic/shared/ai/harness/validators.json +78 -0
- package/dist/agentic/shared/ai/harness/writable-ast-oracles.mjs +2121 -0
- package/dist/agentic/shared/ai/harness/writable-behavior-oracles.mjs +1096 -0
- package/dist/agentic/shared/ai/lessons/_template.md +16 -0
- package/dist/agentic/shared/ai/lessons.md +26 -3
- package/dist/agentic/shared/ai/review-checklist.md +36 -0
- package/dist/agentic/shared/ai/skills/om-auto-implement-spec/SKILL.md +21 -0
- package/dist/agentic/shared/ai/skills/om-auto-review-pr/SKILL.md +4 -0
- package/dist/agentic/shared/ai/skills/om-auto-upgrade-0.6.6-to-0.6.7/SKILL.md +98 -0
- package/dist/agentic/shared/ai/skills/om-backend-ui-design/SKILL.md +20 -241
- package/dist/agentic/shared/ai/skills/om-backend-ui-design/references/crud-surfaces.md +16 -0
- package/dist/agentic/shared/ai/skills/om-backend-ui-design/references/frontend-and-design-system.md +53 -0
- package/dist/agentic/shared/ai/skills/om-backend-ui-design/references/page-and-navigation.md +13 -0
- package/dist/agentic/shared/ai/skills/om-backend-ui-design/references/quality-states.md +11 -0
- package/dist/agentic/shared/ai/skills/om-build-workflow/SKILL.md +30 -0
- package/dist/agentic/shared/ai/skills/om-build-workflow/references/activity-contracts.md +12 -0
- package/dist/agentic/shared/ai/skills/om-build-workflow/references/durability-and-progress.md +13 -0
- package/dist/agentic/shared/ai/skills/om-build-workflow/references/workflow-design.md +14 -0
- package/dist/agentic/shared/ai/skills/om-create-ai-agent/SKILL.md +28 -0
- package/dist/agentic/shared/ai/skills/om-create-ai-agent/references/attachments-and-overrides.md +12 -0
- package/dist/agentic/shared/ai/skills/om-create-ai-agent/references/module-agents-and-tools.md +22 -0
- package/dist/agentic/shared/ai/skills/om-create-ai-agent/references/orchestrator-agents.md +12 -0
- package/dist/agentic/shared/ai/skills/om-create-ai-agent/references/surface-selector.md +19 -0
- package/dist/agentic/shared/ai/skills/om-data-model-design/SKILL.md +16 -601
- package/dist/agentic/shared/ai/skills/om-data-model-design/references/integrity-and-concurrency.md +19 -0
- package/dist/agentic/shared/ai/skills/om-data-model-design/references/migration-workflow.md +13 -0
- package/dist/agentic/shared/ai/skills/om-data-model-design/references/schema-design.md +14 -0
- package/dist/agentic/shared/ai/skills/om-data-model-design/references/sensitive-data.md +10 -0
- package/dist/agentic/shared/ai/skills/om-eject-and-customize/SKILL.md +14 -310
- package/dist/agentic/shared/ai/skills/om-eject-and-customize/references/decision-and-procedure.md +17 -0
- package/dist/agentic/shared/ai/skills/om-evolve-harness/SKILL.md +29 -0
- package/dist/agentic/shared/ai/skills/om-evolve-harness/references/case-template.md +100 -0
- package/dist/agentic/shared/ai/skills/om-evolve-harness/references/case-workflow.md +16 -0
- package/dist/agentic/shared/ai/skills/om-evolve-harness/references/owner-selection.md +17 -0
- package/dist/agentic/shared/ai/skills/om-framework-context/SKILL.md +25 -0
- package/dist/agentic/shared/ai/skills/om-framework-context/references/resolver-procedure.md +11 -0
- package/dist/agentic/shared/ai/skills/om-framework-context/references/skew-and-escalation.md +13 -0
- package/dist/agentic/shared/ai/skills/om-help/SKILL.md +15 -97
- package/dist/agentic/shared/ai/skills/om-help/references/delivery-workflows.md +18 -0
- package/dist/agentic/shared/ai/skills/om-help/references/task-families.md +20 -0
- package/dist/agentic/shared/ai/skills/om-implement-spec/SKILL.md +33 -152
- package/dist/agentic/shared/ai/skills/om-implement-spec/references/phases-and-gates.md +33 -0
- package/dist/agentic/shared/ai/skills/om-implement-spec/references/planning-and-progress.md +40 -0
- package/dist/agentic/shared/ai/skills/om-implement-spec/references/report-templates.md +27 -0
- package/dist/agentic/shared/ai/skills/om-implement-spec/references/spec-resolution.md +24 -0
- package/dist/agentic/shared/ai/skills/om-integration-builder/SKILL.md +24 -708
- package/dist/agentic/shared/ai/skills/om-integration-builder/references/package-and-activation.md +16 -0
- package/dist/agentic/shared/ai/skills/om-integration-builder/references/provider-families.md +14 -0
- package/dist/agentic/shared/ai/skills/om-integration-builder/references/security-and-reliability.md +15 -0
- package/dist/agentic/shared/ai/skills/om-integration-builder/references/sync-and-files.md +12 -0
- package/dist/agentic/shared/ai/skills/om-judge-agent-session/SKILL.md +34 -0
- package/dist/agentic/shared/ai/skills/om-judge-agent-session/agents/openai.yaml +4 -0
- package/dist/agentic/shared/ai/skills/om-judge-agent-session/references/agentic-setup.md +11 -0
- package/dist/agentic/shared/ai/skills/om-judge-agent-session/references/input-normalization.md +31 -0
- package/dist/agentic/shared/ai/skills/om-judge-agent-session/references/judge-workflow.md +39 -0
- package/dist/agentic/shared/ai/skills/om-judge-agent-session/references/report-template.md +43 -0
- package/dist/agentic/shared/ai/skills/om-judge-agent-session/references/rules.md +11 -0
- package/dist/agentic/shared/ai/skills/om-module-scaffold/SKILL.md +20 -820
- package/dist/agentic/shared/ai/skills/om-module-scaffold/references/api-and-domain.md +19 -0
- package/dist/agentic/shared/ai/skills/om-module-scaffold/references/business-one-shot-blueprints.md +172 -0
- package/dist/agentic/shared/ai/skills/om-module-scaffold/references/data-and-migrations.md +12 -0
- package/dist/agentic/shared/ai/skills/om-module-scaffold/references/discovery-surface-catalog.md +83 -0
- package/dist/agentic/shared/ai/skills/om-module-scaffold/references/module-surfaces.md +21 -0
- package/dist/agentic/shared/ai/skills/om-module-scaffold/references/planning.md +11 -0
- package/dist/agentic/shared/ai/skills/om-module-scaffold/references/runtime-cache-and-queues.md +22 -0
- package/dist/agentic/shared/ai/skills/om-module-scaffold/references/verification.md +13 -0
- package/dist/agentic/shared/ai/skills/om-share-this-session/SKILL.md +41 -0
- package/dist/agentic/shared/ai/skills/om-share-this-session/references/agentic-setup.md +19 -0
- package/dist/agentic/shared/ai/skills/om-share-this-session/references/bundle-preparation.md +44 -0
- package/dist/agentic/shared/ai/skills/om-share-this-session/references/consent-and-review.md +39 -0
- package/dist/agentic/shared/ai/skills/om-share-this-session/references/publication.md +20 -0
- package/dist/agentic/shared/ai/skills/om-share-this-session/references/report-templates.md +57 -0
- package/dist/agentic/shared/ai/skills/om-share-this-session/references/rules.md +11 -0
- package/dist/agentic/shared/ai/skills/om-share-this-session/scripts/prepare-share-bundle.mjs +486 -0
- package/dist/agentic/shared/ai/skills/om-system-extension/SKILL.md +19 -843
- package/dist/agentic/shared/ai/skills/om-system-extension/references/extension-branches.md +18 -0
- package/dist/agentic/shared/ai/skills/om-system-extension/references/mechanism-selector.md +27 -0
- package/dist/agentic/shared/ai/skills/om-system-extension/references/read-write-roundtrip.md +13 -0
- package/dist/agentic/shared/ai/skills/om-system-extension/references/unified-overrides.md +41 -0
- package/dist/agentic/shared/ai/skills/om-system-extension/references/verification.md +11 -0
- package/dist/agentic/shared/ai/skills/om-trim-unused-modules/SKILL.md +15 -63
- package/dist/agentic/shared/ai/skills/om-trim-unused-modules/references/dependency-and-redirect.md +15 -0
- package/dist/agentic/shared/ai/skills/om-troubleshooter/SKILL.md +16 -467
- package/dist/agentic/shared/ai/skills/om-troubleshooter/references/diagnosis-map.md +16 -0
- package/dist/agentic/shared/ai/skills/om-troubleshooter/references/regression-oracles.md +14 -0
- package/dist/agentic/shared/ai/skills/tiers.json +143 -21
- package/dist/agentic/shared/ai/skills/tiers.schema.json +52 -40
- package/dist/agentic/shared/ai/specs/README.md +4 -3
- package/dist/agentic/shared/ai/specs/SPEC-000-template.md +226 -10
- package/dist/agentic/shared/ai/trackers/github.md +108 -4
- package/dist/agentic/shared/scripts/agent-harness-tool-server.mjs +187 -0
- package/dist/agentic/shared/scripts/check-lessons.mjs +178 -0
- package/dist/agentic/shared/scripts/evaluate-agent-harness.mjs +2958 -0
- package/dist/agentic/shared/scripts/execution-sandbox.mjs +329 -0
- package/dist/agentic/shared/scripts/framework-context.mjs +888 -0
- package/dist/agentic/shared/scripts/install-skills.mjs +1029 -0
- package/dist/agentic/shared/scripts/install-skills.sh +4 -694
- package/dist/agentic/shared/scripts/prepare-agent-harness-fixture.mjs +222 -0
- package/dist/agentic/shared/scripts/run-agent-harness-release.mjs +1774 -0
- package/dist/bin.js +13 -1
- package/dist/bin.js.map +2 -2
- package/dist/lib/__integration__/TC-INT-008.spec.js +88 -8
- package/dist/lib/__integration__/TC-INT-008.spec.js.map +2 -2
- package/dist/lib/agentic-init.js +13 -3
- package/dist/lib/agentic-init.js.map +2 -2
- package/dist/lib/agentic-setup.js +434 -186
- package/dist/lib/agentic-setup.js.map +2 -2
- package/dist/lib/generators/extensions/injection-widgets.js +20 -0
- package/dist/lib/generators/extensions/injection-widgets.js.map +2 -2
- package/dist/lib/generators/module-extension-facts.js +1840 -0
- package/dist/lib/generators/module-extension-facts.js.map +7 -0
- package/dist/lib/generators/module-fact-sources.js +70 -0
- package/dist/lib/generators/module-fact-sources.js.map +7 -0
- package/dist/lib/generators/module-facts-discovery.js +41 -7
- package/dist/lib/generators/module-facts-discovery.js.map +2 -2
- package/dist/lib/generators/module-facts.js +1387 -45
- package/dist/lib/generators/module-facts.js.map +3 -3
- package/dist/lib/generators/module-override-targets.js +688 -0
- package/dist/lib/generators/module-override-targets.js.map +7 -0
- package/dist/lib/generators/module-registry.js +11 -5
- package/dist/lib/generators/module-registry.js.map +2 -2
- package/dist/lib/load-env.js +24 -0
- package/dist/lib/load-env.js.map +7 -0
- package/dist/lib/resolver.js +2 -0
- package/dist/lib/resolver.js.map +2 -2
- package/dist/lib/telemetry-init.js +358 -0
- package/dist/lib/telemetry-init.js.map +7 -0
- package/dist/lib/testing/integration.js +5 -0
- package/dist/lib/testing/integration.js.map +2 -2
- package/dist/mercato.js +23 -1
- package/dist/mercato.js.map +2 -2
- package/package.json +13 -11
- package/src/bin.ts +33 -1
- package/src/lib/__integration__/TC-INT-008.spec.ts +107 -9
- package/src/lib/__tests__/agentic-init.test.ts +41 -5
- package/src/lib/__tests__/agentic-setup.ownership.test.ts +386 -0
- package/src/lib/__tests__/injection-widget-id-validation.test.ts +184 -0
- package/src/lib/__tests__/load-env.test.ts +65 -0
- package/src/lib/__tests__/telemetry-init.test.ts +298 -0
- package/src/lib/agentic-init.ts +15 -3
- package/src/lib/agentic-setup.ts +573 -222
- package/src/lib/generators/__tests__/activation-adapter-coverage.test.ts +92 -0
- package/src/lib/generators/__tests__/module-extension-activations.test.ts +762 -0
- package/src/lib/generators/__tests__/module-extension-facts.test.ts +404 -0
- package/src/lib/generators/__tests__/module-facts.bc-guard.test.ts +88 -2
- package/src/lib/generators/__tests__/module-facts.customers.fixture.test.ts +1 -0
- package/src/lib/generators/__tests__/module-facts.discovery.test.ts +54 -10
- package/src/lib/generators/__tests__/module-facts.extension-hosts.test.ts +108 -0
- package/src/lib/generators/__tests__/module-facts.owned-contracts.test.ts +561 -0
- package/src/lib/generators/__tests__/module-facts.source-links.test.ts +119 -0
- package/src/lib/generators/__tests__/module-override-targets.roundtrip.test.ts +149 -0
- package/src/lib/generators/__tests__/module-override-targets.test.ts +304 -0
- package/src/lib/generators/__tests__/module-subset.test.ts +18 -0
- package/src/lib/generators/extensions/injection-widgets.ts +51 -0
- package/src/lib/generators/module-extension-facts.ts +2365 -0
- package/src/lib/generators/module-fact-sources.ts +108 -0
- package/src/lib/generators/module-facts-discovery.ts +58 -7
- package/src/lib/generators/module-facts.ts +1851 -46
- package/src/lib/generators/module-override-targets.ts +863 -0
- package/src/lib/generators/module-registry.ts +16 -6
- package/src/lib/load-env.ts +30 -0
- package/src/lib/resolver.ts +3 -0
- package/src/lib/telemetry-init.ts +461 -0
- package/src/lib/testing/__tests__/integration.test.ts +4 -0
- package/src/lib/testing/integration.ts +5 -0
- package/src/mercato.ts +26 -1
- package/dist/agentic/guides/cache.md +0 -45
- package/dist/agentic/guides/core.md +0 -453
- package/dist/agentic/guides/events.md +0 -117
- package/dist/agentic/guides/queue.md +0 -53
- package/dist/agentic/guides/search.md +0 -115
- package/dist/agentic/guides/shared.md +0 -108
- package/dist/agentic/guides/ui.md +0 -347
- package/dist/agentic/shared/ai/skills/om-auto-continue-pr-loop/SKILL.md +0 -27
- package/dist/agentic/shared/ai/skills/om-auto-create-pr-loop/SKILL.md +0 -27
- package/dist/agentic/shared/ai/skills/om-backend-ui-design/references/ui-components.md +0 -260
- package/dist/agentic/shared/ai/skills/om-data-model-design/references/mikro-orm-cheatsheet.md +0 -146
- package/dist/agentic/shared/ai/skills/om-help/references/skills-catalog.md +0 -84
- package/dist/agentic/shared/ai/skills/om-help/references/workflow-sequences.md +0 -193
- package/dist/agentic/shared/ai/skills/om-integration-builder/STANDALONE.md +0 -34
- package/dist/agentic/shared/ai/skills/om-integration-builder/references/adapter-contracts.md +0 -762
- package/dist/agentic/shared/ai/skills/om-module-scaffold/references/naming-conventions.md +0 -50
- package/dist/agentic/shared/ai/skills/om-module-scaffold/references/navigation-patterns.md +0 -97
- package/dist/agentic/shared/ai/skills/om-system-extension/references/extension-contracts.md +0 -271
- package/dist/agentic/shared/ai/skills/om-troubleshooter/references/diagnostic-commands.md +0 -101
|
@@ -0,0 +1,1774 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import { spawnSync } from 'node:child_process'
|
|
4
|
+
import { createHash } from 'node:crypto'
|
|
5
|
+
import fs from 'node:fs'
|
|
6
|
+
import { createRequire } from 'node:module'
|
|
7
|
+
import os from 'node:os'
|
|
8
|
+
import path from 'node:path'
|
|
9
|
+
import process from 'node:process'
|
|
10
|
+
import { pathToFileURL } from 'node:url'
|
|
11
|
+
import { assertSandboxRuntimeAvailable, sandboxedInvocation, verifyLinuxSandboxIsolation } from './execution-sandbox.mjs'
|
|
12
|
+
|
|
13
|
+
const WRITABLE_KINDS = new Set(['implementation', 'regression'])
|
|
14
|
+
const RELEASE_SUPPORTED_RUNNERS = ['codex', 'claude']
|
|
15
|
+
const RELEASE_VALIDATION_COMMANDS = ['yarn generate', 'yarn typecheck', 'yarn lint', 'yarn build']
|
|
16
|
+
const RELEASE_VALIDATION_OUTPUT_ROOTS = new Map([
|
|
17
|
+
['yarn generate', ['.mercato/generated']],
|
|
18
|
+
['yarn typecheck', ['tsconfig.tsbuildinfo']],
|
|
19
|
+
['yarn lint', []],
|
|
20
|
+
['yarn build', ['.mercato/generated', '.mercato/next', 'next-env.d.ts', 'tsconfig.tsbuildinfo']],
|
|
21
|
+
])
|
|
22
|
+
const RUNNERS = new Set(RELEASE_SUPPORTED_RUNNERS)
|
|
23
|
+
const ALLOWED_VALIDATION_COMMANDS = new Set(RELEASE_VALIDATION_COMMANDS)
|
|
24
|
+
const GENERATED_TEST_RUNNERS = new Set(['jest', 'playwright-api', 'playwright-browser'])
|
|
25
|
+
const RESULT_LIMIT = 262_144
|
|
26
|
+
const ERROR_LIMIT = 2_000
|
|
27
|
+
const VIOLATION_LIMIT = 300
|
|
28
|
+
const COPY_EXCLUDED_PREFIXES = [
|
|
29
|
+
'.git', '.next', '.turbo', '.cache', 'build', 'coverage', 'dist', 'node_modules', 'out',
|
|
30
|
+
'.ai/framework-context', '.ai/harness/results', '.ai/reports',
|
|
31
|
+
'.mercato/generated', '.mercato/next', 'next-env.d.ts', 'tsconfig.tsbuildinfo',
|
|
32
|
+
]
|
|
33
|
+
const SAFE_ENV_TEMPLATES = new Set(['.env.example', '.env.sample', '.env.template'])
|
|
34
|
+
const SENSITIVE_AUTH_FILES = new Set(['.git-credentials', '.netrc', '.npmrc', '.pypirc', '.yarnrc', '.yarnrc.yml', '.yarnrc.yaml'])
|
|
35
|
+
const SENSITIVE_AUTH_DATA_FILE = /^(?:auth|credentials?|secrets?|tokens?)(?:\.(?:conf(?:ig)?|ini|json|toml|txt|ya?ml))?$/
|
|
36
|
+
const SENSITIVE_ENV_KEY = /(?:^|_)(?:api_?key|auth|credential|credentials|password|passwd|private_?key|secret|token)(?:_|$)/i
|
|
37
|
+
const GENERATED_YARN_CONFIG_PATH = '.yarnrc.yml'
|
|
38
|
+
const GENERATED_YARN_CONFIG_LIMIT = 16_384
|
|
39
|
+
|
|
40
|
+
function usage() {
|
|
41
|
+
return `Run the complete standalone agent-harness release gate.
|
|
42
|
+
|
|
43
|
+
Usage:
|
|
44
|
+
yarn harness:release --runner <codex|claude> --prepare-targets /absolute/empty-directory --acknowledge-writes
|
|
45
|
+
yarn harness:release --runner <codex|claude> --writable-targets /absolute/release-targets.json --acknowledge-writes
|
|
46
|
+
|
|
47
|
+
Options:
|
|
48
|
+
--root <absolute-app> Controller/generated app root (default: current directory)
|
|
49
|
+
--runner <codex|claude> Required primary runner for every blocking live lane
|
|
50
|
+
--portability-runner <runner> Optional different runner for the 46-case read-only portability lane
|
|
51
|
+
--prepare-targets <absolute> Clone this fresh scaffold once per writable case under an empty/new directory
|
|
52
|
+
--writable-targets <absolute> JSON map of every writable case to a fresh disposable app
|
|
53
|
+
--case-timeout <ms> Per-model invocation timeout floor (default: 120000; a writable case may raise it)
|
|
54
|
+
--validation-timeout <ms> Timeout for each yarn validation (default: 1800000)
|
|
55
|
+
--acknowledge-writes Required: fixture preparation and validation commands write files
|
|
56
|
+
--help Show this help
|
|
57
|
+
|
|
58
|
+
Exactly one target option is required. Automatic targets require a fresh controller
|
|
59
|
+
without local environment, credential, or private-key files; safe .env.example/sample/template
|
|
60
|
+
files are retained. Targets exclude dependencies, build outputs, harness results, and
|
|
61
|
+
generated framework context, then safely reference the controller's protected dependency
|
|
62
|
+
tree. Externally supplied targets must use the same
|
|
63
|
+
node_modules link to that protected tree; nested dependency copies are rejected.
|
|
64
|
+
The complete release requires trusted Bubblewrap plus working user/network namespaces on Linux;
|
|
65
|
+
native macOS, Windows, untrusted runtimes, and unavailable isolation fail closed. The command preflights complete matrix,
|
|
66
|
+
fixture, judge, and target coverage before
|
|
67
|
+
running deterministic validation, selected primary routing, optional portability routing, writable trusted-oracle
|
|
68
|
+
gates, explicit om-judge-agent-session with om-code-review, and the release-matrix validation commands. It stores
|
|
69
|
+
only a sanitized aggregate report under .ai/harness/results/.`
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function parseArgs(argv) {
|
|
73
|
+
const options = {
|
|
74
|
+
root: process.cwd(),
|
|
75
|
+
runner: undefined,
|
|
76
|
+
portabilityRunner: undefined,
|
|
77
|
+
prepareTargets: undefined,
|
|
78
|
+
writableTargets: undefined,
|
|
79
|
+
caseTimeout: 120_000,
|
|
80
|
+
validationTimeout: 1_800_000,
|
|
81
|
+
acknowledgeWrites: false,
|
|
82
|
+
help: false,
|
|
83
|
+
}
|
|
84
|
+
for (let index = 0; index < argv.length; index += 1) {
|
|
85
|
+
const arg = argv[index]
|
|
86
|
+
const value = () => {
|
|
87
|
+
index += 1
|
|
88
|
+
if (index >= argv.length) throw new Error(`${arg} requires a value`)
|
|
89
|
+
return argv[index]
|
|
90
|
+
}
|
|
91
|
+
if (arg === '--help' || arg === '-h') options.help = true
|
|
92
|
+
else if (arg === '--root') options.root = value()
|
|
93
|
+
else if (arg === '--runner') options.runner = value()
|
|
94
|
+
else if (arg === '--portability-runner') options.portabilityRunner = value()
|
|
95
|
+
else if (arg === '--prepare-targets') options.prepareTargets = value()
|
|
96
|
+
else if (arg === '--writable-targets') options.writableTargets = value()
|
|
97
|
+
else if (arg === '--case-timeout') options.caseTimeout = Number(value())
|
|
98
|
+
else if (arg === '--validation-timeout') options.validationTimeout = Number(value())
|
|
99
|
+
else if (arg === '--acknowledge-writes') options.acknowledgeWrites = true
|
|
100
|
+
else throw new Error(`unknown argument: ${arg}`)
|
|
101
|
+
}
|
|
102
|
+
if (options.help) return options
|
|
103
|
+
if (!path.isAbsolute(options.root)) throw new Error('--root must be an absolute path')
|
|
104
|
+
if (!RUNNERS.has(options.runner)) throw new Error('--runner must be codex or claude')
|
|
105
|
+
if (options.portabilityRunner !== undefined && !RUNNERS.has(options.portabilityRunner)) throw new Error('--portability-runner must be codex or claude')
|
|
106
|
+
if (options.portabilityRunner === options.runner) throw new Error('--portability-runner must differ from --runner')
|
|
107
|
+
if (Boolean(options.prepareTargets) === Boolean(options.writableTargets)) throw new Error('pass exactly one of --prepare-targets or --writable-targets')
|
|
108
|
+
if (options.prepareTargets && !path.isAbsolute(options.prepareTargets)) throw new Error('--prepare-targets must be an absolute path')
|
|
109
|
+
if (options.writableTargets && !path.isAbsolute(options.writableTargets)) throw new Error('--writable-targets must be an absolute path')
|
|
110
|
+
if (!options.acknowledgeWrites) throw new Error('the full release gate requires --acknowledge-writes')
|
|
111
|
+
if (!Number.isInteger(options.caseTimeout) || options.caseTimeout < 1_000 || options.caseTimeout > 3_600_000) throw new Error('--case-timeout must be from 1000 to 3600000 milliseconds')
|
|
112
|
+
if (!Number.isInteger(options.validationTimeout) || options.validationTimeout < 1_000 || options.validationTimeout > 7_200_000) throw new Error('--validation-timeout must be from 1000 to 7200000 milliseconds')
|
|
113
|
+
return options
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export function effectiveCaseTimeout(cases, caseId, fallback) {
|
|
117
|
+
const declared = cases.find((item) => item.id === caseId)?.timeoutMs
|
|
118
|
+
return Math.max(fallback, Number.isInteger(declared) ? declared : 0)
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function readJson(file) {
|
|
122
|
+
return JSON.parse(fs.readFileSync(file, 'utf8'))
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function sha256(value) {
|
|
126
|
+
return createHash('sha256').update(value).digest('hex')
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function isPathInside(parent, child) {
|
|
130
|
+
const relative = path.relative(parent, child)
|
|
131
|
+
return relative === '' || (relative !== '..' && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative))
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function validModelSelector(value) {
|
|
135
|
+
return typeof value === 'string' && /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,99}$/.test(value)
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function normalizedRelative(root, absolute) {
|
|
139
|
+
return path.relative(root, absolute).replaceAll(path.sep, '/')
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function copyPathExcluded(relative) {
|
|
143
|
+
return COPY_EXCLUDED_PREFIXES.some((prefix) => relative === prefix || relative.startsWith(`${prefix}/`))
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function sensitiveScaffoldPath(relative, stat) {
|
|
147
|
+
const basename = path.posix.basename(relative).toLowerCase()
|
|
148
|
+
const lowerRelative = relative.toLowerCase()
|
|
149
|
+
if ((basename === '.env' || basename.startsWith('.env.')) && !SAFE_ENV_TEMPLATES.has(basename)) return true
|
|
150
|
+
// A fresh create-mercato-app scaffold always contains this generated root file.
|
|
151
|
+
// Its contents are checked separately against the exact credential-free shape.
|
|
152
|
+
if (lowerRelative === GENERATED_YARN_CONFIG_PATH) return relative !== GENERATED_YARN_CONFIG_PATH
|
|
153
|
+
if (SENSITIVE_AUTH_FILES.has(basename)) return true
|
|
154
|
+
if (/^(?:\.aws|\.azure|\.config\/gcloud|\.config\/gh|\.docker|\.gnupg|\.kube|\.oci|\.pulumi|\.ssh|\.terraform\.d)(?:\/|$)/.test(lowerRelative)) return true
|
|
155
|
+
if (stat?.isDirectory()) return false
|
|
156
|
+
if (SENSITIVE_AUTH_DATA_FILE.test(basename)) return true
|
|
157
|
+
if (/\.(?:key|pem|p12|pfx|jks|keystore)$/.test(basename)) return true
|
|
158
|
+
if (/^(?:id_rsa|id_dsa|id_ecdsa|id_ed25519)$/.test(basename)) return true
|
|
159
|
+
return /(?:^|[._-])(?:credential|credentials|private[._-]?key|service[._-]?account)(?:[._-]|$)/.test(basename)
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
function parseGeneratedQuotedValue(line, prefix) {
|
|
163
|
+
if (!line.startsWith(prefix)) return undefined
|
|
164
|
+
try {
|
|
165
|
+
const serialized = line.slice(prefix.length)
|
|
166
|
+
const value = JSON.parse(serialized)
|
|
167
|
+
return typeof value === 'string' && value.length > 0 && JSON.stringify(value) === serialized ? value : undefined
|
|
168
|
+
} catch {
|
|
169
|
+
return undefined
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function validGeneratedYarnConfig(contents) {
|
|
174
|
+
if (Buffer.byteLength(contents, 'utf8') > GENERATED_YARN_CONFIG_LIMIT || contents.includes('\0')) return false
|
|
175
|
+
const lines = contents.replaceAll('\r\n', '\n').split('\n').filter((line) => line !== '')
|
|
176
|
+
const migrated = lines[0] === 'approvedGitRepositories:'
|
|
177
|
+
if (migrated) {
|
|
178
|
+
if (
|
|
179
|
+
lines.shift() !== 'approvedGitRepositories:'
|
|
180
|
+
|| lines.shift() !== ' - "**"'
|
|
181
|
+
|| lines.shift() !== 'enableScripts: true'
|
|
182
|
+
) return false
|
|
183
|
+
}
|
|
184
|
+
if (lines.shift() !== 'nodeLinker: node-modules') return false
|
|
185
|
+
if (migrated && lines.shift() !== 'npmMinimalAgeGate: 0') return false
|
|
186
|
+
if (!lines.length) return true
|
|
187
|
+
|
|
188
|
+
const unsafeHttpHosts = []
|
|
189
|
+
const readUnsafeHttpHosts = () => {
|
|
190
|
+
if (lines[0] !== 'unsafeHttpWhitelist:') return true
|
|
191
|
+
lines.shift()
|
|
192
|
+
while (lines[0]?.startsWith(' - ')) {
|
|
193
|
+
const host = parseGeneratedQuotedValue(lines.shift(), ' - ')
|
|
194
|
+
if (!host) return false
|
|
195
|
+
unsafeHttpHosts.push(host)
|
|
196
|
+
}
|
|
197
|
+
return unsafeHttpHosts.length > 0
|
|
198
|
+
}
|
|
199
|
+
if (!migrated && !readUnsafeHttpHosts()) return false
|
|
200
|
+
|
|
201
|
+
if (lines.shift() !== 'npmScopes:' || lines.shift() !== ' open-mercato:') return false
|
|
202
|
+
if (migrated && lines.shift() !== ' npmMinimalAgeGate: 0') return false
|
|
203
|
+
const registryUrl = parseGeneratedQuotedValue(lines.shift() ?? '', ' npmRegistryServer: ')
|
|
204
|
+
if (!registryUrl || (!migrated && lines.shift() !== ' npmMinimalAgeGate: 0')) return false
|
|
205
|
+
if (migrated && !readUnsafeHttpHosts()) return false
|
|
206
|
+
if (lines.length) return false
|
|
207
|
+
|
|
208
|
+
let parsedRegistryUrl
|
|
209
|
+
try {
|
|
210
|
+
parsedRegistryUrl = new URL(registryUrl)
|
|
211
|
+
} catch {
|
|
212
|
+
return false
|
|
213
|
+
}
|
|
214
|
+
if (!['http:', 'https:'].includes(parsedRegistryUrl.protocol)) return false
|
|
215
|
+
if (parsedRegistryUrl.username || parsedRegistryUrl.password || parsedRegistryUrl.search || parsedRegistryUrl.hash) return false
|
|
216
|
+
|
|
217
|
+
const expectedUnsafeHosts = parsedRegistryUrl.protocol === 'http:' ? [parsedRegistryUrl.hostname] : []
|
|
218
|
+
if (parsedRegistryUrl.protocol === 'http:' && ['localhost', '127.0.0.1'].includes(parsedRegistryUrl.hostname)) {
|
|
219
|
+
expectedUnsafeHosts.push('host.docker.internal')
|
|
220
|
+
}
|
|
221
|
+
return JSON.stringify(unsafeHttpHosts) === JSON.stringify(expectedUnsafeHosts)
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
function validateGeneratedYarnConfig(source, relative, stat) {
|
|
225
|
+
if (relative !== GENERATED_YARN_CONFIG_PATH) return
|
|
226
|
+
if (!stat.isFile() || stat.isSymbolicLink() || !validGeneratedYarnConfig(fs.readFileSync(source, 'utf8'))) {
|
|
227
|
+
throw new Error('fresh controller contains an unsafe Yarn registry configuration; use a credential-free fresh scaffold')
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
function validateScaffoldCopySource(root) {
|
|
232
|
+
const realRoot = fs.realpathSync(root)
|
|
233
|
+
const visit = (directory) => {
|
|
234
|
+
for (const name of fs.readdirSync(directory).sort()) {
|
|
235
|
+
const source = path.join(directory, name)
|
|
236
|
+
const relative = normalizedRelative(realRoot, source)
|
|
237
|
+
if (copyPathExcluded(relative)) continue
|
|
238
|
+
const stat = fs.lstatSync(source)
|
|
239
|
+
if (sensitiveScaffoldPath(relative, stat)) {
|
|
240
|
+
throw new Error('fresh controller contains a local environment, credential, or private-key file; use a sanitized fresh scaffold')
|
|
241
|
+
}
|
|
242
|
+
validateGeneratedYarnConfig(source, relative, stat)
|
|
243
|
+
if (stat.isSymbolicLink()) {
|
|
244
|
+
const resolved = fs.realpathSync(source)
|
|
245
|
+
const resolvedRelative = normalizedRelative(realRoot, resolved)
|
|
246
|
+
if (!isPathInside(realRoot, resolved) || copyPathExcluded(resolvedRelative)) {
|
|
247
|
+
throw new Error(`fresh scaffold contains an unsafe copied symlink: ${relative}`)
|
|
248
|
+
}
|
|
249
|
+
} else if (stat.isDirectory()) visit(source)
|
|
250
|
+
else if (!stat.isFile()) throw new Error(`fresh scaffold contains an unsupported entry: ${relative}`)
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
visit(realRoot)
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
function copyFreshScaffold(sourceRoot, targetRoot) {
|
|
257
|
+
const realSourceRoot = fs.realpathSync(sourceRoot)
|
|
258
|
+
const visit = (sourceDirectory, targetDirectory) => {
|
|
259
|
+
fs.mkdirSync(targetDirectory, { recursive: true, mode: 0o755 })
|
|
260
|
+
for (const name of fs.readdirSync(sourceDirectory).sort()) {
|
|
261
|
+
const source = path.join(sourceDirectory, name)
|
|
262
|
+
const relative = normalizedRelative(realSourceRoot, source)
|
|
263
|
+
if (copyPathExcluded(relative)) continue
|
|
264
|
+
const stat = fs.lstatSync(source)
|
|
265
|
+
// Fail closed if a sensitive file appears between validation and copy.
|
|
266
|
+
if (sensitiveScaffoldPath(relative, stat)) {
|
|
267
|
+
throw new Error('fresh controller contains a local environment, credential, or private-key file; use a sanitized fresh scaffold')
|
|
268
|
+
}
|
|
269
|
+
const destination = path.join(targetDirectory, name)
|
|
270
|
+
validateGeneratedYarnConfig(source, relative, stat)
|
|
271
|
+
if (stat.isDirectory()) visit(source, destination)
|
|
272
|
+
else if (stat.isFile()) {
|
|
273
|
+
fs.copyFileSync(source, destination, fs.constants.COPYFILE_FICLONE)
|
|
274
|
+
fs.chmodSync(destination, stat.mode & 0o777)
|
|
275
|
+
} else if (stat.isSymbolicLink()) {
|
|
276
|
+
const resolved = fs.realpathSync(source)
|
|
277
|
+
const resolvedRelative = normalizedRelative(realSourceRoot, resolved)
|
|
278
|
+
const target = path.join(targetRoot, resolvedRelative)
|
|
279
|
+
const type = fs.statSync(resolved).isDirectory() ? (process.platform === 'win32' ? 'junction' : 'dir') : 'file'
|
|
280
|
+
fs.symlinkSync(path.relative(path.dirname(destination), target), destination, type)
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
visit(realSourceRoot, targetRoot)
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
export function resolvePreparationRoot(requested, controllerRoot) {
|
|
288
|
+
if (!path.isAbsolute(requested)) throw new Error('--prepare-targets must be an absolute path')
|
|
289
|
+
const resolved = path.resolve(requested)
|
|
290
|
+
if (resolved === path.parse(resolved).root) throw new Error('--prepare-targets cannot be a filesystem root')
|
|
291
|
+
let actual
|
|
292
|
+
if (fs.existsSync(resolved)) {
|
|
293
|
+
const stat = fs.lstatSync(resolved)
|
|
294
|
+
if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error('--prepare-targets must be a regular directory')
|
|
295
|
+
if (fs.readdirSync(resolved).length) throw new Error('--prepare-targets must be empty')
|
|
296
|
+
actual = fs.realpathSync(resolved)
|
|
297
|
+
} else {
|
|
298
|
+
const parent = path.dirname(resolved)
|
|
299
|
+
if (!fs.existsSync(parent)) throw new Error('--prepare-targets parent directory must already exist')
|
|
300
|
+
const stat = fs.lstatSync(parent)
|
|
301
|
+
if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error('--prepare-targets parent must be a regular directory')
|
|
302
|
+
actual = path.join(fs.realpathSync(parent), path.basename(resolved))
|
|
303
|
+
}
|
|
304
|
+
const controller = fs.realpathSync(controllerRoot)
|
|
305
|
+
if (isPathInside(controller, actual) || isPathInside(actual, controller)) {
|
|
306
|
+
throw new Error('--prepare-targets must be separate from the controller app')
|
|
307
|
+
}
|
|
308
|
+
return actual
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
export function prepareWritableTargets({ root, prepareRoot, caseIds }) {
|
|
312
|
+
const controller = fs.realpathSync(root)
|
|
313
|
+
if (fs.existsSync(path.join(controller, '.ai', 'harness', 'DISPOSABLE'))) {
|
|
314
|
+
throw new Error('automatic target preparation requires an unmodified fresh controller scaffold')
|
|
315
|
+
}
|
|
316
|
+
const targetRoot = resolvePreparationRoot(prepareRoot, controller)
|
|
317
|
+
if (!Array.isArray(caseIds) || new Set(caseIds).size !== caseIds.length || caseIds.some((id) => !/^OMH-[0-9]{3}$/.test(id))) {
|
|
318
|
+
throw new Error('automatic writable target case IDs are invalid or duplicated')
|
|
319
|
+
}
|
|
320
|
+
const dependencyRoot = path.join(controller, 'node_modules')
|
|
321
|
+
let dependencyStat
|
|
322
|
+
try { dependencyStat = fs.lstatSync(dependencyRoot) } catch { throw new Error('controller node_modules is unavailable; run yarn install') }
|
|
323
|
+
if (!dependencyStat.isDirectory() || dependencyStat.isSymbolicLink()) throw new Error('controller node_modules must be a regular installed dependency directory')
|
|
324
|
+
validateScaffoldCopySource(controller)
|
|
325
|
+
if (!fs.existsSync(targetRoot)) fs.mkdirSync(targetRoot, { mode: 0o700 })
|
|
326
|
+
const targetRootStat = fs.lstatSync(targetRoot)
|
|
327
|
+
if (!targetRootStat.isDirectory() || targetRootStat.isSymbolicLink() || fs.readdirSync(targetRoot).length) {
|
|
328
|
+
throw new Error('automatic writable target root stopped being an empty regular directory')
|
|
329
|
+
}
|
|
330
|
+
const targets = {}
|
|
331
|
+
for (const caseId of caseIds) {
|
|
332
|
+
const target = path.join(targetRoot, caseId)
|
|
333
|
+
fs.mkdirSync(target, { mode: 0o700 })
|
|
334
|
+
copyFreshScaffold(controller, target)
|
|
335
|
+
fs.symlinkSync(dependencyRoot, path.join(target, 'node_modules'), process.platform === 'win32' ? 'junction' : 'dir')
|
|
336
|
+
targets[caseId] = fs.realpathSync(target)
|
|
337
|
+
}
|
|
338
|
+
const manifest = { schemaVersion: 1, targets }
|
|
339
|
+
fs.writeFileSync(path.join(targetRoot, 'release-targets.json'), `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600 })
|
|
340
|
+
return manifest
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
function sortedUnique(values) {
|
|
344
|
+
return [...new Set(values)].sort()
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
function difference(left, right) {
|
|
348
|
+
const rightSet = new Set(right)
|
|
349
|
+
return sortedUnique(left.filter((value) => !rightSet.has(value)))
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
function duplicates(values) {
|
|
353
|
+
const seen = new Set()
|
|
354
|
+
const repeated = new Set()
|
|
355
|
+
for (const value of values) {
|
|
356
|
+
if (seen.has(value)) repeated.add(value)
|
|
357
|
+
seen.add(value)
|
|
358
|
+
}
|
|
359
|
+
return [...repeated].sort()
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
function resolveCaseSelector(selector, allCaseIds) {
|
|
363
|
+
if (selector === 'all') return [...allCaseIds]
|
|
364
|
+
return Array.isArray(selector) ? [...selector] : []
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
function addCoverageViolation(violations, label, ids) {
|
|
368
|
+
if (ids.length) violations.push(`${label}: ${ids.join(', ')}`)
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
function validTargetsManifest(targetsManifest) {
|
|
372
|
+
return targetsManifest?.schemaVersion === 1 && targetsManifest?.targets && typeof targetsManifest.targets === 'object' && !Array.isArray(targetsManifest.targets)
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
export function generatedTestExecutionContract(entry) {
|
|
376
|
+
const cliPackage = entry.runner === 'jest' ? 'jest' : '@playwright/test'
|
|
377
|
+
const argv = entry.runner === 'jest'
|
|
378
|
+
? ['<resolved-cli>', '--config', 'jest.config.cjs', '--runInBand', '--runTestsByPath', entry.artifact, '--cacheDirectory', '<isolated-temp>/jest-cache', '--json', '--outputFile', '<isolated-temp>/jest-results.json']
|
|
379
|
+
: ['<resolved-cli>', 'test', '--config', '.ai/qa/tests/playwright.config.ts', entry.artifact, '--retries=0', '--workers=1', '--forbid-only', '--reporter=json', '--output', '<isolated-temp>/playwright-output', '--update-snapshots=none']
|
|
380
|
+
return { executor: 'node', cliPackage, argv, command: ['node', ...argv].join(' ') }
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
function readGeneratedTestReport(file, tempRoot, label) {
|
|
384
|
+
const stat = fs.lstatSync(file)
|
|
385
|
+
if (!stat.isFile() || stat.isSymbolicLink() || stat.size < 2 || stat.size > 8 * 1024 * 1024
|
|
386
|
+
|| !isPathInside(fs.realpathSync(tempRoot), fs.realpathSync(file))) {
|
|
387
|
+
throw new Error(`${label} report must be a bounded regular file in the isolated output root`)
|
|
388
|
+
}
|
|
389
|
+
return readJson(file)
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
function collectPlaywrightTests(suites, output = []) {
|
|
393
|
+
for (const suite of Array.isArray(suites) ? suites : []) {
|
|
394
|
+
for (const spec of Array.isArray(suite?.specs) ? suite.specs : []) {
|
|
395
|
+
for (const test of Array.isArray(spec?.tests) ? spec.tests : []) output.push(test)
|
|
396
|
+
}
|
|
397
|
+
collectPlaywrightTests(suite?.suites, output)
|
|
398
|
+
}
|
|
399
|
+
return output
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
function generatedTestCounts(testRunner, tempRoot) {
|
|
403
|
+
if (testRunner === 'jest') {
|
|
404
|
+
const report = readGeneratedTestReport(path.join(tempRoot, 'jest-results.json'), tempRoot, 'Jest')
|
|
405
|
+
const counts = {
|
|
406
|
+
passedTests: Number(report.numPassedTests ?? 0),
|
|
407
|
+
skippedTests: Number(report.numPendingTests ?? 0),
|
|
408
|
+
todoTests: Number(report.numTodoTests ?? 0),
|
|
409
|
+
expectedFailureTests: 0,
|
|
410
|
+
}
|
|
411
|
+
const assertions = (Array.isArray(report.testResults) ? report.testResults : [])
|
|
412
|
+
.flatMap((suite) => Array.isArray(suite?.assertionResults) ? suite.assertionResults : [])
|
|
413
|
+
if (report.success !== true || Number(report.numFailedTests ?? 0) !== 0 || counts.passedTests < 1
|
|
414
|
+
|| counts.skippedTests !== 0 || counts.todoTests !== 0
|
|
415
|
+
|| assertions.some((assertion) => assertion?.status !== 'passed')) {
|
|
416
|
+
throw new Error('generated Jest report must contain at least one passing test and zero failed, skipped, todo, or expected-failure tests')
|
|
417
|
+
}
|
|
418
|
+
return counts
|
|
419
|
+
}
|
|
420
|
+
const report = readGeneratedTestReport(path.join(tempRoot, 'playwright-results.json'), tempRoot, 'Playwright')
|
|
421
|
+
const tests = collectPlaywrightTests(report.suites)
|
|
422
|
+
const skippedTests = tests.filter((test) => test?.expectedStatus === 'skipped' || test?.status === 'skipped').length
|
|
423
|
+
const expectedFailureTests = tests.filter((test) => !['passed', 'skipped'].includes(test?.expectedStatus)).length
|
|
424
|
+
const passedTests = tests.filter((test) => test?.expectedStatus === 'passed' && test?.status === 'expected'
|
|
425
|
+
&& Array.isArray(test.results) && test.results.length > 0 && test.results.every((result) => result?.status === 'passed')).length
|
|
426
|
+
const counts = { passedTests, skippedTests, todoTests: 0, expectedFailureTests }
|
|
427
|
+
if (tests.length < 1 || passedTests !== tests.length || skippedTests !== 0 || expectedFailureTests !== 0
|
|
428
|
+
|| Number(report?.stats?.skipped ?? 0) !== 0 || Number(report?.stats?.unexpected ?? 0) !== 0
|
|
429
|
+
|| Number(report?.stats?.flaky ?? 0) !== 0 || (Array.isArray(report?.errors) && report.errors.length > 0)) {
|
|
430
|
+
throw new Error('generated Playwright report must contain at least one passing test and zero skipped, focused, flaky, unexpected, or expected-failure tests')
|
|
431
|
+
}
|
|
432
|
+
return counts
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
function resolveTargetTestCli(target, runner) {
|
|
436
|
+
const dependencyRoot = fs.realpathSync(path.join(target, 'node_modules'))
|
|
437
|
+
const requireFromTarget = createRequire(path.join(target, 'package.json'))
|
|
438
|
+
const cli = runner === 'jest'
|
|
439
|
+
? path.join(path.dirname(requireFromTarget.resolve('jest/package.json')), 'bin', 'jest.js')
|
|
440
|
+
: requireFromTarget.resolve('@playwright/test/cli')
|
|
441
|
+
const realCli = fs.realpathSync(cli)
|
|
442
|
+
const stat = fs.lstatSync(cli)
|
|
443
|
+
if (!isPathInside(dependencyRoot, realCli) || !stat.isFile() || stat.isSymbolicLink()) {
|
|
444
|
+
throw new Error(`${runner} CLI must be a regular file in the protected target dependency tree`)
|
|
445
|
+
}
|
|
446
|
+
return realCli
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
function resolveBrowserRuntime(target, isolatedCache) {
|
|
450
|
+
const dependencyRoot = fs.realpathSync(path.join(target, 'node_modules'))
|
|
451
|
+
const requireFromTarget = createRequire(path.join(target, 'package.json'))
|
|
452
|
+
const playwrightCoreEntry = fs.realpathSync(requireFromTarget.resolve('playwright-core'))
|
|
453
|
+
if (!isPathInside(dependencyRoot, playwrightCoreEntry)) throw new Error('playwright-core must resolve from the protected target dependency tree')
|
|
454
|
+
let playwrightCoreRoot = path.dirname(playwrightCoreEntry)
|
|
455
|
+
while (path.dirname(playwrightCoreRoot) !== playwrightCoreRoot && path.basename(playwrightCoreRoot) !== 'playwright-core') playwrightCoreRoot = path.dirname(playwrightCoreRoot)
|
|
456
|
+
const browsersFile = path.join(playwrightCoreRoot, 'browsers.json')
|
|
457
|
+
const browsersStat = fs.lstatSync(browsersFile)
|
|
458
|
+
if (!browsersStat.isFile() || browsersStat.isSymbolicLink() || browsersStat.size > 262_144) throw new Error('playwright-core browsers.json is unsafe')
|
|
459
|
+
const browserDeclaration = readJson(browsersFile)?.browsers?.find((entry) => entry.name === 'chromium-headless-shell')
|
|
460
|
+
if (!/^[0-9]+$/.test(browserDeclaration?.revision ?? '')) throw new Error('Playwright does not declare a bounded Chromium headless-shell revision')
|
|
461
|
+
const chromiumExecutable = requireFromTarget('@playwright/test').chromium.executablePath()
|
|
462
|
+
let chromiumRoot = path.dirname(fs.realpathSync(chromiumExecutable))
|
|
463
|
+
while (path.dirname(chromiumRoot) !== chromiumRoot && !/^chromium-[0-9]+$/.test(path.basename(chromiumRoot))) chromiumRoot = path.dirname(chromiumRoot)
|
|
464
|
+
if (!/^chromium-[0-9]+$/.test(path.basename(chromiumRoot))) throw new Error('the Playwright browser cache root could not be bounded')
|
|
465
|
+
const browserRoot = path.join(path.dirname(chromiumRoot), `chromium_headless_shell-${browserDeclaration.revision}`)
|
|
466
|
+
const browserStat = fs.lstatSync(browserRoot)
|
|
467
|
+
if (!browserStat.isDirectory() || browserStat.isSymbolicLink()) throw new Error('the exact Playwright Chromium headless-shell runtime is unavailable')
|
|
468
|
+
const marker = path.join(browserRoot, 'INSTALLATION_COMPLETE')
|
|
469
|
+
if (!fs.lstatSync(marker).isFile()) throw new Error('the Playwright Chromium headless-shell runtime is incomplete')
|
|
470
|
+
const executableNames = new Set(process.platform === 'win32' ? ['headless_shell.exe'] : ['chrome-headless-shell', 'headless_shell'])
|
|
471
|
+
const executables = []
|
|
472
|
+
const visit = (directory) => {
|
|
473
|
+
for (const name of fs.readdirSync(directory)) {
|
|
474
|
+
const child = path.join(directory, name)
|
|
475
|
+
const entry = fs.lstatSync(child)
|
|
476
|
+
if (entry.isSymbolicLink() || (!entry.isDirectory() && !entry.isFile())) throw new Error('the Playwright Chromium headless-shell runtime contains an unsafe entry')
|
|
477
|
+
if (entry.isDirectory()) visit(child)
|
|
478
|
+
else if (executableNames.has(name)) executables.push(child)
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
visit(browserRoot)
|
|
482
|
+
if (executables.length !== 1) throw new Error('the Playwright Chromium headless-shell executable could not be identified exactly')
|
|
483
|
+
const cacheLink = path.join(isolatedCache, path.basename(browserRoot))
|
|
484
|
+
fs.symlinkSync(browserRoot, cacheLink, process.platform === 'win32' ? 'junction' : 'dir')
|
|
485
|
+
return { browserRoot: fs.realpathSync(browserRoot), isolatedCache, executable: fs.realpathSync(executables[0]) }
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
export function buildReleasePlan({ cases, registry, releaseMatrix, fixtures, seeds, targetsManifest, root, primaryRunner, portabilityRunner, targetsMayNotExist = false }) {
|
|
489
|
+
const allCaseIds = cases.map((item) => item.id)
|
|
490
|
+
const writableCases = cases.filter((item) => WRITABLE_KINDS.has(item.evaluationKind))
|
|
491
|
+
const writableCaseIds = writableCases.map((item) => item.id)
|
|
492
|
+
const reviewEligibleCaseIds = [...writableCaseIds]
|
|
493
|
+
const violations = []
|
|
494
|
+
const release = releaseMatrix?.releaseSuite ?? {}
|
|
495
|
+
const deterministicIds = resolveCaseSelector(releaseMatrix?.deterministic?.caseIds, allCaseIds)
|
|
496
|
+
const deterministicMissingCaseIds = difference(allCaseIds, deterministicIds)
|
|
497
|
+
const deterministicUnexpectedCaseIds = difference(deterministicIds, allCaseIds)
|
|
498
|
+
addCoverageViolation(violations, 'deterministic matrix is missing cases', deterministicMissingCaseIds)
|
|
499
|
+
addCoverageViolation(violations, 'deterministic matrix contains unknown cases', deterministicUnexpectedCaseIds)
|
|
500
|
+
if (releaseMatrix?.deterministic?.caseIds !== 'all') violations.push('deterministic matrix must use the exact all-cases selector')
|
|
501
|
+
|
|
502
|
+
const supportedRunners = Array.isArray(release.supportedRunners) ? release.supportedRunners : []
|
|
503
|
+
if (JSON.stringify(supportedRunners) !== JSON.stringify(RELEASE_SUPPORTED_RUNNERS)) {
|
|
504
|
+
violations.push(`release supported runners must be exactly: ${RELEASE_SUPPORTED_RUNNERS.join(', ')}`)
|
|
505
|
+
}
|
|
506
|
+
if (!RUNNERS.has(primaryRunner)) violations.push('release requires one valid primary runner')
|
|
507
|
+
if (portabilityRunner !== undefined && !RUNNERS.has(portabilityRunner)) violations.push('release portability runner is invalid')
|
|
508
|
+
if (portabilityRunner !== undefined && portabilityRunner === primaryRunner) violations.push('release portability runner must differ from the primary runner')
|
|
509
|
+
const configuredRunnerNames = Object.keys(releaseMatrix?.routing?.runners ?? {})
|
|
510
|
+
if (JSON.stringify(configuredRunnerNames) !== JSON.stringify(RELEASE_SUPPORTED_RUNNERS)) {
|
|
511
|
+
violations.push(`routing matrix must configure exactly: ${RELEASE_SUPPORTED_RUNNERS.join(', ')}`)
|
|
512
|
+
}
|
|
513
|
+
for (const runner of RELEASE_SUPPORTED_RUNNERS) {
|
|
514
|
+
if (!validModelSelector(releaseMatrix?.routing?.runners?.[runner]?.modelSelector)) violations.push(`routing matrix lacks a valid bounded ${runner} model selector`)
|
|
515
|
+
}
|
|
516
|
+
const requiredCaseIds = resolveCaseSelector(releaseMatrix?.routing?.required?.caseIds, allCaseIds)
|
|
517
|
+
const requiredUnknownCaseIds = difference(requiredCaseIds, allCaseIds)
|
|
518
|
+
addCoverageViolation(violations, 'primary routing matrix contains unknown cases', requiredUnknownCaseIds)
|
|
519
|
+
if (releaseMatrix?.routing?.required?.caseIds !== 'all') violations.push('primary routing matrix must use the exact all-cases selector')
|
|
520
|
+
const portabilityCaseIds = resolveCaseSelector(releaseMatrix?.routing?.portability?.caseIds, allCaseIds)
|
|
521
|
+
const portabilityUnknownCaseIds = difference(portabilityCaseIds, allCaseIds)
|
|
522
|
+
addCoverageViolation(violations, 'portability routing matrix contains unknown cases', portabilityUnknownCaseIds)
|
|
523
|
+
if (JSON.stringify(portabilityCaseIds) !== JSON.stringify(writableCaseIds)) {
|
|
524
|
+
violations.push('portability routing matrix must match the exact writable case set in catalog order')
|
|
525
|
+
}
|
|
526
|
+
const routing = []
|
|
527
|
+
if (RUNNERS.has(primaryRunner)) routing.push({ lane: 'primary', runner: primaryRunner, expectedCaseIds: requiredCaseIds, unknownCaseIds: requiredUnknownCaseIds })
|
|
528
|
+
if (RUNNERS.has(portabilityRunner) && portabilityRunner !== primaryRunner) {
|
|
529
|
+
routing.push({ lane: 'portability', runner: portabilityRunner, expectedCaseIds: portabilityCaseIds, unknownCaseIds: portabilityUnknownCaseIds })
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
const writableEntries = Array.isArray(releaseMatrix?.writable) ? releaseMatrix.writable : []
|
|
533
|
+
const assignedWritableIds = writableEntries.map((entry) => entry?.caseId)
|
|
534
|
+
const missingWritableMatrixCaseIds = difference(writableCaseIds, assignedWritableIds)
|
|
535
|
+
const unexpectedWritableMatrixCaseIds = difference(assignedWritableIds, writableCaseIds)
|
|
536
|
+
const duplicateWritableMatrixCaseIds = duplicates(assignedWritableIds)
|
|
537
|
+
addCoverageViolation(violations, 'writable release matrix is missing cases', missingWritableMatrixCaseIds)
|
|
538
|
+
addCoverageViolation(violations, 'writable release matrix contains non-writable cases', unexpectedWritableMatrixCaseIds)
|
|
539
|
+
addCoverageViolation(violations, 'writable release matrix contains duplicate cases', duplicateWritableMatrixCaseIds)
|
|
540
|
+
for (const entry of writableEntries) {
|
|
541
|
+
if (!entry || typeof entry !== 'object' || Array.isArray(entry) || Object.keys(entry).length !== 1 || typeof entry.caseId !== 'string') violations.push(`writable release assignment must be runner-neutral: ${String(entry?.caseId)}`)
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
const configuredReviewIds = Array.isArray(releaseMatrix?.generativeJudge?.caseIds)
|
|
545
|
+
? releaseMatrix.generativeJudge.caseIds
|
|
546
|
+
: []
|
|
547
|
+
const missingReviewMatrixCaseIds = difference(reviewEligibleCaseIds, configuredReviewIds)
|
|
548
|
+
const unexpectedReviewMatrixCaseIds = difference(configuredReviewIds, reviewEligibleCaseIds)
|
|
549
|
+
const duplicateReviewMatrixCaseIds = duplicates(configuredReviewIds)
|
|
550
|
+
addCoverageViolation(violations, 'generated-code review matrix is missing eligible cases', missingReviewMatrixCaseIds)
|
|
551
|
+
addCoverageViolation(violations, 'generated-code review matrix contains ineligible cases', unexpectedReviewMatrixCaseIds)
|
|
552
|
+
addCoverageViolation(violations, 'generated-code review matrix contains duplicate cases', duplicateReviewMatrixCaseIds)
|
|
553
|
+
if (JSON.stringify(configuredReviewIds) !== JSON.stringify(reviewEligibleCaseIds)) {
|
|
554
|
+
violations.push('generated-code review matrix must list every writable case exactly once in catalog order')
|
|
555
|
+
}
|
|
556
|
+
if (release.requireGenerativeJudge !== true) violations.push('release suite must require the generative judge')
|
|
557
|
+
if (release.requireGeneratedCodeReview !== true) violations.push('release suite must retain generated-code review compatibility')
|
|
558
|
+
if (releaseMatrix?.generativeJudge?.skill !== 'om-judge-agent-session') violations.push('generative judge must explicitly use om-judge-agent-session')
|
|
559
|
+
if (releaseMatrix?.generativeJudge?.reviewSkill !== 'om-code-review') violations.push('generative judge must explicitly compose om-code-review')
|
|
560
|
+
if (releaseMatrix?.generativeJudge?.required !== true) violations.push('generative judge must be mandatory for every writable case')
|
|
561
|
+
if (releaseMatrix?.generatedCodeReview?.skill !== 'om-code-review'
|
|
562
|
+
|| JSON.stringify(releaseMatrix.generatedCodeReview?.caseIds) !== JSON.stringify(configuredReviewIds)
|
|
563
|
+
|| JSON.stringify(releaseMatrix.generatedCodeReview?.runners) !== JSON.stringify(releaseMatrix.generativeJudge?.runners)) {
|
|
564
|
+
violations.push('generated-code review compatibility matrix must match the generative judge')
|
|
565
|
+
}
|
|
566
|
+
const reviewRunners = releaseMatrix?.generativeJudge?.runners ?? {}
|
|
567
|
+
for (const runner of RELEASE_SUPPORTED_RUNNERS) {
|
|
568
|
+
if (!validModelSelector(reviewRunners?.[runner]?.modelSelector)) violations.push(`generated-code review lacks a valid bounded ${runner} model selector`)
|
|
569
|
+
}
|
|
570
|
+
for (const runner of Object.keys(reviewRunners)) {
|
|
571
|
+
if (!RUNNERS.has(runner)) violations.push(`generated-code review declares an invalid runner: ${runner}`)
|
|
572
|
+
}
|
|
573
|
+
const reviewSkillFiles = [
|
|
574
|
+
'.ai/skills/om-judge-agent-session/SKILL.md',
|
|
575
|
+
'.ai/skills/om-judge-agent-session/references/agentic-setup.md',
|
|
576
|
+
'.ai/skills/om-judge-agent-session/references/input-normalization.md',
|
|
577
|
+
'.ai/skills/om-judge-agent-session/references/judge-workflow.md',
|
|
578
|
+
'.ai/skills/om-judge-agent-session/references/report-template.md',
|
|
579
|
+
'.ai/skills/om-judge-agent-session/references/rules.md',
|
|
580
|
+
'.agents/skills/om-code-review/SKILL.md',
|
|
581
|
+
'.agents/skills/om-code-review/references/agentic-setup.md',
|
|
582
|
+
'.agents/skills/om-code-review/references/output-format.md',
|
|
583
|
+
'.agents/skills/om-code-review/references/review-checklist.md',
|
|
584
|
+
'.agents/skills/om-code-review/references/rules.md',
|
|
585
|
+
'.agents/skills/.om-external-ownership.json',
|
|
586
|
+
]
|
|
587
|
+
const reviewSkillReady = !root || reviewSkillFiles.every((relative) => fs.existsSync(path.join(root, relative)))
|
|
588
|
+
if (!reviewSkillReady) violations.push('generative judge or code-review skill is not installed with ownership evidence')
|
|
589
|
+
|
|
590
|
+
const registryCaseCountMatches = registry?.catalog?.expectedCaseCount === cases.length
|
|
591
|
+
if (!registryCaseCountMatches) violations.push(`validator registry expects ${String(registry?.catalog?.expectedCaseCount)} cases but catalog contains ${cases.length}`)
|
|
592
|
+
const registryWritableIds = Array.isArray(registry?.catalog?.writableCaseIds) ? registry.catalog.writableCaseIds : []
|
|
593
|
+
const registryMissingWritableCaseIds = difference(writableCaseIds, registryWritableIds)
|
|
594
|
+
const registryUnexpectedWritableCaseIds = difference(registryWritableIds, writableCaseIds)
|
|
595
|
+
addCoverageViolation(violations, 'validator registry is missing writable cases', registryMissingWritableCaseIds)
|
|
596
|
+
addCoverageViolation(violations, 'validator registry contains non-writable cases', registryUnexpectedWritableCaseIds)
|
|
597
|
+
|
|
598
|
+
const missingFixtureCaseIds = []
|
|
599
|
+
const missingOracleCaseIds = []
|
|
600
|
+
for (const item of writableCases) {
|
|
601
|
+
const setupEntries = Array.isArray(item.fixture?.setup) ? item.fixture.setup : []
|
|
602
|
+
const setupIds = setupEntries
|
|
603
|
+
.filter((entry) => typeof entry === 'string' && /^fixture:[a-z0-9-]+$/.test(entry))
|
|
604
|
+
.map((entry) => entry.slice('fixture:'.length))
|
|
605
|
+
const fixtureComplete = item.fixture?.scaffold === 'fresh-standalone'
|
|
606
|
+
&& setupIds.length > 0
|
|
607
|
+
&& setupIds.length === setupEntries.length
|
|
608
|
+
&& setupIds.every((id) => fixtures?.fixtures?.[id] && seeds?.fixtures?.[id])
|
|
609
|
+
if (!fixtureComplete) missingFixtureCaseIds.push(item.id)
|
|
610
|
+
const oracleIds = Array.isArray(item.oracle?.validatorIds) ? item.oracle.validatorIds : []
|
|
611
|
+
const oracleComplete = oracleIds.some((id) => {
|
|
612
|
+
const declaration = registry?.validators?.[id]
|
|
613
|
+
return declaration?.implementation === 'trusted-executable' && Array.isArray(declaration.runners) && declaration.runners.length > 0
|
|
614
|
+
}) && Array.isArray(item.oracle?.expectedArtifacts) && item.oracle.expectedArtifacts.length > 0
|
|
615
|
+
if (!oracleComplete) missingOracleCaseIds.push(item.id)
|
|
616
|
+
}
|
|
617
|
+
addCoverageViolation(violations, 'writable cases lack complete fixture coverage', missingFixtureCaseIds)
|
|
618
|
+
addCoverageViolation(violations, 'writable cases lack trusted oracle coverage', missingOracleCaseIds)
|
|
619
|
+
|
|
620
|
+
const targetEntries = validTargetsManifest(targetsManifest) ? targetsManifest.targets : {}
|
|
621
|
+
if (!validTargetsManifest(targetsManifest)) violations.push('writable target manifest must use schemaVersion 1 and a targets object')
|
|
622
|
+
const targetIds = Object.keys(targetEntries)
|
|
623
|
+
const missingTargetCaseIds = writableCaseIds.filter((id) => typeof targetEntries[id] !== 'string' || !path.isAbsolute(targetEntries[id]))
|
|
624
|
+
const unexpectedTargetCaseIds = difference(targetIds, writableCaseIds)
|
|
625
|
+
const duplicateTargetCaseIds = []
|
|
626
|
+
const invalidTargetCaseIds = []
|
|
627
|
+
const targetOwners = new Map()
|
|
628
|
+
const controllerRoot = root ? fs.realpathSync(root) : undefined
|
|
629
|
+
for (const id of writableCaseIds) {
|
|
630
|
+
const target = targetEntries[id]
|
|
631
|
+
if (typeof target !== 'string' || !path.isAbsolute(target)) continue
|
|
632
|
+
let normalized = path.resolve(target)
|
|
633
|
+
if (root && !targetsMayNotExist) {
|
|
634
|
+
try {
|
|
635
|
+
const stat = fs.lstatSync(target)
|
|
636
|
+
normalized = fs.realpathSync(target)
|
|
637
|
+
const required = ['package.json', 'src/modules.ts', '.ai/harness/cases.json']
|
|
638
|
+
const requiredAreSafe = required.every((relative) => {
|
|
639
|
+
const absolute = path.join(normalized, relative)
|
|
640
|
+
try {
|
|
641
|
+
const entry = fs.lstatSync(absolute)
|
|
642
|
+
return entry.isFile() && !entry.isSymbolicLink() && isPathInside(normalized, fs.realpathSync(absolute))
|
|
643
|
+
} catch { return false }
|
|
644
|
+
})
|
|
645
|
+
const dependencyPath = path.join(normalized, 'node_modules')
|
|
646
|
+
let dependenciesAreSafe = false
|
|
647
|
+
try {
|
|
648
|
+
const dependencyEntry = fs.lstatSync(dependencyPath)
|
|
649
|
+
dependenciesAreSafe = dependencyEntry.isSymbolicLink()
|
|
650
|
+
&& fs.statSync(dependencyPath).isDirectory()
|
|
651
|
+
&& root
|
|
652
|
+
&& fs.realpathSync(dependencyPath) === fs.realpathSync(path.join(root, 'node_modules'))
|
|
653
|
+
} catch { /* invalid target below */ }
|
|
654
|
+
if (!stat.isDirectory() || stat.isSymbolicLink() || normalized === root
|
|
655
|
+
|| !requiredAreSafe || !dependenciesAreSafe
|
|
656
|
+
|| fs.existsSync(path.join(normalized, '.ai', 'harness', 'DISPOSABLE'))) invalidTargetCaseIds.push(id)
|
|
657
|
+
} catch { invalidTargetCaseIds.push(id) }
|
|
658
|
+
}
|
|
659
|
+
if (controllerRoot && (isPathInside(controllerRoot, normalized) || isPathInside(normalized, controllerRoot))) invalidTargetCaseIds.push(id)
|
|
660
|
+
for (const [ownedRoot, ownerId] of targetOwners) {
|
|
661
|
+
if (isPathInside(ownedRoot, normalized) || isPathInside(normalized, ownedRoot)) duplicateTargetCaseIds.push(id, ownerId)
|
|
662
|
+
}
|
|
663
|
+
targetOwners.set(normalized, id)
|
|
664
|
+
}
|
|
665
|
+
addCoverageViolation(violations, 'writable target manifest is missing absolute targets', sortedUnique(missingTargetCaseIds))
|
|
666
|
+
addCoverageViolation(violations, 'writable target manifest contains unknown cases', unexpectedTargetCaseIds)
|
|
667
|
+
addCoverageViolation(violations, 'writable target manifest reuses targets', sortedUnique(duplicateTargetCaseIds))
|
|
668
|
+
addCoverageViolation(violations, 'writable target manifest contains invalid or reused app roots', sortedUnique(invalidTargetCaseIds))
|
|
669
|
+
|
|
670
|
+
const validationCommands = Array.isArray(release.validationCommands) ? release.validationCommands : []
|
|
671
|
+
if (validationCommands.length === 0) violations.push('release suite must declare validation commands')
|
|
672
|
+
for (const command of validationCommands) {
|
|
673
|
+
if (!ALLOWED_VALIDATION_COMMANDS.has(command)) violations.push(`release validation command is not allowed: ${String(command)}`)
|
|
674
|
+
}
|
|
675
|
+
const missingValidationCommands = difference([...ALLOWED_VALIDATION_COMMANDS], validationCommands)
|
|
676
|
+
addCoverageViolation(violations, 'release suite is missing validation commands', missingValidationCommands)
|
|
677
|
+
if (JSON.stringify(validationCommands) !== JSON.stringify(RELEASE_VALIDATION_COMMANDS)) {
|
|
678
|
+
violations.push('release validation commands must be the exact ordered four-command gate')
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
const generatedTestCases = writableCases.filter((item) => item.tags?.some((tag) => ['unit-tests', 'api-tests', 'browser-tests'].includes(tag)))
|
|
682
|
+
const expectedGeneratedTestIds = generatedTestCases.map((item) => item.id)
|
|
683
|
+
const generatedTestEntries = Array.isArray(releaseMatrix?.generatedTests?.entries) ? releaseMatrix.generatedTests.entries : []
|
|
684
|
+
const configuredGeneratedTestIds = generatedTestEntries.map((entry) => entry.caseId)
|
|
685
|
+
const missingGeneratedTestIds = difference(expectedGeneratedTestIds, configuredGeneratedTestIds)
|
|
686
|
+
const unexpectedGeneratedTestIds = difference(configuredGeneratedTestIds, expectedGeneratedTestIds)
|
|
687
|
+
const duplicateGeneratedTestIds = duplicates(configuredGeneratedTestIds)
|
|
688
|
+
if (releaseMatrix?.generatedTests?.required !== true) violations.push('generated test execution must be required')
|
|
689
|
+
addCoverageViolation(violations, 'generated test execution matrix is missing cases', missingGeneratedTestIds)
|
|
690
|
+
addCoverageViolation(violations, 'generated test execution matrix contains ineligible cases', unexpectedGeneratedTestIds)
|
|
691
|
+
addCoverageViolation(violations, 'generated test execution matrix contains duplicate cases', duplicateGeneratedTestIds)
|
|
692
|
+
for (const entry of generatedTestEntries) {
|
|
693
|
+
const item = generatedTestCases.find((candidate) => candidate.id === entry.caseId)
|
|
694
|
+
const expectedRunner = item?.tags?.includes('unit-tests') ? 'jest'
|
|
695
|
+
: item?.tags?.includes('api-tests') ? 'playwright-api'
|
|
696
|
+
: item?.tags?.includes('browser-tests') ? 'playwright-browser' : undefined
|
|
697
|
+
const expectedNetwork = expectedRunner === 'jest' ? 'none' : 'loopback'
|
|
698
|
+
const canonicalPath = expectedRunner === 'jest'
|
|
699
|
+
? /^src\/modules\/[a-z0-9_]+\/(?:[^/]+\/)*__tests__\/[A-Za-z0-9._-]+\.(?:test|spec)\.tsx?$/
|
|
700
|
+
: /^src\/modules\/[a-z0-9_]+\/(?:[^/]+\/)*__integration__\/[A-Za-z0-9._-]+\.spec\.tsx?$/
|
|
701
|
+
if (!item || !GENERATED_TEST_RUNNERS.has(entry.runner) || entry.runner !== expectedRunner
|
|
702
|
+
|| entry.network !== expectedNetwork || typeof entry.artifact !== 'string'
|
|
703
|
+
|| !canonicalPath.test(entry.artifact) || !item.oracle?.expectedArtifacts?.includes(entry.artifact)
|
|
704
|
+
|| !item.allowedWrites?.includes(entry.artifact)) {
|
|
705
|
+
violations.push(`generated test execution assignment is invalid: ${String(entry.caseId)}`)
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
const steps = [
|
|
710
|
+
{ id: 'deterministic:all', kind: 'deterministic', expectedCaseIds: deterministicIds },
|
|
711
|
+
...validationCommands.map((command) => ({ id: `validation:${command.slice('yarn '.length)}`, kind: 'validation', command })),
|
|
712
|
+
...routing.map((entry) => ({ id: `routing:${entry.lane}:${entry.runner}`, kind: 'routing', lane: entry.lane, runner: entry.runner, modelSelector: releaseMatrix.routing.runners[entry.runner].modelSelector, expectedCaseIds: entry.expectedCaseIds })),
|
|
713
|
+
]
|
|
714
|
+
for (const caseId of writableCaseIds) {
|
|
715
|
+
const assignment = writableEntries.find((entry) => entry?.caseId === caseId)
|
|
716
|
+
if (!assignment || !RUNNERS.has(primaryRunner)) continue
|
|
717
|
+
steps.push({ id: `fixture:${caseId}`, kind: 'fixture', caseId })
|
|
718
|
+
steps.push({ id: `writable:${caseId}`, kind: 'writable', caseId, runner: primaryRunner, modelSelector: releaseMatrix.routing.runners[primaryRunner].modelSelector })
|
|
719
|
+
for (const command of validationCommands) {
|
|
720
|
+
steps.push({ id: `target-validation:${caseId}:${command.slice('yarn '.length)}`, kind: 'target-validation', caseId, command })
|
|
721
|
+
}
|
|
722
|
+
const generatedTest = generatedTestEntries.find((entry) => entry.caseId === caseId)
|
|
723
|
+
if (generatedTest) {
|
|
724
|
+
const executionContract = generatedTestExecutionContract(generatedTest)
|
|
725
|
+
steps.push({
|
|
726
|
+
id: `generated-test:${caseId}:${generatedTest.runner}`,
|
|
727
|
+
kind: 'generated-test',
|
|
728
|
+
caseId,
|
|
729
|
+
testRunner: generatedTest.runner,
|
|
730
|
+
artifact: generatedTest.artifact,
|
|
731
|
+
network: generatedTest.network,
|
|
732
|
+
...executionContract,
|
|
733
|
+
})
|
|
734
|
+
}
|
|
735
|
+
if (configuredReviewIds.includes(caseId)) {
|
|
736
|
+
const modelSelector = releaseMatrix.generativeJudge?.runners?.[primaryRunner]?.modelSelector
|
|
737
|
+
if (!validModelSelector(modelSelector)) violations.push(`generative judge lacks a valid bounded ${primaryRunner} model selector for ${caseId}`)
|
|
738
|
+
steps.push({ id: `review:${caseId}`, kind: 'review', caseId, runner: primaryRunner, modelSelector })
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
return {
|
|
743
|
+
runnerPolicy: { primaryRunner: RUNNERS.has(primaryRunner) ? primaryRunner : null, portabilityRunner: RUNNERS.has(portabilityRunner) ? portabilityRunner : null },
|
|
744
|
+
catalog: {
|
|
745
|
+
caseCount: cases.length,
|
|
746
|
+
writableCaseCount: writableCaseIds.length,
|
|
747
|
+
reviewEligibleCaseCount: reviewEligibleCaseIds.length,
|
|
748
|
+
},
|
|
749
|
+
coverage: {
|
|
750
|
+
deterministic: { expectedCaseIds: allCaseIds, configuredCaseIds: deterministicIds, missingMatrixCaseIds: deterministicMissingCaseIds, unexpectedMatrixCaseIds: deterministicUnexpectedCaseIds },
|
|
751
|
+
routing,
|
|
752
|
+
writable: {
|
|
753
|
+
expectedCaseIds: writableCaseIds,
|
|
754
|
+
configuredCaseIds: assignedWritableIds,
|
|
755
|
+
missingMatrixCaseIds: missingWritableMatrixCaseIds,
|
|
756
|
+
unexpectedMatrixCaseIds: unexpectedWritableMatrixCaseIds,
|
|
757
|
+
duplicateMatrixCaseIds: duplicateWritableMatrixCaseIds,
|
|
758
|
+
missingTargetCaseIds: sortedUnique(missingTargetCaseIds),
|
|
759
|
+
unexpectedTargetCaseIds,
|
|
760
|
+
duplicateTargetCaseIds: sortedUnique(duplicateTargetCaseIds),
|
|
761
|
+
invalidTargetCaseIds: sortedUnique(invalidTargetCaseIds),
|
|
762
|
+
missingFixtureCaseIds: sortedUnique(missingFixtureCaseIds),
|
|
763
|
+
missingOracleCaseIds: sortedUnique(missingOracleCaseIds),
|
|
764
|
+
},
|
|
765
|
+
review: { expectedCaseIds: reviewEligibleCaseIds, configuredCaseIds: configuredReviewIds, missingMatrixCaseIds: missingReviewMatrixCaseIds, unexpectedMatrixCaseIds: unexpectedReviewMatrixCaseIds, skillReady: reviewSkillReady },
|
|
766
|
+
registry: { caseCountMatches: registryCaseCountMatches, missingWritableCaseIds: registryMissingWritableCaseIds, unexpectedWritableCaseIds: registryUnexpectedWritableCaseIds },
|
|
767
|
+
validation: { expectedCommands: [...ALLOWED_VALIDATION_COMMANDS], configuredCommands: validationCommands, missingCommands: missingValidationCommands },
|
|
768
|
+
targetValidation: { expectedCaseIds: writableCaseIds, commands: validationCommands },
|
|
769
|
+
generatedTests: {
|
|
770
|
+
expectedCaseIds: expectedGeneratedTestIds,
|
|
771
|
+
configuredCaseIds: configuredGeneratedTestIds,
|
|
772
|
+
missingMatrixCaseIds: missingGeneratedTestIds,
|
|
773
|
+
unexpectedMatrixCaseIds: unexpectedGeneratedTestIds,
|
|
774
|
+
duplicateMatrixCaseIds: duplicateGeneratedTestIds,
|
|
775
|
+
},
|
|
776
|
+
},
|
|
777
|
+
targets: targetEntries,
|
|
778
|
+
steps,
|
|
779
|
+
violations: sortedUnique(violations),
|
|
780
|
+
}
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
export function releaseHostPrerequisiteViolations(releaseMatrix, platform = process.platform, env = process.env) {
|
|
784
|
+
const entries = Array.isArray(releaseMatrix?.generatedTests?.entries) ? releaseMatrix.generatedTests.entries : []
|
|
785
|
+
const requiresLoopback = entries.some((entry) => entry?.network === 'loopback')
|
|
786
|
+
const modes = ['none', ...(requiresLoopback ? ['loopback'] : [])]
|
|
787
|
+
const violations = []
|
|
788
|
+
for (const mode of modes) {
|
|
789
|
+
try {
|
|
790
|
+
assertSandboxRuntimeAvailable(mode, platform, env)
|
|
791
|
+
} catch (error) {
|
|
792
|
+
violations.push(error instanceof Error ? error.message : String(error))
|
|
793
|
+
}
|
|
794
|
+
}
|
|
795
|
+
if (platform === 'linux' && requiresLoopback && violations.length === 0) {
|
|
796
|
+
try {
|
|
797
|
+
verifyLinuxSandboxIsolation(env)
|
|
798
|
+
} catch (error) {
|
|
799
|
+
violations.push(error instanceof Error ? error.message : String(error))
|
|
800
|
+
}
|
|
801
|
+
}
|
|
802
|
+
return [...new Set(violations)]
|
|
803
|
+
}
|
|
804
|
+
|
|
805
|
+
function percentage(numerator, denominator) {
|
|
806
|
+
return denominator === 0 ? 0 : Math.round((numerator / denominator) * 10_000) / 100
|
|
807
|
+
}
|
|
808
|
+
|
|
809
|
+
function percentile(values, value) {
|
|
810
|
+
if (values.length === 0) return 0
|
|
811
|
+
const sorted = [...values].sort((left, right) => left - right)
|
|
812
|
+
return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * value) - 1)]
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
function violationCategory(value) {
|
|
816
|
+
const text = String(value)
|
|
817
|
+
if (/forbidden|unsafe|out-of-root|outside allowlist|protected roots|inspection|modified (?:target|bundle)|trace unavailable|no observed context/i.test(text)) return 'safety'
|
|
818
|
+
if (/route|skill|context|decision|pattern|budget|missing|required|unexpected|mismatch/i.test(text)) return 'contract'
|
|
819
|
+
if (/process|environment|structured output|timed out|runner/i.test(text)) return 'execution'
|
|
820
|
+
return 'quality'
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
export function aggregateQualityMetrics(results) {
|
|
824
|
+
const contextTokens = []
|
|
825
|
+
const initialContextTokens = []
|
|
826
|
+
let passed = 0
|
|
827
|
+
let corrections = 0
|
|
828
|
+
let resultsWithCorrections = 0
|
|
829
|
+
let firstPassPassed = 0
|
|
830
|
+
let totalViolations = 0
|
|
831
|
+
let resultsWithViolations = 0
|
|
832
|
+
const violationsByCategory = { safety: 0, contract: 0, execution: 0, quality: 0 }
|
|
833
|
+
const verdicts = { approve: 0, requestChanges: 0, error: 0 }
|
|
834
|
+
for (const result of results) {
|
|
835
|
+
if (result.status === 'pass') passed += 1
|
|
836
|
+
const resultCorrections = Number.isInteger(result.corrections) ? result.corrections : Math.max(0, (result.attempts ?? 1) - 1)
|
|
837
|
+
corrections += resultCorrections
|
|
838
|
+
if (resultCorrections > 0) resultsWithCorrections += 1
|
|
839
|
+
if (result.status === 'pass' && resultCorrections === 0) firstPassPassed += 1
|
|
840
|
+
if (Number.isInteger(result.actualContext?.estimatedTokens)) contextTokens.push(result.actualContext.estimatedTokens)
|
|
841
|
+
if (Number.isInteger(result.actualContext?.estimatedInitialTokens)) initialContextTokens.push(result.actualContext.estimatedInitialTokens)
|
|
842
|
+
const resultViolations = Array.isArray(result.violations) ? result.violations : []
|
|
843
|
+
if (resultViolations.length) resultsWithViolations += 1
|
|
844
|
+
totalViolations += resultViolations.length
|
|
845
|
+
for (const violation of resultViolations) violationsByCategory[violationCategory(violation)] += 1
|
|
846
|
+
if (result.verdict === 'approve') verdicts.approve += 1
|
|
847
|
+
else if (result.verdict === 'request changes') verdicts.requestChanges += 1
|
|
848
|
+
else if (result.verdict === 'error') verdicts.error += 1
|
|
849
|
+
}
|
|
850
|
+
const totalContextTokens = contextTokens.reduce((sum, value) => sum + value, 0)
|
|
851
|
+
const totalInitialContextTokens = initialContextTokens.reduce((sum, value) => sum + value, 0)
|
|
852
|
+
return {
|
|
853
|
+
outcomes: { results: results.length, passed, failed: results.length - passed },
|
|
854
|
+
firstPass: { eligibleResults: results.length, passedFirstPass: firstPassPassed, ratePct: percentage(firstPassPassed, results.length) },
|
|
855
|
+
corrections: { total: corrections, resultsWithCorrections, ratePct: percentage(resultsWithCorrections, results.length) },
|
|
856
|
+
context: {
|
|
857
|
+
measuredResults: contextTokens.length,
|
|
858
|
+
totalEstimatedTokens: totalContextTokens,
|
|
859
|
+
averageEstimatedTokens: contextTokens.length ? Math.round(totalContextTokens / contextTokens.length) : 0,
|
|
860
|
+
p95EstimatedTokens: percentile(contextTokens, 0.95),
|
|
861
|
+
maxEstimatedTokens: contextTokens.length ? Math.max(...contextTokens) : 0,
|
|
862
|
+
totalEstimatedInitialTokens: totalInitialContextTokens,
|
|
863
|
+
averageEstimatedInitialTokens: initialContextTokens.length ? Math.round(totalInitialContextTokens / initialContextTokens.length) : 0,
|
|
864
|
+
},
|
|
865
|
+
misuse: { totalViolations, resultsWithViolations, ratePct: percentage(resultsWithViolations, results.length), byCategory: violationsByCategory },
|
|
866
|
+
reviewVerdicts: verdicts,
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
|
|
870
|
+
export function sanitizeReportText(value, roots = []) {
|
|
871
|
+
let result = String(value ?? '')
|
|
872
|
+
for (const root of [...roots, os.homedir()].filter(Boolean).sort((left, right) => right.length - left.length)) result = result.split(root).join('<redacted-path>')
|
|
873
|
+
const environmentSecrets = [...new Set(Object.entries(process.env)
|
|
874
|
+
.filter(([key, entry]) => SENSITIVE_ENV_KEY.test(key) && typeof entry === 'string' && entry.length >= 4)
|
|
875
|
+
.map(([, entry]) => entry))]
|
|
876
|
+
.sort((left, right) => right.length - left.length)
|
|
877
|
+
for (const secret of environmentSecrets) result = result.split(secret).join('<redacted-secret>')
|
|
878
|
+
result = result.replace(/\b([a-z][a-z0-9+.-]*:\/\/)[^\s/@]+@/gi, '$1<redacted-secret>@')
|
|
879
|
+
result = result.replace(/\b(?:sk|pk|ghp|github_pat|xox[baprs]|AKIA)[-_A-Za-z0-9]{10,}\b/g, '<redacted-secret>')
|
|
880
|
+
result = result.replace(/\b(?:api[_-]?key|token|secret|password)\s*[:=]\s*[^\s,;]+/gi, '<redacted-secret>')
|
|
881
|
+
return result.slice(0, ERROR_LIMIT)
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
function sanitizeObject(value, roots) {
|
|
885
|
+
if (typeof value === 'string') return sanitizeReportText(value, roots)
|
|
886
|
+
if (Array.isArray(value)) return value.map((entry) => sanitizeObject(entry, roots))
|
|
887
|
+
if (value && typeof value === 'object') return Object.fromEntries(Object.entries(value).map(([key, entry]) => [key, sanitizeObject(entry, roots)]))
|
|
888
|
+
return value
|
|
889
|
+
}
|
|
890
|
+
|
|
891
|
+
function schemaTypeMatches(value, expected) {
|
|
892
|
+
if (expected === 'null') return value === null
|
|
893
|
+
if (expected === 'array') return Array.isArray(value)
|
|
894
|
+
if (expected === 'object') return value !== null && typeof value === 'object' && !Array.isArray(value)
|
|
895
|
+
if (expected === 'integer') return Number.isInteger(value)
|
|
896
|
+
if (expected === 'number') return typeof value === 'number' && Number.isFinite(value)
|
|
897
|
+
return typeof value === expected
|
|
898
|
+
}
|
|
899
|
+
|
|
900
|
+
function resolveSchemaReference(rootSchema, reference) {
|
|
901
|
+
if (!reference.startsWith('#/')) throw new Error(`unsupported release schema reference ${reference}`)
|
|
902
|
+
return reference.slice(2).split('/').reduce((value, key) => value?.[key.replaceAll('~1', '/').replaceAll('~0', '~')], rootSchema)
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
function validateJsonSchema(value, schema, rootSchema = schema, location = '$') {
|
|
906
|
+
if (schema.$ref) return validateJsonSchema(value, resolveSchemaReference(rootSchema, schema.$ref), rootSchema, location)
|
|
907
|
+
const errors = []
|
|
908
|
+
const types = Array.isArray(schema.type) ? schema.type : schema.type ? [schema.type] : []
|
|
909
|
+
if (types.length && !types.some((type) => schemaTypeMatches(value, type))) return [`${location} has the wrong type`]
|
|
910
|
+
if (Object.hasOwn(schema, 'const') && value !== schema.const) errors.push(`${location} differs from its constant`)
|
|
911
|
+
if (schema.enum && !schema.enum.some((entry) => JSON.stringify(entry) === JSON.stringify(value))) errors.push(`${location} is outside its enum`)
|
|
912
|
+
if (typeof value === 'string') {
|
|
913
|
+
if (Number.isInteger(schema.maxLength) && value.length > schema.maxLength) errors.push(`${location} exceeds maxLength`)
|
|
914
|
+
if (schema.pattern && !new RegExp(schema.pattern).test(value)) errors.push(`${location} does not match its pattern`)
|
|
915
|
+
}
|
|
916
|
+
if (typeof value === 'number') {
|
|
917
|
+
if (Number.isFinite(schema.minimum) && value < schema.minimum) errors.push(`${location} is below minimum`)
|
|
918
|
+
if (Number.isFinite(schema.maximum) && value > schema.maximum) errors.push(`${location} exceeds maximum`)
|
|
919
|
+
}
|
|
920
|
+
if (Array.isArray(value)) {
|
|
921
|
+
if (Number.isInteger(schema.maxItems) && value.length > schema.maxItems) errors.push(`${location} exceeds maxItems`)
|
|
922
|
+
if (schema.uniqueItems && new Set(value.map((entry) => JSON.stringify(entry))).size !== value.length) errors.push(`${location} must contain unique items`)
|
|
923
|
+
if (schema.items) value.forEach((entry, index) => errors.push(...validateJsonSchema(entry, schema.items, rootSchema, `${location}[${index}]`)))
|
|
924
|
+
}
|
|
925
|
+
if (value !== null && typeof value === 'object' && !Array.isArray(value)) {
|
|
926
|
+
for (const required of schema.required ?? []) if (!Object.hasOwn(value, required)) errors.push(`${location}.${required} is required`)
|
|
927
|
+
if (schema.additionalProperties === false) {
|
|
928
|
+
for (const key of Object.keys(value)) if (!Object.hasOwn(schema.properties ?? {}, key)) errors.push(`${location}.${key} is not allowed`)
|
|
929
|
+
}
|
|
930
|
+
for (const [key, childSchema] of Object.entries(schema.properties ?? {})) {
|
|
931
|
+
if (Object.hasOwn(value, key)) errors.push(...validateJsonSchema(value[key], childSchema, rootSchema, `${location}.${key}`))
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
return errors
|
|
935
|
+
}
|
|
936
|
+
|
|
937
|
+
function resultFiles(root) {
|
|
938
|
+
const directory = path.join(root, '.ai', 'harness', 'results')
|
|
939
|
+
if (!fs.existsSync(directory)) return new Set()
|
|
940
|
+
return new Set(fs.readdirSync(directory).filter((file) => file.endsWith('.json')))
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
function readNewResults(root, before) {
|
|
944
|
+
const directory = path.join(root, '.ai', 'harness', 'results')
|
|
945
|
+
if (!fs.existsSync(directory)) return []
|
|
946
|
+
const results = []
|
|
947
|
+
for (const file of fs.readdirSync(directory).filter((entry) => entry.endsWith('.json') && !before.has(entry)).sort()) {
|
|
948
|
+
const absolute = path.join(directory, file)
|
|
949
|
+
const stat = fs.lstatSync(absolute)
|
|
950
|
+
if (!stat.isFile() || stat.isSymbolicLink() || stat.size > RESULT_LIMIT || file.includes('-release-')) continue
|
|
951
|
+
try { results.push({ path: path.relative(root, absolute).replaceAll(path.sep, '/'), value: readJson(absolute) }) } catch { /* evaluator owns schema validation */ }
|
|
952
|
+
}
|
|
953
|
+
return results
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
function execute(command, args, cwd, timeout, env = process.env) {
|
|
957
|
+
const started = Date.now()
|
|
958
|
+
const result = spawnSync(command, args, { cwd, encoding: 'utf8', timeout, maxBuffer: 8 * 1024 * 1024, env })
|
|
959
|
+
return { ...result, durationMs: Date.now() - started }
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
function protectedTreeFingerprint(target) {
|
|
963
|
+
if (!fs.existsSync(target)) return '<missing>'
|
|
964
|
+
const hash = createHash('sha256')
|
|
965
|
+
const visit = (absolute, relative = '.') => {
|
|
966
|
+
let stat
|
|
967
|
+
try { stat = fs.lstatSync(absolute) } catch { hash.update(`${relative}:<unreadable>\n`); return }
|
|
968
|
+
hash.update(`${relative}:${stat.mode}:${stat.size}:${stat.mtimeMs}:${stat.ctimeMs}\n`)
|
|
969
|
+
if (stat.isSymbolicLink()) {
|
|
970
|
+
try { hash.update(`link:${fs.readlinkSync(absolute)}\n`) } catch { /* metadata still detects the entry */ }
|
|
971
|
+
return
|
|
972
|
+
}
|
|
973
|
+
if (!stat.isDirectory()) return
|
|
974
|
+
for (const entry of fs.readdirSync(absolute).sort()) visit(path.join(absolute, entry), relative === '.' ? entry : `${relative}/${entry}`)
|
|
975
|
+
}
|
|
976
|
+
visit(target)
|
|
977
|
+
return hash.digest('hex')
|
|
978
|
+
}
|
|
979
|
+
|
|
980
|
+
export function dependencyContentFingerprint(appRoot) {
|
|
981
|
+
const dependencyRoot = path.join(appRoot, 'node_modules')
|
|
982
|
+
const stat = fs.lstatSync(dependencyRoot)
|
|
983
|
+
if (!stat.isDirectory() || stat.isSymbolicLink()) throw new Error('controller node_modules must remain a regular directory')
|
|
984
|
+
const hash = createHash('sha256')
|
|
985
|
+
const ownershipFiles = ['package.json', 'yarn.lock']
|
|
986
|
+
for (const relative of ownershipFiles) {
|
|
987
|
+
const absolute = path.join(appRoot, relative)
|
|
988
|
+
if (!fs.existsSync(absolute)) continue
|
|
989
|
+
const fileStat = fs.lstatSync(absolute)
|
|
990
|
+
if (!fileStat.isFile() || fileStat.isSymbolicLink() || fileStat.size > 8 * 1024 * 1024) throw new Error(`dependency ownership file is unsafe: ${relative}`)
|
|
991
|
+
hash.update(relative)
|
|
992
|
+
hash.update(fs.readFileSync(absolute))
|
|
993
|
+
}
|
|
994
|
+
let entries = 0
|
|
995
|
+
const visit = (directory, relativeDirectory = '') => {
|
|
996
|
+
for (const name of fs.readdirSync(directory).sort()) {
|
|
997
|
+
entries += 1
|
|
998
|
+
if (entries > 500_000) throw new Error('dependency content fingerprint exceeds its 500000-entry safety bound')
|
|
999
|
+
const absolute = path.join(directory, name)
|
|
1000
|
+
const relative = relativeDirectory ? `${relativeDirectory}/${name}` : name
|
|
1001
|
+
const entry = fs.lstatSync(absolute)
|
|
1002
|
+
hash.update(`${relative}:${entry.mode}:${entry.size}\n`)
|
|
1003
|
+
if (entry.isSymbolicLink()) hash.update(`link:${fs.readlinkSync(absolute)}\n`)
|
|
1004
|
+
else if (entry.isDirectory()) visit(absolute, relative)
|
|
1005
|
+
else if (entry.isFile()) hash.update(fs.readFileSync(absolute))
|
|
1006
|
+
else throw new Error(`dependency tree contains an unsupported entry: ${relative}`)
|
|
1007
|
+
}
|
|
1008
|
+
}
|
|
1009
|
+
visit(dependencyRoot)
|
|
1010
|
+
return hash.digest('hex')
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
function targetSnapshot(root) {
|
|
1014
|
+
const result = new Map()
|
|
1015
|
+
const protectedRoots = new Set(['.git', 'node_modules', '.next', 'dist', '.ai/harness/results'])
|
|
1016
|
+
const visit = (directory, relativeDirectory = '') => {
|
|
1017
|
+
for (const name of fs.readdirSync(directory).sort()) {
|
|
1018
|
+
const relative = relativeDirectory ? `${relativeDirectory}/${name}` : name
|
|
1019
|
+
if ([...protectedRoots].some((entry) => relative === entry || relative.startsWith(`${entry}/`))) continue
|
|
1020
|
+
const absolute = path.join(directory, name)
|
|
1021
|
+
let entry
|
|
1022
|
+
try { entry = fs.lstatSync(absolute) } catch { result.set(relative, '<unreadable>'); continue }
|
|
1023
|
+
if (entry.isSymbolicLink()) {
|
|
1024
|
+
try { result.set(relative, `<symlink:${fs.readlinkSync(absolute)}>`) } catch { result.set(relative, '<symlink:unreadable>') }
|
|
1025
|
+
} else if (entry.isDirectory()) {
|
|
1026
|
+
result.set(relative, `<directory:${entry.mode & 0o777}>`)
|
|
1027
|
+
visit(absolute, relative)
|
|
1028
|
+
}
|
|
1029
|
+
else if (entry.isFile()) {
|
|
1030
|
+
try { result.set(relative, `<file:${entry.mode & 0o777}:${sha256(fs.readFileSync(absolute))}>`) } catch { result.set(relative, '<unreadable>') }
|
|
1031
|
+
} else result.set(relative, `<special:${entry.mode & 0o170000}>`)
|
|
1032
|
+
}
|
|
1033
|
+
}
|
|
1034
|
+
visit(root)
|
|
1035
|
+
for (const relative of ['.git', 'node_modules', '.next', 'dist', '.ai/harness/results']) {
|
|
1036
|
+
result.set(relative, protectedTreeFingerprint(path.join(root, relative)))
|
|
1037
|
+
}
|
|
1038
|
+
return result
|
|
1039
|
+
}
|
|
1040
|
+
|
|
1041
|
+
function containedValidationSymlink(relative, fingerprint, command, after) {
|
|
1042
|
+
if (!validationOutputRoot(relative, command, after)) return false
|
|
1043
|
+
const target = fingerprint.slice('<symlink:'.length, -1)
|
|
1044
|
+
if (!target || path.isAbsolute(target)) return false
|
|
1045
|
+
const resolved = path.normalize(path.join(path.dirname(relative), target))
|
|
1046
|
+
return resolved !== '..' && !resolved.startsWith(`..${path.sep}`) && !path.isAbsolute(resolved)
|
|
1047
|
+
}
|
|
1048
|
+
|
|
1049
|
+
function unsafeChangedTargetEntries(after, changed, command) {
|
|
1050
|
+
const protectedRoots = new Set(['.git', 'node_modules', '.next', 'dist', '.ai/harness/results'])
|
|
1051
|
+
return changed.flatMap((relative) => {
|
|
1052
|
+
if (protectedRoots.has(relative)) return []
|
|
1053
|
+
const fingerprint = after.get(relative)
|
|
1054
|
+
if (typeof fingerprint !== 'string') return []
|
|
1055
|
+
if (fingerprint.startsWith('<symlink:') && !containedValidationSymlink(relative, fingerprint, command, after)) return [`${relative} (symbolic link)`]
|
|
1056
|
+
if (fingerprint.startsWith('<special:')) return [`${relative} (special file)`]
|
|
1057
|
+
if (fingerprint === '<unreadable>') return [`${relative} (unreadable)`]
|
|
1058
|
+
return []
|
|
1059
|
+
})
|
|
1060
|
+
}
|
|
1061
|
+
|
|
1062
|
+
function targetSnapshotFingerprint(snapshot) {
|
|
1063
|
+
const hash = createHash('sha256')
|
|
1064
|
+
for (const [relative, fingerprint] of [...snapshot.entries()].sort(([left], [right]) => left.localeCompare(right))) {
|
|
1065
|
+
hash.update(relative)
|
|
1066
|
+
hash.update('\0')
|
|
1067
|
+
hash.update(fingerprint)
|
|
1068
|
+
hash.update('\0')
|
|
1069
|
+
}
|
|
1070
|
+
return hash.digest('hex')
|
|
1071
|
+
}
|
|
1072
|
+
|
|
1073
|
+
export function targetContentFingerprint(root) {
|
|
1074
|
+
return targetSnapshotFingerprint(targetSnapshot(root))
|
|
1075
|
+
}
|
|
1076
|
+
|
|
1077
|
+
function stepResult(step, execution, artifacts, roots, expectedArtifacts) {
|
|
1078
|
+
const artifactMismatch = Number.isInteger(expectedArtifacts) && artifacts.length !== expectedArtifacts
|
|
1079
|
+
const status = execution.status === 0 && !execution.error && !artifactMismatch ? 'pass' : 'fail'
|
|
1080
|
+
const diagnostic = artifactMismatch
|
|
1081
|
+
? `expected ${expectedArtifacts} result artifacts, found ${artifacts.length}`
|
|
1082
|
+
: execution.error?.message || (status === 'fail'
|
|
1083
|
+
? [execution.stderr, execution.stdout].filter(Boolean).join('\n')
|
|
1084
|
+
: execution.stderr)
|
|
1085
|
+
return {
|
|
1086
|
+
id: step.id,
|
|
1087
|
+
kind: step.kind,
|
|
1088
|
+
...(step.caseId ? { caseId: step.caseId } : {}),
|
|
1089
|
+
...(step.runner ? { runner: step.runner } : {}),
|
|
1090
|
+
...(step.testRunner ? { testRunner: step.testRunner } : {}),
|
|
1091
|
+
...(step.artifact ? { artifact: step.artifact } : {}),
|
|
1092
|
+
...(step.network ? { network: step.network } : {}),
|
|
1093
|
+
...(step.executor ? { executor: step.executor } : {}),
|
|
1094
|
+
...(step.cliPackage ? { cliPackage: step.cliPackage } : {}),
|
|
1095
|
+
...(step.cliSha256 ? { cliSha256: step.cliSha256 } : {}),
|
|
1096
|
+
...(step.argv ? { argv: step.argv } : {}),
|
|
1097
|
+
...(step.command ? { command: step.command } : {}),
|
|
1098
|
+
status,
|
|
1099
|
+
exitStatus: execution.status,
|
|
1100
|
+
durationMs: execution.durationMs,
|
|
1101
|
+
resultPaths: artifacts.map((entry) => entry.path),
|
|
1102
|
+
...(diagnostic ? { sanitizedError: sanitizeReportText(diagnostic, roots) } : {}),
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
|
|
1106
|
+
function skippedStep(step, reason) {
|
|
1107
|
+
return {
|
|
1108
|
+
id: step.id,
|
|
1109
|
+
kind: step.kind,
|
|
1110
|
+
...(step.caseId ? { caseId: step.caseId } : {}),
|
|
1111
|
+
...(step.runner ? { runner: step.runner } : {}),
|
|
1112
|
+
...(step.testRunner ? { testRunner: step.testRunner } : {}),
|
|
1113
|
+
...(step.artifact ? { artifact: step.artifact } : {}),
|
|
1114
|
+
...(step.network ? { network: step.network } : {}),
|
|
1115
|
+
...(step.executor ? { executor: step.executor } : {}),
|
|
1116
|
+
...(step.cliPackage ? { cliPackage: step.cliPackage } : {}),
|
|
1117
|
+
...(step.cliSha256 ? { cliSha256: step.cliSha256 } : {}),
|
|
1118
|
+
...(step.argv ? { argv: step.argv } : {}),
|
|
1119
|
+
...(step.command ? { command: step.command } : {}),
|
|
1120
|
+
status: 'skipped',
|
|
1121
|
+
exitStatus: null,
|
|
1122
|
+
durationMs: 0,
|
|
1123
|
+
resultPaths: [],
|
|
1124
|
+
sanitizedError: reason,
|
|
1125
|
+
}
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
export function createMinimalValidationEnvironment(tempRoot, pathValue = process.env.PATH ?? '') {
|
|
1129
|
+
const home = path.join(tempRoot, 'home')
|
|
1130
|
+
const temporary = path.join(tempRoot, 'tmp')
|
|
1131
|
+
const cache = path.join(tempRoot, 'cache')
|
|
1132
|
+
for (const directory of [home, temporary, cache]) fs.mkdirSync(directory, { recursive: true, mode: 0o700 })
|
|
1133
|
+
const env = {
|
|
1134
|
+
PATH: pathValue,
|
|
1135
|
+
HOME: home,
|
|
1136
|
+
TMPDIR: temporary,
|
|
1137
|
+
TEMP: temporary,
|
|
1138
|
+
TMP: temporary,
|
|
1139
|
+
XDG_CONFIG_HOME: path.join(home, '.config'),
|
|
1140
|
+
XDG_CACHE_HOME: cache,
|
|
1141
|
+
XDG_DATA_HOME: path.join(home, '.local', 'share'),
|
|
1142
|
+
CI: '1',
|
|
1143
|
+
NO_COLOR: '1',
|
|
1144
|
+
NEXT_TELEMETRY_DISABLED: '1',
|
|
1145
|
+
YARN_ENABLE_TELEMETRY: '0',
|
|
1146
|
+
COREPACK_ENABLE_DOWNLOAD_PROMPT: '0',
|
|
1147
|
+
TZ: 'UTC',
|
|
1148
|
+
__CF_USER_TEXT_ENCODING: '0x1F5:0x0:0x0',
|
|
1149
|
+
}
|
|
1150
|
+
const configuredCorepackHome = process.env.COREPACK_HOME || path.join(os.homedir(), '.cache', 'node', 'corepack')
|
|
1151
|
+
const corepackHome = fs.existsSync(configuredCorepackHome) && fs.statSync(configuredCorepackHome).isDirectory()
|
|
1152
|
+
? fs.realpathSync(configuredCorepackHome)
|
|
1153
|
+
: undefined
|
|
1154
|
+
if (corepackHome) env.COREPACK_HOME = corepackHome
|
|
1155
|
+
for (const key of ['SystemRoot', 'WINDIR', 'PATHEXT', 'ComSpec']) {
|
|
1156
|
+
if (typeof process.env[key] === 'string') env[key] = process.env[key]
|
|
1157
|
+
}
|
|
1158
|
+
return { env, toolReadRoots: corepackHome ? [corepackHome] : [] }
|
|
1159
|
+
}
|
|
1160
|
+
|
|
1161
|
+
function validationOutputRoot(relative, command, after) {
|
|
1162
|
+
if (
|
|
1163
|
+
relative === '.env'
|
|
1164
|
+
&& (command === 'yarn generate' || command === 'yarn build')
|
|
1165
|
+
&& after.get('.env') === after.get('.env.example')
|
|
1166
|
+
) return '.env'
|
|
1167
|
+
return (RELEASE_VALIDATION_OUTPUT_ROOTS.get(command) ?? []).find((root) => relative === root
|
|
1168
|
+
|| relative.startsWith(`${root}/`)
|
|
1169
|
+
|| (root.startsWith(`${relative}/`) && after.get(relative)?.startsWith('<directory:')))
|
|
1170
|
+
}
|
|
1171
|
+
|
|
1172
|
+
function cloneValidationDependencies(source, destination) {
|
|
1173
|
+
const sourceRoot = fs.realpathSync(source)
|
|
1174
|
+
const visit = (directory) => {
|
|
1175
|
+
for (const name of fs.readdirSync(directory)) {
|
|
1176
|
+
const absolute = path.join(directory, name)
|
|
1177
|
+
const entry = fs.lstatSync(absolute)
|
|
1178
|
+
if (entry.isSymbolicLink()) {
|
|
1179
|
+
const resolved = fs.realpathSync(absolute)
|
|
1180
|
+
if (resolved !== sourceRoot && !isPathInside(sourceRoot, resolved)) {
|
|
1181
|
+
throw new Error(`dependency tree contains an escaping symbolic link: ${path.relative(sourceRoot, absolute)}`)
|
|
1182
|
+
}
|
|
1183
|
+
} else if (entry.isDirectory()) visit(absolute)
|
|
1184
|
+
else if (!entry.isFile()) throw new Error(`dependency tree contains an unsupported entry: ${path.relative(sourceRoot, absolute)}`)
|
|
1185
|
+
}
|
|
1186
|
+
}
|
|
1187
|
+
visit(sourceRoot)
|
|
1188
|
+
fs.cpSync(sourceRoot, destination, {
|
|
1189
|
+
recursive: true,
|
|
1190
|
+
dereference: false,
|
|
1191
|
+
verbatimSymlinks: true,
|
|
1192
|
+
mode: fs.constants.COPYFILE_FICLONE,
|
|
1193
|
+
})
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1196
|
+
function prepareValidationWorkspace(target, tempRoot, { allowContainedDependencies = false } = {}) {
|
|
1197
|
+
validateScaffoldCopySource(target)
|
|
1198
|
+
const workspace = path.join(tempRoot, 'target')
|
|
1199
|
+
copyFreshScaffold(target, workspace)
|
|
1200
|
+
const dependencyPath = path.join(target, 'node_modules')
|
|
1201
|
+
let dependencyRoot
|
|
1202
|
+
if (fs.existsSync(dependencyPath)) {
|
|
1203
|
+
const sourceDependencyRoot = fs.realpathSync(dependencyPath)
|
|
1204
|
+
if (!allowContainedDependencies && isPathInside(fs.realpathSync(target), sourceDependencyRoot)) {
|
|
1205
|
+
throw new Error('writable target node_modules must resolve outside the writable target')
|
|
1206
|
+
}
|
|
1207
|
+
cloneValidationDependencies(sourceDependencyRoot, path.join(workspace, 'node_modules'))
|
|
1208
|
+
dependencyRoot = fs.realpathSync(path.join(workspace, 'node_modules'))
|
|
1209
|
+
}
|
|
1210
|
+
return { workspace: fs.realpathSync(workspace), dependencyRoot }
|
|
1211
|
+
}
|
|
1212
|
+
|
|
1213
|
+
export function runTargetValidationSteps({ steps, target, timeout, roots, yarnCommand = process.platform === 'win32' ? 'yarn.cmd' : 'yarn', pathValue, allowContainedDependencies = false }) {
|
|
1214
|
+
const results = []
|
|
1215
|
+
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'om-harness-validation-'))
|
|
1216
|
+
const reportRoots = [...roots, tempRoot, fs.realpathSync(tempRoot)]
|
|
1217
|
+
const { env, toolReadRoots } = createMinimalValidationEnvironment(tempRoot, pathValue)
|
|
1218
|
+
const originalFingerprint = targetContentFingerprint(target)
|
|
1219
|
+
let workspace
|
|
1220
|
+
let dependencyRoot
|
|
1221
|
+
let workspaceError
|
|
1222
|
+
try {
|
|
1223
|
+
({ workspace, dependencyRoot } = prepareValidationWorkspace(target, tempRoot, { allowContainedDependencies }))
|
|
1224
|
+
} catch (error) {
|
|
1225
|
+
workspaceError = error
|
|
1226
|
+
}
|
|
1227
|
+
let before = workspace ? targetSnapshot(workspace) : new Map()
|
|
1228
|
+
let workspaceTainted = false
|
|
1229
|
+
try {
|
|
1230
|
+
for (const step of steps) {
|
|
1231
|
+
if (workspaceTainted) {
|
|
1232
|
+
results.push(skippedStep(step, 'validation workspace was tainted by an out-of-contract mutation'))
|
|
1233
|
+
continue
|
|
1234
|
+
}
|
|
1235
|
+
let execution
|
|
1236
|
+
if (workspaceError) {
|
|
1237
|
+
execution = { status: null, error: workspaceError, stdout: '', stderr: '', durationMs: 0 }
|
|
1238
|
+
} else try {
|
|
1239
|
+
const readOnlyRoots = [...toolReadRoots]
|
|
1240
|
+
if (dependencyRoot) readOnlyRoots.unshift(dependencyRoot)
|
|
1241
|
+
const invocation = sandboxedInvocation({
|
|
1242
|
+
command: yarnCommand,
|
|
1243
|
+
args: [step.command.slice('yarn '.length)],
|
|
1244
|
+
cwd: workspace,
|
|
1245
|
+
writableRoots: [workspace, tempRoot],
|
|
1246
|
+
readOnlyRoots,
|
|
1247
|
+
networkMode: 'none',
|
|
1248
|
+
env,
|
|
1249
|
+
})
|
|
1250
|
+
execution = execute(invocation.command, invocation.args, invocation.cwd, timeout, invocation.env)
|
|
1251
|
+
} catch (error) {
|
|
1252
|
+
execution = { status: null, error, stdout: '', stderr: '', durationMs: 0 }
|
|
1253
|
+
}
|
|
1254
|
+
const after = workspace ? targetSnapshot(workspace) : before
|
|
1255
|
+
const changed = [...new Set([...before.keys(), ...after.keys()])].filter((key) => before.get(key) !== after.get(key)).sort()
|
|
1256
|
+
const unsafeEntries = unsafeChangedTargetEntries(after, changed, step.command)
|
|
1257
|
+
const disallowedEntries = changed.filter((relative) => !validationOutputRoot(relative, step.command, after))
|
|
1258
|
+
const touchedOutputRoots = [...new Set(changed.map((relative) => validationOutputRoot(relative, step.command, after)).filter(Boolean))].sort()
|
|
1259
|
+
const result = stepResult(step, execution, [], reportRoots)
|
|
1260
|
+
result.resultPaths = touchedOutputRoots
|
|
1261
|
+
if (disallowedEntries.length || unsafeEntries.length) {
|
|
1262
|
+
result.status = 'fail'
|
|
1263
|
+
const diagnostics = [
|
|
1264
|
+
...(disallowedEntries.length ? [`validation mutated paths outside explicit command outputs: ${disallowedEntries.join(', ')}`] : []),
|
|
1265
|
+
...(unsafeEntries.length ? [`validation created unsafe filesystem entries: ${unsafeEntries.join(', ')}`] : []),
|
|
1266
|
+
]
|
|
1267
|
+
result.sanitizedError = sanitizeReportText(diagnostics.join('; '), reportRoots)
|
|
1268
|
+
workspaceTainted = true
|
|
1269
|
+
}
|
|
1270
|
+
if (targetContentFingerprint(target) !== originalFingerprint) {
|
|
1271
|
+
result.status = 'fail'
|
|
1272
|
+
result.sanitizedError = sanitizeReportText('original writable target changed during isolated validation', roots)
|
|
1273
|
+
workspaceTainted = true
|
|
1274
|
+
}
|
|
1275
|
+
results.push(result)
|
|
1276
|
+
before = after
|
|
1277
|
+
}
|
|
1278
|
+
return results
|
|
1279
|
+
} finally {
|
|
1280
|
+
fs.rmSync(tempRoot, { recursive: true, force: true })
|
|
1281
|
+
}
|
|
1282
|
+
}
|
|
1283
|
+
|
|
1284
|
+
export function runGeneratedTestStep({ step, target, timeout, roots, browserRuntimeResolver = resolveBrowserRuntime }) {
|
|
1285
|
+
const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'om-harness-generated-test-'))
|
|
1286
|
+
const { env, toolReadRoots } = createMinimalValidationEnvironment(tempRoot)
|
|
1287
|
+
const dependencyPath = path.join(target, 'node_modules')
|
|
1288
|
+
const readOnlyRoots = [fs.realpathSync(target), ...(fs.existsSync(dependencyPath) ? [fs.realpathSync(dependencyPath)] : []), ...toolReadRoots]
|
|
1289
|
+
let execution
|
|
1290
|
+
let before
|
|
1291
|
+
let cli
|
|
1292
|
+
let testCounts
|
|
1293
|
+
try {
|
|
1294
|
+
const artifact = path.resolve(target, step.artifact)
|
|
1295
|
+
const artifactStat = fs.lstatSync(artifact)
|
|
1296
|
+
if (!isPathInside(fs.realpathSync(target), fs.realpathSync(artifact)) || !artifactStat.isFile() || artifactStat.isSymbolicLink()) {
|
|
1297
|
+
throw new Error('generated test artifact must be a regular file inside the target')
|
|
1298
|
+
}
|
|
1299
|
+
if (!GENERATED_TEST_RUNNERS.has(step.testRunner) || !['none', 'loopback'].includes(step.network)) {
|
|
1300
|
+
throw new Error('generated test runner contract is invalid')
|
|
1301
|
+
}
|
|
1302
|
+
if (step.testRunner === 'jest' && step.network !== 'none') throw new Error('generated Jest tests must run without network access')
|
|
1303
|
+
if (step.testRunner !== 'jest' && step.network !== 'loopback') throw new Error('generated Playwright tests must run with isolated loopback only')
|
|
1304
|
+
cli = resolveTargetTestCli(target, step.testRunner)
|
|
1305
|
+
if (step.testRunner === 'playwright-browser') {
|
|
1306
|
+
const isolatedBrowserCache = path.join(tempRoot, 'browser-cache')
|
|
1307
|
+
fs.mkdirSync(isolatedBrowserCache, { mode: 0o700 })
|
|
1308
|
+
const browser = browserRuntimeResolver(target, isolatedBrowserCache)
|
|
1309
|
+
readOnlyRoots.push(browser.browserRoot)
|
|
1310
|
+
env.PLAYWRIGHT_BROWSERS_PATH = browser.isolatedCache
|
|
1311
|
+
}
|
|
1312
|
+
if (step.testRunner !== 'jest') env.PLAYWRIGHT_JSON_OUTPUT_FILE = path.join(tempRoot, 'playwright-results.json')
|
|
1313
|
+
const contract = generatedTestExecutionContract({ runner: step.testRunner, artifact: step.artifact })
|
|
1314
|
+
if (step.executor !== contract.executor || step.cliPackage !== contract.cliPackage
|
|
1315
|
+
|| JSON.stringify(step.argv) !== JSON.stringify(contract.argv) || step.command !== contract.command) {
|
|
1316
|
+
throw new Error('generated test execution differs from the fixed direct controller contract')
|
|
1317
|
+
}
|
|
1318
|
+
const runnerArgs = contract.argv.map((value) => value
|
|
1319
|
+
.replace('<resolved-cli>', cli)
|
|
1320
|
+
.replace('<isolated-temp>', tempRoot))
|
|
1321
|
+
before = targetSnapshot(target)
|
|
1322
|
+
const invocation = sandboxedInvocation({
|
|
1323
|
+
command: process.execPath,
|
|
1324
|
+
args: runnerArgs,
|
|
1325
|
+
cwd: target,
|
|
1326
|
+
writableRoots: [tempRoot],
|
|
1327
|
+
readOnlyRoots,
|
|
1328
|
+
networkMode: step.network,
|
|
1329
|
+
env,
|
|
1330
|
+
})
|
|
1331
|
+
execution = execute(invocation.command, invocation.args, invocation.cwd, timeout, invocation.env)
|
|
1332
|
+
if (execution.status === 0 && !execution.error) testCounts = generatedTestCounts(step.testRunner, tempRoot)
|
|
1333
|
+
} catch (error) {
|
|
1334
|
+
execution = { status: null, error, stdout: '', stderr: '', durationMs: 0 }
|
|
1335
|
+
}
|
|
1336
|
+
try {
|
|
1337
|
+
const after = targetSnapshot(target)
|
|
1338
|
+
const changed = before
|
|
1339
|
+
? [...new Set([...before.keys(), ...after.keys()])].filter((key) => before.get(key) !== after.get(key)).sort()
|
|
1340
|
+
: []
|
|
1341
|
+
const unsafeEntries = unsafeChangedTargetEntries(after, changed)
|
|
1342
|
+
const result = {
|
|
1343
|
+
...stepResult(step, execution, [], roots),
|
|
1344
|
+
...(typeof cli === 'string' && fs.existsSync(cli) ? { cliSha256: sha256(fs.readFileSync(cli)) } : {}),
|
|
1345
|
+
...(testCounts ? { testCounts } : {}),
|
|
1346
|
+
}
|
|
1347
|
+
if (unsafeEntries.length) {
|
|
1348
|
+
result.status = 'fail'
|
|
1349
|
+
result.sanitizedError = sanitizeReportText(`generated test created unsafe filesystem entries: ${unsafeEntries.join(', ')}`, roots)
|
|
1350
|
+
}
|
|
1351
|
+
return result
|
|
1352
|
+
} finally {
|
|
1353
|
+
fs.rmSync(tempRoot, { recursive: true, force: true })
|
|
1354
|
+
}
|
|
1355
|
+
}
|
|
1356
|
+
|
|
1357
|
+
function executedCoverage(plan, steps, results) {
|
|
1358
|
+
const deterministic = steps.find((step) => step.kind === 'deterministic')?.observedCaseIds ?? []
|
|
1359
|
+
const routing = plan.coverage.routing.map((entry) => {
|
|
1360
|
+
const executedCaseIds = sortedUnique(results.filter((result) => result.runner === entry.runner && result.evaluationKind === 'routing' && !result.sourceResult).map((result) => result.caseId))
|
|
1361
|
+
return { runner: entry.runner, expectedCaseIds: entry.expectedCaseIds, executedCaseIds, missingCaseIds: difference(entry.expectedCaseIds, executedCaseIds) }
|
|
1362
|
+
})
|
|
1363
|
+
const writableExecuted = sortedUnique(results.filter((result) => WRITABLE_KINDS.has(result.evaluationKind) && !result.sourceResult).map((result) => result.caseId))
|
|
1364
|
+
const reviewExecuted = sortedUnique(results.filter((result) => result.sourceResult).map((result) => result.caseId))
|
|
1365
|
+
const validationExecuted = steps.filter((step) => step.kind === 'validation' && step.status !== 'skipped').map((step) => `yarn ${step.id.slice('validation:'.length)}`)
|
|
1366
|
+
const targetValidationExecuted = []
|
|
1367
|
+
const targetValidationPassed = []
|
|
1368
|
+
const targetValidationFailed = []
|
|
1369
|
+
for (const caseId of plan.coverage.targetValidation.expectedCaseIds) {
|
|
1370
|
+
const caseSteps = steps.filter((step) => step.kind === 'target-validation' && step.caseId === caseId)
|
|
1371
|
+
if (caseSteps.length !== plan.coverage.targetValidation.commands.length || caseSteps.some((step) => step.status === 'skipped')) continue
|
|
1372
|
+
targetValidationExecuted.push(caseId)
|
|
1373
|
+
if (caseSteps.every((step) => step.status === 'pass')) targetValidationPassed.push(caseId)
|
|
1374
|
+
else targetValidationFailed.push(caseId)
|
|
1375
|
+
}
|
|
1376
|
+
const generatedTestExecuted = sortedUnique(steps.filter((step) => step.kind === 'generated-test' && step.status !== 'skipped').map((step) => step.caseId))
|
|
1377
|
+
const generatedTestPassed = sortedUnique(steps.filter((step) => step.kind === 'generated-test' && step.status === 'pass').map((step) => step.caseId))
|
|
1378
|
+
const generatedTestFailed = sortedUnique(steps.filter((step) => step.kind === 'generated-test' && step.status === 'fail').map((step) => step.caseId))
|
|
1379
|
+
return {
|
|
1380
|
+
deterministic: { ...plan.coverage.deterministic, executedCaseIds: deterministic, missingCaseIds: difference(plan.coverage.deterministic.expectedCaseIds, deterministic) },
|
|
1381
|
+
routing,
|
|
1382
|
+
writable: { ...plan.coverage.writable, executedCaseIds: writableExecuted, missingCaseIds: difference(plan.coverage.writable.expectedCaseIds, writableExecuted) },
|
|
1383
|
+
review: { ...plan.coverage.review, executedCaseIds: reviewExecuted, missingCaseIds: difference(plan.coverage.review.expectedCaseIds, reviewExecuted) },
|
|
1384
|
+
registry: plan.coverage.registry,
|
|
1385
|
+
validation: { ...plan.coverage.validation, executedCommands: validationExecuted, missingExecutedCommands: difference(plan.coverage.validation.expectedCommands, validationExecuted) },
|
|
1386
|
+
targetValidation: {
|
|
1387
|
+
...plan.coverage.targetValidation,
|
|
1388
|
+
executedCaseIds: targetValidationExecuted,
|
|
1389
|
+
passedCaseIds: targetValidationPassed,
|
|
1390
|
+
failedCaseIds: targetValidationFailed,
|
|
1391
|
+
missingCaseIds: difference(plan.coverage.targetValidation.expectedCaseIds, targetValidationExecuted),
|
|
1392
|
+
},
|
|
1393
|
+
generatedTests: {
|
|
1394
|
+
...plan.coverage.generatedTests,
|
|
1395
|
+
executedCaseIds: generatedTestExecuted,
|
|
1396
|
+
passedCaseIds: generatedTestPassed,
|
|
1397
|
+
failedCaseIds: generatedTestFailed,
|
|
1398
|
+
missingCaseIds: difference(plan.coverage.generatedTests.expectedCaseIds, generatedTestExecuted),
|
|
1399
|
+
},
|
|
1400
|
+
}
|
|
1401
|
+
}
|
|
1402
|
+
|
|
1403
|
+
function writeReport(root, report, roots, schema) {
|
|
1404
|
+
const directory = path.join(root, '.ai', 'harness', 'results')
|
|
1405
|
+
fs.mkdirSync(directory, { recursive: true })
|
|
1406
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, '-')
|
|
1407
|
+
const file = path.join(directory, `${stamp}-release-suite.json`)
|
|
1408
|
+
const sanitized = sanitizeObject(report, roots)
|
|
1409
|
+
const schemaErrors = validateJsonSchema(sanitized, schema)
|
|
1410
|
+
if (schemaErrors.length) throw new Error(`release report schema validation failed: ${schemaErrors.slice(0, 8).join('; ')}`)
|
|
1411
|
+
fs.writeFileSync(file, `${JSON.stringify(sanitized, null, 2)}\n`, { mode: 0o600 })
|
|
1412
|
+
return path.relative(root, file).replaceAll(path.sep, '/')
|
|
1413
|
+
}
|
|
1414
|
+
|
|
1415
|
+
function writeTargetValidationResult(root, target, caseId, sourceArtifact, commandSteps, generatedTestStep, schema) {
|
|
1416
|
+
const sourcePath = path.join(root, sourceArtifact.path)
|
|
1417
|
+
const targetFingerprint = targetContentFingerprint(target)
|
|
1418
|
+
if (targetFingerprint !== sourceArtifact.value.writable.targetFingerprint) {
|
|
1419
|
+
throw new Error('original writable target changed after its trusted oracle; target validation must run only in the isolated copy')
|
|
1420
|
+
}
|
|
1421
|
+
const record = {
|
|
1422
|
+
schemaVersion: 1,
|
|
1423
|
+
caseId,
|
|
1424
|
+
sourceResult: { path: sourceArtifact.path, sha256: sha256(fs.readFileSync(sourcePath)) },
|
|
1425
|
+
beforeValidationFingerprint: sourceArtifact.value.writable.targetFingerprint,
|
|
1426
|
+
targetFingerprint,
|
|
1427
|
+
commands: commandSteps.map((step) => ({
|
|
1428
|
+
command: step.command,
|
|
1429
|
+
status: step.status,
|
|
1430
|
+
exitStatus: step.exitStatus,
|
|
1431
|
+
durationMs: step.durationMs,
|
|
1432
|
+
})),
|
|
1433
|
+
generatedTests: generatedTestStep ? [{
|
|
1434
|
+
runner: generatedTestStep.testRunner,
|
|
1435
|
+
artifact: generatedTestStep.artifact,
|
|
1436
|
+
artifactSha256: sha256(fs.readFileSync(path.join(target, generatedTestStep.artifact))),
|
|
1437
|
+
executor: generatedTestStep.executor,
|
|
1438
|
+
cliPackage: generatedTestStep.cliPackage,
|
|
1439
|
+
cliSha256: generatedTestStep.cliSha256,
|
|
1440
|
+
argv: generatedTestStep.argv,
|
|
1441
|
+
command: generatedTestStep.command,
|
|
1442
|
+
network: generatedTestStep.network,
|
|
1443
|
+
testCounts: generatedTestStep.testCounts,
|
|
1444
|
+
status: generatedTestStep.status,
|
|
1445
|
+
exitStatus: generatedTestStep.exitStatus,
|
|
1446
|
+
durationMs: generatedTestStep.durationMs,
|
|
1447
|
+
}] : [],
|
|
1448
|
+
status: commandSteps.every((step) => step.status === 'pass') && (!generatedTestStep || generatedTestStep.status === 'pass') ? 'pass' : 'fail',
|
|
1449
|
+
}
|
|
1450
|
+
const schemaErrors = validateJsonSchema(record, schema)
|
|
1451
|
+
if (schemaErrors.length) throw new Error(`target validation result schema failed: ${schemaErrors.slice(0, 8).join('; ')}`)
|
|
1452
|
+
const directory = path.join(root, '.ai', 'harness', 'results')
|
|
1453
|
+
fs.mkdirSync(directory, { recursive: true })
|
|
1454
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, '-')
|
|
1455
|
+
const absolute = path.join(directory, `${stamp}-target-validation-${caseId}.json`)
|
|
1456
|
+
fs.writeFileSync(absolute, `${JSON.stringify(record, null, 2)}\n`, { mode: 0o600 })
|
|
1457
|
+
return { absolute, relative: path.relative(root, absolute).replaceAll(path.sep, '/'), value: record }
|
|
1458
|
+
}
|
|
1459
|
+
|
|
1460
|
+
function initialReport(plan, startedAt) {
|
|
1461
|
+
return {
|
|
1462
|
+
schemaVersion: 1,
|
|
1463
|
+
status: 'fail',
|
|
1464
|
+
startedAt,
|
|
1465
|
+
finishedAt: new Date().toISOString(),
|
|
1466
|
+
durationMs: 0,
|
|
1467
|
+
runnerPolicy: plan.runnerPolicy,
|
|
1468
|
+
catalog: plan.catalog,
|
|
1469
|
+
coverage: executedCoverage(plan, [], []),
|
|
1470
|
+
steps: [],
|
|
1471
|
+
metrics: aggregateQualityMetrics([]),
|
|
1472
|
+
violations: plan.violations.map((entry) => entry.slice(0, VIOLATION_LIMIT)),
|
|
1473
|
+
}
|
|
1474
|
+
}
|
|
1475
|
+
|
|
1476
|
+
export function main(argv = process.argv.slice(2)) {
|
|
1477
|
+
const wallStarted = Date.now()
|
|
1478
|
+
const startedAt = new Date().toISOString()
|
|
1479
|
+
let options
|
|
1480
|
+
try { options = parseArgs(argv) } catch (error) { console.error(error.message); console.error(usage()); return 2 }
|
|
1481
|
+
if (options.help) { console.log(usage()); return 0 }
|
|
1482
|
+
let root
|
|
1483
|
+
try { root = fs.realpathSync(options.root) } catch { console.error('release root is unavailable'); return 2 }
|
|
1484
|
+
const harnessDir = path.join(root, '.ai', 'harness')
|
|
1485
|
+
if (!fs.existsSync(harnessDir)) { console.error(`harness directory not found under release root`); return 2 }
|
|
1486
|
+
let cases
|
|
1487
|
+
let registry
|
|
1488
|
+
let releaseMatrix
|
|
1489
|
+
let fixtures
|
|
1490
|
+
let seeds
|
|
1491
|
+
let releaseResultSchema
|
|
1492
|
+
let targetValidationResultSchema
|
|
1493
|
+
try {
|
|
1494
|
+
cases = readJson(path.join(harnessDir, 'cases.json'))
|
|
1495
|
+
registry = readJson(path.join(harnessDir, 'validators.json'))
|
|
1496
|
+
releaseMatrix = readJson(path.join(harnessDir, 'release-matrix.json'))
|
|
1497
|
+
fixtures = readJson(path.join(harnessDir, 'fixtures', 'index.json'))
|
|
1498
|
+
seeds = readJson(path.join(harnessDir, 'fixtures', 'seeds.json'))
|
|
1499
|
+
releaseResultSchema = readJson(path.join(harnessDir, 'release-result.schema.json'))
|
|
1500
|
+
targetValidationResultSchema = readJson(path.join(harnessDir, 'target-validation-result.schema.json'))
|
|
1501
|
+
} catch {
|
|
1502
|
+
console.error('release harness inputs are missing or invalid')
|
|
1503
|
+
return 2
|
|
1504
|
+
}
|
|
1505
|
+
const hostPrerequisiteViolations = releaseHostPrerequisiteViolations(releaseMatrix)
|
|
1506
|
+
if (hostPrerequisiteViolations.length) {
|
|
1507
|
+
console.error('release host prerequisites are not satisfied:')
|
|
1508
|
+
for (const violation of hostPrerequisiteViolations) console.error(`- ${violation}`)
|
|
1509
|
+
return 2
|
|
1510
|
+
}
|
|
1511
|
+
let targetsManifest
|
|
1512
|
+
let controllerDependencyFingerprint
|
|
1513
|
+
try { controllerDependencyFingerprint = dependencyContentFingerprint(root) } catch (error) {
|
|
1514
|
+
console.error(`controller dependency verification failed: ${error.message}`)
|
|
1515
|
+
return 2
|
|
1516
|
+
}
|
|
1517
|
+
if (options.writableTargets) {
|
|
1518
|
+
try {
|
|
1519
|
+
const stat = fs.lstatSync(options.writableTargets)
|
|
1520
|
+
if (!stat.isFile() || stat.isSymbolicLink() || stat.size > RESULT_LIMIT) throw new Error('manifest must be a bounded regular file')
|
|
1521
|
+
targetsManifest = readJson(options.writableTargets)
|
|
1522
|
+
} catch (error) {
|
|
1523
|
+
console.error(`cannot read writable target manifest: ${error.message}`)
|
|
1524
|
+
return 2
|
|
1525
|
+
}
|
|
1526
|
+
} else {
|
|
1527
|
+
let prepareRoot
|
|
1528
|
+
try { prepareRoot = resolvePreparationRoot(options.prepareTargets, root) } catch (error) { console.error(error.message); return 2 }
|
|
1529
|
+
const writableIds = cases.filter((item) => WRITABLE_KINDS.has(item.evaluationKind)).map((item) => item.id)
|
|
1530
|
+
const proposedManifest = { schemaVersion: 1, targets: Object.fromEntries(writableIds.map((id) => [id, path.join(prepareRoot, id)])) }
|
|
1531
|
+
const preliminary = buildReleasePlan({ cases, registry, releaseMatrix, fixtures, seeds, targetsManifest: proposedManifest, root, primaryRunner: options.runner, portabilityRunner: options.portabilityRunner, targetsMayNotExist: true })
|
|
1532
|
+
const proposedRoots = Object.values(proposedManifest.targets)
|
|
1533
|
+
if (preliminary.violations.length) {
|
|
1534
|
+
const report = initialReport(preliminary, startedAt)
|
|
1535
|
+
report.durationMs = Date.now() - wallStarted
|
|
1536
|
+
report.finishedAt = new Date().toISOString()
|
|
1537
|
+
const reportPath = writeReport(root, report, [root, ...proposedRoots], releaseResultSchema)
|
|
1538
|
+
console.error(`Release preflight failed before target preparation; report: ${reportPath}`)
|
|
1539
|
+
return 1
|
|
1540
|
+
}
|
|
1541
|
+
try {
|
|
1542
|
+
targetsManifest = prepareWritableTargets({ root, prepareRoot, caseIds: writableIds })
|
|
1543
|
+
} catch (error) {
|
|
1544
|
+
preliminary.violations = [`automatic writable target preparation failed: ${error.message}`]
|
|
1545
|
+
const report = initialReport(preliminary, startedAt)
|
|
1546
|
+
report.durationMs = Date.now() - wallStarted
|
|
1547
|
+
report.finishedAt = new Date().toISOString()
|
|
1548
|
+
const reportPath = writeReport(root, report, [root, ...proposedRoots], releaseResultSchema)
|
|
1549
|
+
console.error(`Automatic target preparation failed; report: ${reportPath}`)
|
|
1550
|
+
return 1
|
|
1551
|
+
}
|
|
1552
|
+
}
|
|
1553
|
+
const plan = buildReleasePlan({ cases, registry, releaseMatrix, fixtures, seeds, targetsManifest, root, primaryRunner: options.runner, portabilityRunner: options.portabilityRunner })
|
|
1554
|
+
const targetRoots = Object.values(plan.targets).filter((entry) => typeof entry === 'string' && path.isAbsolute(entry)).map((entry) => path.resolve(entry))
|
|
1555
|
+
const roots = [root, ...targetRoots]
|
|
1556
|
+
if (plan.violations.length) {
|
|
1557
|
+
const report = initialReport(plan, startedAt)
|
|
1558
|
+
report.durationMs = Date.now() - wallStarted
|
|
1559
|
+
report.finishedAt = new Date().toISOString()
|
|
1560
|
+
const reportPath = writeReport(root, report, roots, releaseResultSchema)
|
|
1561
|
+
console.error(`Release preflight failed; report: ${reportPath}`)
|
|
1562
|
+
for (const violation of plan.violations) console.error(`- ${violation}`)
|
|
1563
|
+
return 1
|
|
1564
|
+
}
|
|
1565
|
+
console.log(`Runner policy: primary=${plan.runnerPolicy.primaryRunner}; portability=${plan.runnerPolicy.portabilityRunner ?? 'not requested'}`)
|
|
1566
|
+
|
|
1567
|
+
const evaluator = path.join(root, 'scripts', 'evaluate-agent-harness.mjs')
|
|
1568
|
+
const preparer = path.join(root, 'scripts', 'prepare-agent-harness-fixture.mjs')
|
|
1569
|
+
const sharedDependencyBefore = controllerDependencyFingerprint
|
|
1570
|
+
const steps = []
|
|
1571
|
+
const resultArtifacts = []
|
|
1572
|
+
const deterministicStep = plan.steps.find((step) => step.kind === 'deterministic')
|
|
1573
|
+
const validationSteps = plan.steps.filter((step) => step.kind === 'validation')
|
|
1574
|
+
const foundationTempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'om-harness-foundation-'))
|
|
1575
|
+
const { env: foundationEnv } = createMinimalValidationEnvironment(foundationTempRoot)
|
|
1576
|
+
let deterministicResult
|
|
1577
|
+
try {
|
|
1578
|
+
const deterministicExecution = execute(
|
|
1579
|
+
process.execPath,
|
|
1580
|
+
[evaluator, '--root', root, '--all'],
|
|
1581
|
+
root,
|
|
1582
|
+
options.caseTimeout * Math.max(1, plan.catalog.caseCount) + 60_000,
|
|
1583
|
+
foundationEnv,
|
|
1584
|
+
)
|
|
1585
|
+
const deterministicObserved = sortedUnique((deterministicExecution.stdout ?? '').match(/^PASS (OMH-[0-9]{3})/gm)?.map((entry) => entry.slice('PASS '.length)) ?? [])
|
|
1586
|
+
deterministicResult = stepResult(deterministicStep, deterministicExecution, [], roots)
|
|
1587
|
+
deterministicResult.observedCaseIds = deterministicObserved
|
|
1588
|
+
if (deterministicObserved.length !== plan.catalog.caseCount) {
|
|
1589
|
+
deterministicResult.status = 'fail'
|
|
1590
|
+
deterministicResult.sanitizedError = `expected ${plan.catalog.caseCount} deterministic cases, observed ${deterministicObserved.length}`
|
|
1591
|
+
}
|
|
1592
|
+
steps.push(deterministicResult)
|
|
1593
|
+
|
|
1594
|
+
if (deterministicResult.status === 'pass') {
|
|
1595
|
+
steps.push(...runTargetValidationSteps({
|
|
1596
|
+
steps: validationSteps,
|
|
1597
|
+
target: root,
|
|
1598
|
+
timeout: options.validationTimeout,
|
|
1599
|
+
roots,
|
|
1600
|
+
allowContainedDependencies: true,
|
|
1601
|
+
}))
|
|
1602
|
+
} else {
|
|
1603
|
+
for (const step of validationSteps) steps.push(skippedStep(step, 'deterministic gate failed'))
|
|
1604
|
+
}
|
|
1605
|
+
} finally {
|
|
1606
|
+
fs.rmSync(foundationTempRoot, { recursive: true, force: true })
|
|
1607
|
+
}
|
|
1608
|
+
|
|
1609
|
+
const foundationsPassed = deterministicResult.status === 'pass'
|
|
1610
|
+
&& steps.filter((step) => step.kind === 'validation').every((step) => step.status === 'pass')
|
|
1611
|
+
const modelSteps = plan.steps.filter((step) => ['routing', 'fixture', 'writable', 'target-validation', 'generated-test', 'review'].includes(step.kind))
|
|
1612
|
+
if (!foundationsPassed) {
|
|
1613
|
+
for (const step of modelSteps) steps.push(skippedStep(step, 'deterministic or validation foundation failed'))
|
|
1614
|
+
} else {
|
|
1615
|
+
for (const step of plan.steps.filter((entry) => entry.kind === 'routing')) {
|
|
1616
|
+
const before = resultFiles(root)
|
|
1617
|
+
const routingArgs = [evaluator, '--root', root, '--runner', step.runner]
|
|
1618
|
+
if (step.lane === 'primary') routingArgs.push('--all')
|
|
1619
|
+
routingArgs.push('--model', step.modelSelector, '--timeout', String(options.caseTimeout))
|
|
1620
|
+
const routingTimeout = step.expectedCaseIds.reduce(
|
|
1621
|
+
(total, caseId) => total + effectiveCaseTimeout(cases, caseId, options.caseTimeout),
|
|
1622
|
+
60_000,
|
|
1623
|
+
)
|
|
1624
|
+
const execution = execute(process.execPath, routingArgs, root, routingTimeout)
|
|
1625
|
+
const artifacts = readNewResults(root, before)
|
|
1626
|
+
resultArtifacts.push(...artifacts)
|
|
1627
|
+
steps.push(stepResult(step, execution, artifacts, roots, step.expectedCaseIds.length))
|
|
1628
|
+
}
|
|
1629
|
+
|
|
1630
|
+
for (const caseId of plan.coverage.writable.expectedCaseIds) {
|
|
1631
|
+
const fixtureStep = plan.steps.find((step) => step.id === `fixture:${caseId}`)
|
|
1632
|
+
const writableStep = plan.steps.find((step) => step.id === `writable:${caseId}`)
|
|
1633
|
+
const targetValidationSteps = plan.steps.filter((step) => step.kind === 'target-validation' && step.caseId === caseId)
|
|
1634
|
+
const generatedTestStep = plan.steps.find((step) => step.kind === 'generated-test' && step.caseId === caseId)
|
|
1635
|
+
const reviewStep = plan.steps.find((step) => step.id === `review:${caseId}`)
|
|
1636
|
+
const target = plan.targets[caseId]
|
|
1637
|
+
const fixtureExecution = execute(process.execPath, [preparer, '--case', caseId, '--target', target, '--acknowledge-writes'], root, 120_000)
|
|
1638
|
+
const fixtureResult = stepResult(fixtureStep, fixtureExecution, [], roots)
|
|
1639
|
+
steps.push(fixtureResult)
|
|
1640
|
+
if (fixtureResult.status !== 'pass') {
|
|
1641
|
+
steps.push(skippedStep(writableStep, 'fixture preparation failed'))
|
|
1642
|
+
for (const step of targetValidationSteps) steps.push(skippedStep(step, 'writable gate did not pass'))
|
|
1643
|
+
if (generatedTestStep) steps.push(skippedStep(generatedTestStep, 'writable gate did not pass'))
|
|
1644
|
+
if (reviewStep) steps.push(skippedStep(reviewStep, 'writable gate did not pass'))
|
|
1645
|
+
continue
|
|
1646
|
+
}
|
|
1647
|
+
const beforeWritable = resultFiles(root)
|
|
1648
|
+
const caseTimeout = effectiveCaseTimeout(cases, caseId, options.caseTimeout)
|
|
1649
|
+
const writableExecution = execute(process.execPath, [
|
|
1650
|
+
evaluator, '--root', root, '--runner', writableStep.runner, '--case', caseId,
|
|
1651
|
+
'--model', writableStep.modelSelector, '--timeout', String(caseTimeout),
|
|
1652
|
+
'--writable-root', target, '--acknowledge-writes',
|
|
1653
|
+
], root, caseTimeout + 120_000)
|
|
1654
|
+
const writableArtifacts = readNewResults(root, beforeWritable)
|
|
1655
|
+
resultArtifacts.push(...writableArtifacts)
|
|
1656
|
+
const writableResult = stepResult(writableStep, writableExecution, writableArtifacts, roots, 1)
|
|
1657
|
+
steps.push(writableResult)
|
|
1658
|
+
const sourceArtifact = writableArtifacts.find((entry) => entry.value.caseId === caseId && WRITABLE_KINDS.has(entry.value.evaluationKind))
|
|
1659
|
+
const targetValidationResults = runTargetValidationSteps({
|
|
1660
|
+
steps: targetValidationSteps,
|
|
1661
|
+
target,
|
|
1662
|
+
timeout: options.validationTimeout,
|
|
1663
|
+
roots,
|
|
1664
|
+
})
|
|
1665
|
+
steps.push(...targetValidationResults)
|
|
1666
|
+
const failedTargetCommands = targetValidationResults.filter((step) => step.status !== 'pass')
|
|
1667
|
+
let generatedTestResult
|
|
1668
|
+
if (generatedTestStep) {
|
|
1669
|
+
if (writableResult.status !== 'pass' || !sourceArtifact || sourceArtifact.value.status !== 'pass') {
|
|
1670
|
+
generatedTestResult = skippedStep(generatedTestStep, 'trusted writable oracle did not pass')
|
|
1671
|
+
} else if (failedTargetCommands.length) {
|
|
1672
|
+
generatedTestResult = skippedStep(generatedTestStep, 'target validation did not pass')
|
|
1673
|
+
} else {
|
|
1674
|
+
generatedTestResult = runGeneratedTestStep({
|
|
1675
|
+
step: generatedTestStep,
|
|
1676
|
+
target,
|
|
1677
|
+
timeout: options.validationTimeout,
|
|
1678
|
+
roots,
|
|
1679
|
+
})
|
|
1680
|
+
}
|
|
1681
|
+
steps.push(generatedTestResult)
|
|
1682
|
+
}
|
|
1683
|
+
if (!reviewStep) continue
|
|
1684
|
+
if (writableResult.status !== 'pass' || !sourceArtifact || sourceArtifact.value.status !== 'pass') {
|
|
1685
|
+
steps.push(skippedStep(reviewStep, 'writable gate did not pass'))
|
|
1686
|
+
continue
|
|
1687
|
+
}
|
|
1688
|
+
if (failedTargetCommands.length) {
|
|
1689
|
+
steps.push(skippedStep(reviewStep, `target validation failed: ${failedTargetCommands.map((step) => step.command).join(', ')}`))
|
|
1690
|
+
continue
|
|
1691
|
+
}
|
|
1692
|
+
if (generatedTestResult && generatedTestResult.status !== 'pass') {
|
|
1693
|
+
steps.push(skippedStep(reviewStep, 'generated test execution did not pass'))
|
|
1694
|
+
continue
|
|
1695
|
+
}
|
|
1696
|
+
let targetValidationArtifact
|
|
1697
|
+
try {
|
|
1698
|
+
targetValidationArtifact = writeTargetValidationResult(
|
|
1699
|
+
root, target, caseId, sourceArtifact, targetValidationResults, generatedTestResult, targetValidationResultSchema,
|
|
1700
|
+
)
|
|
1701
|
+
} catch (error) {
|
|
1702
|
+
const finalValidation = targetValidationResults.at(-1)
|
|
1703
|
+
finalValidation.status = 'fail'
|
|
1704
|
+
finalValidation.sanitizedError = sanitizeReportText(`validation attestation failed: ${error.message}`, roots)
|
|
1705
|
+
steps.push(skippedStep(reviewStep, 'target validation attestation failed'))
|
|
1706
|
+
continue
|
|
1707
|
+
}
|
|
1708
|
+
const beforeReview = resultFiles(root)
|
|
1709
|
+
const reviewExecution = execute(process.execPath, [
|
|
1710
|
+
evaluator, '--root', root, '--runner', reviewStep.runner,
|
|
1711
|
+
'--model', reviewStep.modelSelector, '--timeout', String(caseTimeout),
|
|
1712
|
+
'--judge-writable-result', path.join(root, sourceArtifact.path), '--writable-root', target,
|
|
1713
|
+
'--judge-validation-result', targetValidationArtifact.absolute,
|
|
1714
|
+
], root, caseTimeout + 60_000)
|
|
1715
|
+
const reviewArtifacts = readNewResults(root, beforeReview)
|
|
1716
|
+
resultArtifacts.push(...reviewArtifacts)
|
|
1717
|
+
steps.push(stepResult(reviewStep, reviewExecution, reviewArtifacts, roots, 1))
|
|
1718
|
+
}
|
|
1719
|
+
}
|
|
1720
|
+
|
|
1721
|
+
if (sharedDependencyBefore) {
|
|
1722
|
+
let sharedDependencyAfter
|
|
1723
|
+
let dependencyError
|
|
1724
|
+
try { sharedDependencyAfter = dependencyContentFingerprint(root) } catch (error) { dependencyError = error }
|
|
1725
|
+
steps.push({
|
|
1726
|
+
id: 'shared-dependencies:unchanged',
|
|
1727
|
+
kind: 'dependency-guard',
|
|
1728
|
+
status: !dependencyError && sharedDependencyAfter === sharedDependencyBefore ? 'pass' : 'fail',
|
|
1729
|
+
exitStatus: null,
|
|
1730
|
+
durationMs: 0,
|
|
1731
|
+
resultPaths: [],
|
|
1732
|
+
...(!dependencyError && sharedDependencyAfter === sharedDependencyBefore ? {} : {
|
|
1733
|
+
sanitizedError: dependencyError
|
|
1734
|
+
? sanitizeReportText(`shared dependency verification failed: ${dependencyError.message}`, roots)
|
|
1735
|
+
: 'shared dependency tree changed during release execution; restore dependencies and rerun with fresh targets',
|
|
1736
|
+
}),
|
|
1737
|
+
})
|
|
1738
|
+
}
|
|
1739
|
+
|
|
1740
|
+
const values = resultArtifacts.map((entry) => entry.value)
|
|
1741
|
+
const coverage = executedCoverage(plan, steps, values)
|
|
1742
|
+
const executionViolations = steps
|
|
1743
|
+
.filter((step) => step.status !== 'pass')
|
|
1744
|
+
.map((step) => `${step.id} ${step.status}${step.sanitizedError ? `: ${step.sanitizedError}` : ''}`)
|
|
1745
|
+
.slice(0, 256)
|
|
1746
|
+
const coverageIncomplete = coverage.deterministic.missingCaseIds.length
|
|
1747
|
+
|| coverage.routing.some((entry) => entry.missingCaseIds.length)
|
|
1748
|
+
|| coverage.writable.missingCaseIds.length
|
|
1749
|
+
|| coverage.review.missingCaseIds.length
|
|
1750
|
+
|| coverage.validation.missingExecutedCommands.length
|
|
1751
|
+
|| coverage.targetValidation.missingCaseIds.length
|
|
1752
|
+
|| coverage.targetValidation.failedCaseIds.length
|
|
1753
|
+
|| coverage.generatedTests.missingCaseIds.length
|
|
1754
|
+
|| coverage.generatedTests.failedCaseIds.length
|
|
1755
|
+
const status = executionViolations.length === 0 && !coverageIncomplete ? 'pass' : 'fail'
|
|
1756
|
+
const report = {
|
|
1757
|
+
schemaVersion: 1,
|
|
1758
|
+
status,
|
|
1759
|
+
startedAt,
|
|
1760
|
+
finishedAt: new Date().toISOString(),
|
|
1761
|
+
durationMs: Date.now() - wallStarted,
|
|
1762
|
+
runnerPolicy: plan.runnerPolicy,
|
|
1763
|
+
catalog: plan.catalog,
|
|
1764
|
+
coverage,
|
|
1765
|
+
steps,
|
|
1766
|
+
metrics: aggregateQualityMetrics(values),
|
|
1767
|
+
violations: executionViolations.map((entry) => entry.slice(0, VIOLATION_LIMIT)),
|
|
1768
|
+
}
|
|
1769
|
+
const reportPath = writeReport(root, report, roots, releaseResultSchema)
|
|
1770
|
+
console.log(`${status === 'pass' ? 'PASS' : 'FAIL'} release suite — ${reportPath}`)
|
|
1771
|
+
return status === 'pass' ? 0 : 1
|
|
1772
|
+
}
|
|
1773
|
+
|
|
1774
|
+
if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url) process.exitCode = main()
|