@1aboveio/engineering-skills 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -2
- package/package.json +34 -4
- package/skills/better-review/LICENSE +3 -0
- package/skills/better-review/SKILL.md +249 -0
- package/skills/better-review/agents/openai.yaml +9 -0
- package/skills/better-review/references/domain-hazards.md +53 -0
- package/skills/better-review/references/passes.md +147 -0
- package/skills/better-review/references/result.md +151 -0
- package/skills/better-review/release-check.mjs +93 -0
- package/skills/better-review/scripts/pr-context.mjs +212 -0
- package/skills/better-review/scripts/pr-context.test.mjs +168 -0
- package/skills/better-review/scripts/rereview-scope.mjs +503 -0
- package/skills/better-review/scripts/rereview-scope.test.mjs +587 -0
- package/skills/better-review/scripts/scan-diff.mjs +365 -0
- package/skills/better-review/scripts/scan-diff.test.mjs +414 -0
- package/skills/better-review/scripts/vendor-sync.test.mjs +96 -0
- package/skills/e2e-test/LICENSE +3 -0
- package/skills/e2e-test/SKILL.md +98 -0
- package/skills/e2e-test/assets/ci-gates.cloudbuild.yaml +24 -0
- package/skills/e2e-test/references/authoring/auth-flows.md +159 -0
- package/skills/e2e-test/references/authoring/playwright-config.md +73 -0
- package/skills/e2e-test/references/authoring/playwright-patterns.md +144 -0
- package/skills/e2e-test/references/ci-integration.md +25 -0
- package/skills/e2e-test/references/ci-playwright-container.md +290 -0
- package/skills/e2e-test/references/presentation-sweep.md +104 -0
- package/skills/e2e-test/references/reviewing.md +41 -0
- package/skills/e2e-test/references/route-manifest.md +51 -0
- package/skills/e2e-test/release-check.mjs +93 -0
- package/skills/e2e-test/scripts/detect-routes-nextjs.mjs +200 -0
- package/skills/e2e-test/scripts/presentation-checks.mjs +171 -0
- package/skills/e2e-test/scripts/presentation-checks.test.mjs +106 -0
- package/skills/e2e-test/scripts/presentation-perceivability.mjs +179 -0
- package/skills/e2e-test/scripts/presentation-perceivability.test.mjs +117 -0
- package/skills/e2e-test/scripts/presentation-reachability.mjs +154 -0
- package/skills/e2e-test/scripts/presentation-reachability.test.mjs +69 -0
- package/skills/e2e-test/scripts/presentation-render-health.mjs +141 -0
- package/skills/e2e-test/scripts/presentation-render-health.test.mjs +98 -0
- package/skills/e2e-test/scripts/presentation-sweep.mjs +147 -0
- package/skills/e2e-test/scripts/presentation-sweep.test.mjs +71 -0
- package/skills/e2e-test/scripts/presentation-temporal.mjs +127 -0
- package/skills/e2e-test/scripts/presentation-temporal.test.mjs +59 -0
- package/skills/e2e-test/scripts/project-route-manifest.mjs +82 -0
- package/skills/e2e-test/scripts/project-route-manifest.test.mjs +76 -0
- package/skills/e2e-test/scripts/validate-manifest.mjs +106 -0
- package/skills/engineering-runtime/LICENSE +3 -0
- package/skills/engineering-runtime/SKILL.md +11 -0
- package/skills/engineering-runtime/agents/openai.yaml +6 -0
- package/skills/engineering-runtime/coherence/workflow.json +597 -0
- package/skills/engineering-runtime/engineering-runtime.test.mjs +68 -0
- package/skills/engineering-runtime/release-check.mjs +93 -0
- package/skills/engineering-runtime/scripts/exact-head-artifact.mjs +131 -0
- package/skills/engineering-runtime/scripts/head-check-set.mjs +398 -0
- package/skills/engineering-runtime/scripts/invocation-policy.mjs +187 -0
- package/skills/engineering-runtime/scripts/invocation-policy.test.mjs +95 -0
- package/skills/engineering-runtime/scripts/main-module.mjs +79 -0
- package/skills/engineering-runtime/scripts/main-module.test.mjs +288 -0
- package/skills/engineering-runtime/scripts/mergify-yaml.mjs +11 -0
- package/skills/engineering-runtime/scripts/package-lock.json +43 -0
- package/skills/engineering-runtime/scripts/package.json +10 -0
- package/skills/engineering-runtime/scripts/required-check-plan.mjs +223 -0
- package/skills/engineering-runtime/scripts/required-check-plan.test.mjs +96 -0
- package/skills/engineering-runtime/scripts/review-completion.mjs +56 -0
- package/skills/engineering-runtime/scripts/workflow-coherence.mjs +595 -0
- package/skills/engineering-runtime/scripts/workflow-policy.mjs +189 -0
- package/skills/engineering-runtime/scripts/yaml-mapping.mjs +18 -0
- package/skills/ensure-coverage/LICENSE +3 -0
- package/skills/ensure-coverage/SKILL.md +129 -0
- package/skills/ensure-coverage/references/breadth/coverage-ledger.md +92 -0
- package/skills/ensure-coverage/references/breadth/inventory-contract.md +85 -0
- package/skills/ensure-coverage/references/breadth/surface-baseline.md +44 -0
- package/skills/ensure-coverage/references/breadth/surface-discovery.md +16 -0
- package/skills/ensure-coverage/references/depth/characterization.md +68 -0
- package/skills/ensure-coverage/references/depth/coverage.config.example.json +19 -0
- package/skills/ensure-coverage/references/depth/grading.md +35 -0
- package/skills/ensure-coverage/references/depth/mock-policy.md +57 -0
- package/skills/ensure-coverage/references/depth/test-smells.md +39 -0
- package/skills/ensure-coverage/references/enforcement/ci-contract.md +44 -0
- package/skills/ensure-coverage/references/examples/coverage-ledger.md +110 -0
- package/skills/ensure-coverage/references/examples/refund-flow.md +35 -0
- package/skills/ensure-coverage/references/presentation/axis.md +64 -0
- package/skills/ensure-coverage/references/process/audit-mode.md +27 -0
- package/skills/ensure-coverage/references/process/output-template.md +139 -0
- package/skills/ensure-coverage/references/process/test-strategy.md +56 -0
- package/skills/ensure-coverage/release-check.mjs +93 -0
- package/skills/ensure-coverage/schemas/coverage-config.schema.json +67 -0
- package/skills/ensure-coverage/schemas/coverage-file.schema.json +140 -0
- package/skills/ensure-coverage/scripts/README.md +99 -0
- package/skills/ensure-coverage/scripts/adapters/nextjs-inventory.mjs +182 -0
- package/skills/ensure-coverage/scripts/adapters/nextjs-inventory.test.mjs +96 -0
- package/skills/ensure-coverage/scripts/check-quarantine-expiry.mjs +105 -0
- package/skills/ensure-coverage/scripts/check-quarantine-expiry.test.mjs +47 -0
- package/skills/ensure-coverage/scripts/ci-audit.mjs +137 -0
- package/skills/ensure-coverage/scripts/ci-audit.test.mjs +86 -0
- package/skills/ensure-coverage/scripts/coverage-ledger.mjs +674 -0
- package/skills/ensure-coverage/scripts/coverage-ledger.test.mjs +447 -0
- package/skills/ensure-coverage/scripts/design-parity.mjs +591 -0
- package/skills/ensure-coverage/scripts/design-parity.test.mjs +552 -0
- package/skills/ensure-coverage/scripts/lint-tests.mjs +282 -0
- package/skills/ensure-coverage/scripts/lint-tests.test.mjs +289 -0
- package/skills/ensure-coverage/scripts/mock-policy-config.mjs +176 -0
- package/skills/ensure-coverage/scripts/mock-policy-config.test.mjs +119 -0
- package/skills/ensure-coverage/scripts/package.json +19 -0
- package/skills/golden-path/LICENSE +3 -0
- package/skills/golden-path/SKILL.md +230 -0
- package/skills/golden-path/agents/openai.yaml +3 -0
- package/skills/golden-path/assets/golden-paths.example.json +57 -0
- package/skills/golden-path/references/contract.md +239 -0
- package/skills/golden-path/references/migration.md +111 -0
- package/skills/golden-path/release-check.mjs +93 -0
- package/skills/golden-path/scripts/golden-path-core.mjs +1779 -0
- package/skills/golden-path/scripts/golden-path.mjs +6 -0
- package/skills/golden-path/scripts/golden-path.test.mjs +1457 -0
- package/skills/harness-runtime/LICENSE +3 -0
- package/skills/harness-runtime/SKILL.md +19 -0
- package/skills/harness-runtime/agents/openai.yaml +6 -0
- package/skills/harness-runtime/bin/discover-models.mjs +4 -0
- package/skills/harness-runtime/bin/model-catalog.mjs +4 -0
- package/skills/harness-runtime/contracts.md +15 -0
- package/skills/harness-runtime/discover-models.mjs +392 -0
- package/skills/harness-runtime/discover-models.test.mjs +363 -0
- package/skills/harness-runtime/fixtures/native-question-schemas.json +33 -0
- package/skills/harness-runtime/fixtures/question-responses.json +54 -0
- package/skills/harness-runtime/harness-runtime.test.mjs +967 -0
- package/skills/harness-runtime/index.mjs +767 -0
- package/skills/harness-runtime/model-catalog-seed.test.mjs +829 -0
- package/skills/harness-runtime/model-catalog.mjs +787 -0
- package/skills/harness-runtime/model-catalog.test.mjs +366 -0
- package/skills/harness-runtime/native-question-contracts.md +37 -0
- package/skills/harness-runtime/references/model-catalog-seed.json +408 -0
- package/skills/harness-runtime/references/model-catalog.md +57 -0
- package/skills/harness-runtime/release-check.mjs +93 -0
- package/skills/resolve-issues/LICENSE +3 -0
- package/skills/resolve-issues/SKILL.md +250 -0
- package/skills/resolve-issues/agents/openai.yaml +9 -0
- package/skills/resolve-issues/plugin-hooks.json +17 -0
- package/skills/resolve-issues/references/canary.md +28 -0
- package/skills/resolve-issues/references/ci.md +105 -0
- package/skills/resolve-issues/references/done.md +25 -0
- package/skills/resolve-issues/references/dropbox.md +90 -0
- package/skills/resolve-issues/references/expire.md +318 -0
- package/skills/resolve-issues/references/exploration.md +202 -0
- package/skills/resolve-issues/references/implementation.md +303 -0
- package/skills/resolve-issues/references/preflight.md +445 -0
- package/skills/resolve-issues/references/review.md +228 -0
- package/skills/resolve-issues/references/shared-modules.md +44 -0
- package/skills/resolve-issues/references/task-plan.md +264 -0
- package/skills/resolve-issues/release-check.mjs +93 -0
- package/skills/resolve-issues/scripts/cleanup-runs.mjs +364 -0
- package/skills/resolve-issues/scripts/cleanup-runs.test.mjs +280 -0
- package/skills/resolve-issues/scripts/discover-models.mjs +9 -0
- package/skills/resolve-issues/scripts/dropbox.mjs +473 -0
- package/skills/resolve-issues/scripts/dropbox.test.mjs +346 -0
- package/skills/resolve-issues/scripts/findings-reuse.mjs +98 -0
- package/skills/resolve-issues/scripts/findings-reuse.test.mjs +88 -0
- package/skills/resolve-issues/scripts/github-reviews.mjs +384 -0
- package/skills/resolve-issues/scripts/github-reviews.test.mjs +264 -0
- package/skills/resolve-issues/scripts/install-hooks.mjs +135 -0
- package/skills/resolve-issues/scripts/install-hooks.test.mjs +79 -0
- package/skills/resolve-issues/scripts/issue-tree.mjs +162 -0
- package/skills/resolve-issues/scripts/issue-tree.test.mjs +107 -0
- package/skills/resolve-issues/scripts/manual-invocation.test.mjs +14 -0
- package/skills/resolve-issues/scripts/model-catalog.mjs +9 -0
- package/skills/resolve-issues/scripts/next-actions-capacity.test.mjs +149 -0
- package/skills/resolve-issues/scripts/next-actions.mjs +856 -0
- package/skills/resolve-issues/scripts/next-actions.test.mjs +915 -0
- package/skills/resolve-issues/scripts/plan-sync.mjs +269 -0
- package/skills/resolve-issues/scripts/plan-sync.test.mjs +156 -0
- package/skills/resolve-issues/scripts/preflight-models.mjs +736 -0
- package/skills/resolve-issues/scripts/preflight-models.test.mjs +448 -0
- package/skills/resolve-issues/scripts/ready-check.mjs +185 -0
- package/skills/resolve-issues/scripts/ready-check.test.mjs +217 -0
- package/skills/resolve-issues/scripts/rehydrate.mjs +98 -0
- package/skills/resolve-issues/scripts/rehydrate.test.mjs +91 -0
- package/skills/resolve-issues/scripts/review-verdict.mjs +509 -0
- package/skills/resolve-issues/scripts/review-verdict.test.mjs +1060 -0
- package/skills/resolve-issues/scripts/size-check.mjs +492 -0
- package/skills/resolve-issues/scripts/size-check.test.mjs +458 -0
- package/skills/resolve-release/LICENSE +3 -0
- package/skills/resolve-release/SKILL.md +215 -0
- package/skills/resolve-release/agents/openai.yaml +8 -0
- package/skills/resolve-release/flows/cloud-run-canary.md +13 -0
- package/skills/resolve-release/flows/cloud-run.md +7 -0
- package/skills/resolve-release/flows/root-tag-npm.md +13 -0
- package/skills/resolve-release/flows/workspace-npm.md +22 -0
- package/skills/resolve-release/references/canary.md +51 -0
- package/skills/resolve-release/references/candidate.md +41 -0
- package/skills/resolve-release/references/cut.md +131 -0
- package/skills/resolve-release/references/dark-launch.md +66 -0
- package/skills/resolve-release/references/flows.md +150 -0
- package/skills/resolve-release/references/preflight.md +154 -0
- package/skills/resolve-release/references/prepared-metadata.md +260 -0
- package/skills/resolve-release/references/promotion.md +82 -0
- package/skills/resolve-release/references/publish.md +232 -0
- package/skills/resolve-release/references/workspace-npm.md +103 -0
- package/skills/resolve-release/release-check.mjs +93 -0
- package/skills/resolve-release/scripts/apply.mjs +212 -0
- package/skills/resolve-release/scripts/apply.test.mjs +99 -0
- package/skills/resolve-release/scripts/dark-flags.mjs +134 -0
- package/skills/resolve-release/scripts/dark-flags.test.mjs +100 -0
- package/skills/resolve-release/scripts/flow.mjs +500 -0
- package/skills/resolve-release/scripts/flow.test.mjs +340 -0
- package/skills/resolve-release/scripts/green-head.mjs +231 -0
- package/skills/resolve-release/scripts/green-head.test.mjs +101 -0
- package/skills/resolve-release/scripts/npm-version-absence.mjs +138 -0
- package/skills/resolve-release/scripts/npm-version-absence.test.mjs +426 -0
- package/skills/resolve-release/scripts/plan-npm.test.mjs +659 -0
- package/skills/resolve-release/scripts/plan-toml-boundary.test.mjs +317 -0
- package/skills/resolve-release/scripts/plan-uv.test.mjs +291 -0
- package/skills/resolve-release/scripts/plan.mjs +806 -0
- package/skills/resolve-release/scripts/plan.test.mjs +428 -0
- package/skills/rush-issues/LICENSE +3 -0
- package/skills/rush-issues/SKILL.md +58 -0
- package/skills/rush-issues/agents/openai.yaml +8 -0
- package/skills/rush-issues/rehydrate.mjs +10 -0
- package/skills/rush-issues/release-check.mjs +93 -0
- package/skills/rush-release/LICENSE +3 -0
- package/skills/rush-release/SKILL.md +35 -0
- package/skills/rush-release/agents/openai.yaml +8 -0
- package/skills/rush-release/release-check.mjs +93 -0
- package/skills/test-quality/LICENSE +3 -0
- package/skills/test-quality/SKILL.md +79 -0
- package/skills/test-quality/references/audit.md +80 -0
- package/skills/test-quality/references/lint.md +24 -0
- package/skills/test-quality/references/principles.md +143 -0
- package/skills/test-quality/references/testing-conventions.md +9 -0
- package/skills/test-quality/release-check.mjs +93 -0
- package/skills/test-quality/scripts/lint-tests.mjs +574 -0
- package/skills/test-quality/scripts/lint-tests.test.mjs +591 -0
- package/skills/test-quality/scripts/mock-policy-config.mjs +176 -0
- package/skills/test-quality/scripts/mock-policy-config.test.mjs +119 -0
- package/skills/test-quality/scripts/persist-conventions-core.mjs +117 -0
- package/skills/test-quality/scripts/persist-conventions.mjs +17 -0
- package/skills/test-quality/scripts/persist-conventions.test.mjs +104 -0
|
@@ -0,0 +1,448 @@
|
|
|
1
|
+
import { test } from 'node:test'
|
|
2
|
+
import assert from 'node:assert/strict'
|
|
3
|
+
import { spawnSync } from 'node:child_process'
|
|
4
|
+
import { fileURLToPath } from 'node:url'
|
|
5
|
+
|
|
6
|
+
import {
|
|
7
|
+
recommendImplementer,
|
|
8
|
+
rankReviewers,
|
|
9
|
+
fillExplorer,
|
|
10
|
+
fillDiagnoser,
|
|
11
|
+
slotPoints,
|
|
12
|
+
buildAsk,
|
|
13
|
+
buildPreflightModels,
|
|
14
|
+
TEAM_REVIEWER_POLICY,
|
|
15
|
+
} from './preflight-models.mjs'
|
|
16
|
+
|
|
17
|
+
const points = [
|
|
18
|
+
{ id: 'openai/gpt-6-astra', effort: 'high', tier: 'frontier', family: 'openai', key: 'openai/gpt-6-astra@high' },
|
|
19
|
+
{ id: 'openai/gpt-5.6-sol', effort: 'high', tier: 'frontier', family: 'openai', key: 'openai/gpt-5.6-sol@high' },
|
|
20
|
+
{ id: 'openai/gpt-6.1-sol', effort: 'high', tier: 'frontier', family: 'openai', key: 'openai/gpt-6.1-sol@high' },
|
|
21
|
+
{ id: 'openai/gpt-5.5', effort: 'high', tier: 'deep-reasoner', family: 'openai', key: 'openai/gpt-5.5@high' },
|
|
22
|
+
{ id: 'openai/gpt-5.6-terra', effort: 'high', tier: 'balanced-coder', family: 'openai', key: 'openai/gpt-5.6-terra@high' },
|
|
23
|
+
{ id: 'openai/gpt-6-luna', effort: 'high', tier: 'balanced-coder', family: 'openai', key: 'openai/gpt-6-luna@high' },
|
|
24
|
+
{ id: 'xiaomi/mimo-v2.6-pro', effort: 'high', tier: 'frontier', family: 'xiaomi', key: 'xiaomi/mimo-v2.6-pro@high' },
|
|
25
|
+
{ id: 'xai/grok-4.7', effort: 'high', tier: 'frontier', family: 'xai', key: 'xai/grok-4.7@high' },
|
|
26
|
+
{ id: 'xai/grok-4.6', effort: 'high', tier: 'deep-reasoner', family: 'xai', key: 'xai/grok-4.6@high' },
|
|
27
|
+
{ id: 'xai/grok-4.5', effort: 'high', tier: 'balanced-coder', family: 'xai', key: 'xai/grok-4.5@high' },
|
|
28
|
+
{ id: 'anthropic/claude-opus', effort: 'high', tier: 'frontier', family: 'anthropic', key: 'anthropic/claude-opus@high' },
|
|
29
|
+
]
|
|
30
|
+
|
|
31
|
+
test('routine risk recommends a balanced-coder, not frontier', () => {
|
|
32
|
+
const pick = recommendImplementer(points, 'routine')
|
|
33
|
+
assert.equal(pick.tier, 'balanced-coder')
|
|
34
|
+
assert.ok(['xai/grok-4.5', 'openai/gpt-5.6-terra'].includes(pick.id))
|
|
35
|
+
})
|
|
36
|
+
|
|
37
|
+
test('routine risk walks the team ladder: gpt-6.1-sol@high, gpt-6-luna@high, grok-4.6@high, then mimo-v2.6-pro@high', () => {
|
|
38
|
+
const pick = recommendImplementer(points, 'routine', 'pi')
|
|
39
|
+
assert.equal(pick.key, 'openai/gpt-6.1-sol@high')
|
|
40
|
+
assert.equal(pick.tier, 'frontier')
|
|
41
|
+
assert.equal(rankReviewers(points, pick, 'pi').fill.key, 'openai/gpt-6-astra@medium')
|
|
42
|
+
const withoutSol = points.filter((p) => p.id !== 'openai/gpt-6.1-sol')
|
|
43
|
+
const second = recommendImplementer(withoutSol, 'routine', 'pi')
|
|
44
|
+
assert.equal(second.key, 'openai/gpt-6-luna@high')
|
|
45
|
+
assert.equal(second.tier, 'balanced-coder')
|
|
46
|
+
assert.equal(rankReviewers(points, second, 'pi').fill.key, 'openai/gpt-6-astra@medium')
|
|
47
|
+
const withoutLuna = withoutSol.filter((p) => p.id !== 'openai/gpt-6-luna')
|
|
48
|
+
const third = recommendImplementer(withoutLuna, 'routine', 'pi')
|
|
49
|
+
assert.equal(third.key, 'xai/grok-4.6@high')
|
|
50
|
+
// a cross-family implementation is still reviewed by the team primary, a distinct id and family
|
|
51
|
+
const grokReview = rankReviewers(points, third, 'pi')
|
|
52
|
+
assert.equal(grokReview.fill.key, 'openai/gpt-6-astra@medium')
|
|
53
|
+
assert.equal(grokReview.reducedDiversity, false)
|
|
54
|
+
const withoutGrok46 = withoutLuna.filter((p) => p.id !== 'xai/grok-4.6')
|
|
55
|
+
const fourth = recommendImplementer(withoutGrok46, 'routine', 'pi')
|
|
56
|
+
assert.equal(fourth.key, 'xiaomi/mimo-v2.6-pro@high')
|
|
57
|
+
assert.equal(rankReviewers(points, fourth, 'pi').fill.key, 'openai/gpt-6-astra@medium')
|
|
58
|
+
// past the ladder, the tier order picks a balanced-coder
|
|
59
|
+
const withoutMimo = withoutGrok46.filter((p) => p.id !== 'xiaomi/mimo-v2.6-pro')
|
|
60
|
+
assert.equal(recommendImplementer(withoutMimo, 'routine', 'pi').tier, 'balanced-coder')
|
|
61
|
+
})
|
|
62
|
+
|
|
63
|
+
test('codex routine ladder is gpt-6.1-sol@high, then gpt-6-luna@high', () => {
|
|
64
|
+
const codex = [
|
|
65
|
+
{ id: 'gpt-6.1-sol', effort: 'high', tier: 'frontier', family: 'openai', key: 'gpt-6.1-sol@high' },
|
|
66
|
+
{ id: 'gpt-6-luna', effort: 'high', tier: 'balanced-coder', family: 'openai', key: 'gpt-6-luna@high' },
|
|
67
|
+
]
|
|
68
|
+
assert.equal(recommendImplementer(codex, 'routine', 'codex').key, 'gpt-6.1-sol@high')
|
|
69
|
+
assert.equal(recommendImplementer(codex.slice(1), 'routine', 'codex').key, 'gpt-6-luna@high')
|
|
70
|
+
})
|
|
71
|
+
|
|
72
|
+
test('escalation fix workers walk the high-risk ladder', () => {
|
|
73
|
+
assert.equal(fillDiagnoser(points, 'pi').fill.key, 'openai/gpt-6.1-sol@high')
|
|
74
|
+
const withoutSol61 = points.filter((p) => p.id !== 'openai/gpt-6.1-sol')
|
|
75
|
+
assert.equal(fillDiagnoser(withoutSol61, 'pi').fill.key, 'openai/gpt-6-astra@high')
|
|
76
|
+
const withoutAstra = withoutSol61.filter((p) => p.id !== 'openai/gpt-6-astra')
|
|
77
|
+
assert.equal(fillDiagnoser(withoutAstra, 'pi').fill.key, 'openai/gpt-5.6-sol@high')
|
|
78
|
+
const withoutSol56 = withoutAstra.filter((p) => p.id !== 'openai/gpt-5.6-sol')
|
|
79
|
+
assert.equal(fillDiagnoser(withoutSol56, 'pi').fill.key, 'xai/grok-4.7@high')
|
|
80
|
+
})
|
|
81
|
+
|
|
82
|
+
test('explorer takes gpt-6.1-sol@medium first, then gpt-6-luna, grok-4.6, mimo-v2.6-pro (all @high), then the balanced-coder tier', () => {
|
|
83
|
+
const explorer = fillExplorer(points, 'pi')
|
|
84
|
+
assert.equal(explorer.reason, 'team-default')
|
|
85
|
+
assert.equal(explorer.fill.key, 'openai/gpt-6.1-sol@medium')
|
|
86
|
+
assert.equal(explorer.fill.effort, 'medium')
|
|
87
|
+
assert.equal(explorer.fill.source, 'team-explorer-policy')
|
|
88
|
+
const withoutSol = points.filter((p) => p.id !== 'openai/gpt-6.1-sol')
|
|
89
|
+
const second = fillExplorer(withoutSol, 'pi')
|
|
90
|
+
assert.equal(second.reason, 'team-default')
|
|
91
|
+
assert.equal(second.fill.key, 'openai/gpt-6-luna@high')
|
|
92
|
+
const withoutLuna = withoutSol.filter((p) => p.id !== 'openai/gpt-6-luna')
|
|
93
|
+
assert.equal(fillExplorer(withoutLuna, 'pi').fill.key, 'xai/grok-4.6@high')
|
|
94
|
+
const withoutGrok46 = withoutLuna.filter((p) => p.id !== 'xai/grok-4.6')
|
|
95
|
+
assert.equal(fillExplorer(withoutGrok46, 'pi').fill.key, 'xiaomi/mimo-v2.6-pro@high')
|
|
96
|
+
const noLadder = withoutGrok46.filter((p) => p.id !== 'xiaomi/mimo-v2.6-pro')
|
|
97
|
+
const tierFill = fillExplorer(noLadder, 'pi')
|
|
98
|
+
assert.equal(tierFill.reason, 'balanced-coder')
|
|
99
|
+
assert.equal(tierFill.fill.tier, 'balanced-coder')
|
|
100
|
+
})
|
|
101
|
+
|
|
102
|
+
test('a discovered gpt-6.1-sol fills the explorer at medium, not at its @high implementer rung', () => {
|
|
103
|
+
const onlySolHigh = [points.find((p) => p.id === 'openai/gpt-6.1-sol'), points.find((p) => p.id === 'openai/gpt-5.6-terra')]
|
|
104
|
+
assert.equal(fillExplorer(onlySolHigh, 'pi').fill.key, 'openai/gpt-6.1-sol@medium')
|
|
105
|
+
})
|
|
106
|
+
|
|
107
|
+
test('the explorer takes gpt-6.1-sol@medium while the routine implementer takes gpt-6.1-sol@high', () => {
|
|
108
|
+
assert.equal(fillExplorer(points, 'pi').fill.key, 'openai/gpt-6.1-sol@medium')
|
|
109
|
+
assert.equal(recommendImplementer(points, 'routine', 'pi').key, 'openai/gpt-6.1-sol@high')
|
|
110
|
+
})
|
|
111
|
+
|
|
112
|
+
test('an undiscovered explorer rung falls through to the next discovered point', () => {
|
|
113
|
+
// Discovery yields neither gpt-6.1-sol nor gpt-6-luna, so the explorer lands on the balanced-coder tier.
|
|
114
|
+
const tiered = [
|
|
115
|
+
{ id: 'gpt-6-astra', tiers: { high: 'frontier' }, source: 'team-seed' },
|
|
116
|
+
{ id: 'gpt-5.6-sol', tiers: { high: 'frontier' }, source: 'team-seed' },
|
|
117
|
+
{ id: 'gpt-5.6-terra', tiers: { high: 'balanced-coder' }, source: 'team-seed' },
|
|
118
|
+
]
|
|
119
|
+
const result = buildPreflightModels({
|
|
120
|
+
harness: 'codex',
|
|
121
|
+
risk: 'routine',
|
|
122
|
+
discoverFn: () => ({
|
|
123
|
+
harness: 'codex',
|
|
124
|
+
completeness: 'full',
|
|
125
|
+
discovered: tiered.map(({ id }) => ({ id, source: 'current-turn' })),
|
|
126
|
+
notes: [],
|
|
127
|
+
}),
|
|
128
|
+
tierFn: () => ({ models: tiered, seedDisagreements: [], staleClassifications: [] }),
|
|
129
|
+
})
|
|
130
|
+
assert.equal(result.explorer.fill.key, 'gpt-5.6-terra@high')
|
|
131
|
+
assert.equal(result.explorer.reason, 'balanced-coder')
|
|
132
|
+
assert.equal(result.ask.explorerFill.id, 'gpt-5.6-terra@high')
|
|
133
|
+
// the same harness with gpt-6.1-sol discovered takes the medium rung
|
|
134
|
+
const withSol = [...tiered, { id: 'gpt-6.1-sol', tiers: { high: 'frontier' }, source: 'team-seed' }]
|
|
135
|
+
const sol = buildPreflightModels({
|
|
136
|
+
harness: 'codex',
|
|
137
|
+
risk: 'routine',
|
|
138
|
+
discoverFn: () => ({
|
|
139
|
+
harness: 'codex',
|
|
140
|
+
completeness: 'full',
|
|
141
|
+
discovered: withSol.map(({ id }) => ({ id, source: 'current-turn' })),
|
|
142
|
+
notes: [],
|
|
143
|
+
}),
|
|
144
|
+
tierFn: () => ({ models: withSol, seedDisagreements: [], staleClassifications: [] }),
|
|
145
|
+
})
|
|
146
|
+
assert.equal(sol.explorer.fill.key, 'gpt-6.1-sol@medium')
|
|
147
|
+
assert.equal(sol.implementer.key, 'gpt-6.1-sol@high', 'the routine implementer takes the @high rung')
|
|
148
|
+
})
|
|
149
|
+
|
|
150
|
+
test('high risk walks the team ladder: gpt-6.1-sol, gpt-6-astra, gpt-5.6-sol, grok-4.7 (all @high), then the deep-reasoner tier', () => {
|
|
151
|
+
const pick = recommendImplementer(points, 'high', 'pi')
|
|
152
|
+
assert.equal(pick.id, 'openai/gpt-6.1-sol')
|
|
153
|
+
assert.equal(pick.tier, 'frontier')
|
|
154
|
+
assert.equal(pick.effort, 'high')
|
|
155
|
+
// the gpt-6.1-sol implementation is reviewed by Astra (distinct id, same family)
|
|
156
|
+
const firstReview = rankReviewers(points, pick, 'pi')
|
|
157
|
+
assert.equal(firstReview.fill.key, 'openai/gpt-6-astra@medium')
|
|
158
|
+
assert.equal(firstReview.reason, 'team-default')
|
|
159
|
+
// without gpt-6.1-sol, Astra implements and is reviewed by the implementer-aware fallback
|
|
160
|
+
const withoutSol61 = points.filter((p) => p.id !== 'openai/gpt-6.1-sol')
|
|
161
|
+
const second = recommendImplementer(withoutSol61, 'high', 'pi')
|
|
162
|
+
assert.equal(second.key, 'openai/gpt-6-astra@high')
|
|
163
|
+
const secondReview = rankReviewers(points, second, 'pi')
|
|
164
|
+
assert.equal(secondReview.fill.key, 'openai/gpt-5.6-sol@medium')
|
|
165
|
+
assert.equal(secondReview.reason, 'implementer-aware-default')
|
|
166
|
+
// then gpt-5.6-sol and grok-4.7, each reviewed by gpt-6-astra@medium
|
|
167
|
+
const withoutAstra = withoutSol61.filter((p) => p.id !== 'openai/gpt-6-astra')
|
|
168
|
+
const third = recommendImplementer(withoutAstra, 'high', 'pi')
|
|
169
|
+
assert.equal(third.key, 'openai/gpt-5.6-sol@high')
|
|
170
|
+
assert.equal(rankReviewers(points, third, 'pi').fill.key, 'openai/gpt-6-astra@medium')
|
|
171
|
+
const withoutSol56 = withoutAstra.filter((p) => p.id !== 'openai/gpt-5.6-sol')
|
|
172
|
+
const fourth = recommendImplementer(withoutSol56, 'high', 'pi')
|
|
173
|
+
assert.equal(fourth.key, 'xai/grok-4.7@high')
|
|
174
|
+
assert.equal(rankReviewers(points, fourth, 'pi').fill.key, 'openai/gpt-6-astra@medium')
|
|
175
|
+
// without the harness (no team ladder) the tier order picks a deep-reasoner, never frontier
|
|
176
|
+
assert.equal(recommendImplementer(points, 'high').tier, 'deep-reasoner')
|
|
177
|
+
// without any ladder rung discovered, the tier order falls back to the other deep-reasoner
|
|
178
|
+
const withoutLadder = withoutSol56.filter((p) => !['xai/grok-4.7', 'xai/grok-4.6'].includes(p.id))
|
|
179
|
+
assert.equal(recommendImplementer(withoutLadder, 'high', 'pi').id, 'openai/gpt-5.5')
|
|
180
|
+
})
|
|
181
|
+
|
|
182
|
+
test('the recommended frontier implementer is labelled as the team preference, not an alternate', () => {
|
|
183
|
+
const implementer = recommendImplementer(points, 'high', 'pi')
|
|
184
|
+
const reviewer = rankReviewers(points, implementer, 'pi')
|
|
185
|
+
const ask = buildAsk(implementer, points, reviewer, { risk: 'high' })
|
|
186
|
+
assert.equal(ask.options[0].modelId, 'openai/gpt-6.1-sol')
|
|
187
|
+
assert.match(ask.options[0].description, /recommended · frontier — team implementer ladder preference/)
|
|
188
|
+
// without a harness there is no ladder to consult, so the other frontier point is a plain alternate
|
|
189
|
+
const otherFrontier = [...ask.options, ...ask.alternates].find((o) => o.modelId === 'openai/gpt-5.6-sol')
|
|
190
|
+
assert.match(otherFrontier.description, /alternate only/)
|
|
191
|
+
})
|
|
192
|
+
|
|
193
|
+
test('the ask lists the discovered ladder rungs first, in ladder order, before other points', () => {
|
|
194
|
+
// Real Pi discovery shape: deep-reasoners whose ids sort before the ladder rungs, plus a
|
|
195
|
+
// balanced-coder with a cheap id. Before the fix these pushed the rungs into `alternates`.
|
|
196
|
+
const many = [
|
|
197
|
+
...points,
|
|
198
|
+
{ id: 'deepseek/deepseek-v4-pro', effort: 'max', tier: 'deep-reasoner', family: 'deepseek', key: 'deepseek/deepseek-v4-pro@max' },
|
|
199
|
+
{ id: 'kimi-coding/k3', effort: 'high', tier: 'deep-reasoner', family: 'kimi', key: 'kimi-coding/k3@high' },
|
|
200
|
+
{ id: 'deepseek/deepseek-flash', effort: 'max', tier: 'balanced-coder', family: 'deepseek', key: 'deepseek/deepseek-flash@max' },
|
|
201
|
+
]
|
|
202
|
+
const high = recommendImplementer(many, 'high', 'pi')
|
|
203
|
+
const highAsk = buildAsk(many.find((p) => p.key === high.key), many, rankReviewers(many, high, 'pi'), { risk: 'high', harness: 'pi' })
|
|
204
|
+
assert.deepEqual(
|
|
205
|
+
highAsk.options.map((o) => o.id),
|
|
206
|
+
['openai/gpt-6.1-sol@high', 'openai/gpt-6-astra@high', 'openai/gpt-5.6-sol@high', 'xai/grok-4.7@high'],
|
|
207
|
+
'high-risk options are the ladder',
|
|
208
|
+
)
|
|
209
|
+
assert.match(highAsk.options[1].description, /team ladder rung 2/)
|
|
210
|
+
assert.ok(highAsk.alternates.some((o) => o.modelId === 'deepseek/deepseek-v4-pro'), 'non-ladder deep-reasoners become alternates')
|
|
211
|
+
|
|
212
|
+
const routine = recommendImplementer(many, 'routine', 'pi')
|
|
213
|
+
const routineAsk = buildAsk(routine, many, rankReviewers(many, routine, 'pi'), { risk: 'routine', harness: 'pi' })
|
|
214
|
+
assert.deepEqual(
|
|
215
|
+
routineAsk.options.map((o) => o.id),
|
|
216
|
+
['openai/gpt-6.1-sol@high', 'openai/gpt-6-luna@high', 'xai/grok-4.6@high', 'xiaomi/mimo-v2.6-pro@high'],
|
|
217
|
+
'routine options are the ladder',
|
|
218
|
+
)
|
|
219
|
+
// an undiscovered rung is skipped, not left as a hole: the next point fills the fourth slot
|
|
220
|
+
const withoutMimo = many.filter((p) => p.id !== 'xiaomi/mimo-v2.6-pro')
|
|
221
|
+
const routine2 = recommendImplementer(withoutMimo, 'routine', 'pi')
|
|
222
|
+
const ask2 = buildAsk(routine2, withoutMimo, rankReviewers(withoutMimo, routine2, 'pi'), { risk: 'routine', harness: 'pi' })
|
|
223
|
+
assert.deepEqual(ask2.options.slice(0, 3).map((o) => o.id), ['openai/gpt-6.1-sol@high', 'openai/gpt-6-luna@high', 'xai/grok-4.6@high'])
|
|
224
|
+
assert.equal(ask2.options.length, 4)
|
|
225
|
+
})
|
|
226
|
+
|
|
227
|
+
test('non-Astra implementation prefers the gpt-6-astra@medium policy review when discovered', () => {
|
|
228
|
+
const implementer = recommendImplementer(points, 'routine', 'pi')
|
|
229
|
+
const review = rankReviewers(points, implementer, 'pi')
|
|
230
|
+
assert.equal(review.reason, 'team-default')
|
|
231
|
+
assert.equal(review.fill.id, TEAM_REVIEWER_POLICY.pi.primary.id)
|
|
232
|
+
assert.equal(review.fill.effort, 'medium')
|
|
233
|
+
assert.equal(review.fill.tier, 'frontier')
|
|
234
|
+
assert.equal(review.fill.source, 'team-review-policy')
|
|
235
|
+
assert.equal(review.fill.citation, 'docs/research/2026-09-23-reviewer-benchmark-astra-vs-sol.md')
|
|
236
|
+
assert.notEqual(review.fill.id, implementer.id)
|
|
237
|
+
assert.equal(review.ranking[0].key, 'openai/gpt-6-astra@medium')
|
|
238
|
+
assert.equal(review.ranking[1].key, 'openai/gpt-5.6-sol@medium')
|
|
239
|
+
assert.ok(!review.ranking.some((candidate) => candidate.effort === 'high' && candidate.id === 'openai/gpt-6-astra'), 'the policy point replaces the catalog high point in the ranking')
|
|
240
|
+
})
|
|
241
|
+
|
|
242
|
+
test('Astra implementation uses the gpt-5.6-sol@medium implementer-aware default', () => {
|
|
243
|
+
const implementer = points.find((p) => p.id === 'openai/gpt-6-astra')
|
|
244
|
+
const review = rankReviewers(points, implementer, 'pi')
|
|
245
|
+
assert.equal(review.reason, 'implementer-aware-default')
|
|
246
|
+
assert.notEqual(review.fill.id, implementer.id)
|
|
247
|
+
assert.equal(review.fill.id, TEAM_REVIEWER_POLICY.pi.fallback.id)
|
|
248
|
+
assert.equal(review.fill.effort, 'medium')
|
|
249
|
+
assert.equal(review.fill.key, 'openai/gpt-5.6-sol@medium')
|
|
250
|
+
})
|
|
251
|
+
|
|
252
|
+
test('another Astra effort is still the same implementer model id', () => {
|
|
253
|
+
const implementer = {
|
|
254
|
+
...points.find((p) => p.id === 'openai/gpt-6-astra'),
|
|
255
|
+
effort: 'xhigh',
|
|
256
|
+
key: 'openai/gpt-6-astra@xhigh',
|
|
257
|
+
}
|
|
258
|
+
const review = rankReviewers(points, implementer, 'pi')
|
|
259
|
+
assert.equal(review.fill.id, 'openai/gpt-5.6-sol')
|
|
260
|
+
assert.notEqual(review.fill.id, implementer.id)
|
|
261
|
+
})
|
|
262
|
+
|
|
263
|
+
test('a gpt-6.1-sol implementation still gets the Astra reviewer (distinct id, same family)', () => {
|
|
264
|
+
const implementer = points.find((p) => p.id === 'openai/gpt-6.1-sol')
|
|
265
|
+
const review = rankReviewers(points, implementer, 'pi')
|
|
266
|
+
assert.equal(review.reason, 'team-default')
|
|
267
|
+
assert.equal(review.fill.id, 'openai/gpt-6-astra')
|
|
268
|
+
assert.equal(review.reducedDiversity, true, 'same family is flagged, not forbidden')
|
|
269
|
+
})
|
|
270
|
+
|
|
271
|
+
test('missing implementer-aware fallback uses the distinct-model ladder', () => {
|
|
272
|
+
const withoutSol56 = points.filter((p) => p.id !== 'openai/gpt-5.6-sol')
|
|
273
|
+
const implementer = withoutSol56.find((p) => p.id === 'openai/gpt-6-astra')
|
|
274
|
+
const review = rankReviewers(withoutSol56, implementer, 'pi')
|
|
275
|
+
assert.equal(review.reason, 'ladder')
|
|
276
|
+
// diversity-first: a cross-family deep-reasoner (grok-4.6) outranks cross-family frontier points
|
|
277
|
+
assert.equal(review.fill.id, 'xai/grok-4.6')
|
|
278
|
+
const withoutGrok = withoutSol56.filter((p) => p.family !== 'xai' && p.family !== 'xiaomi')
|
|
279
|
+
assert.equal(rankReviewers(withoutGrok, implementer, 'pi').fill.id, 'anthropic/claude-opus')
|
|
280
|
+
})
|
|
281
|
+
|
|
282
|
+
test('missing Astra primary uses the distinct-model ladder even when gpt-5.6-sol is available', () => {
|
|
283
|
+
const withoutAstra = points.filter((p) => p.id !== 'openai/gpt-6-astra')
|
|
284
|
+
const implementer = withoutAstra.find((p) => p.id === 'xai/grok-4.5')
|
|
285
|
+
const review = rankReviewers(withoutAstra, implementer, 'pi')
|
|
286
|
+
assert.equal(review.reason, 'ladder')
|
|
287
|
+
assert.equal(review.fill.id, 'openai/gpt-5.5', 'a cross-family deep-reasoner leads the ladder')
|
|
288
|
+
assert.equal(review.ranking[0].id, 'openai/gpt-5.6-sol', 'the fallback frontier point stays first in the ranking for a later Astra override')
|
|
289
|
+
})
|
|
290
|
+
|
|
291
|
+
test('scoped-coder points never enter slotPoints', () => {
|
|
292
|
+
const tiered = [
|
|
293
|
+
{ id: 'scoped/x', tiers: { high: 'scoped-coder' }, source: 'banked' },
|
|
294
|
+
{ id: 'ok/y', tiers: { high: 'balanced-coder' }, source: 'banked' },
|
|
295
|
+
]
|
|
296
|
+
const out = slotPoints(tiered, { harness: 'pi' })
|
|
297
|
+
assert.deepEqual(out.map((p) => p.id), ['ok/y'])
|
|
298
|
+
})
|
|
299
|
+
|
|
300
|
+
test('ask lists recommended implementer first, caps at 4, keeps frontier visible', () => {
|
|
301
|
+
const many = [
|
|
302
|
+
...points,
|
|
303
|
+
{ id: 'a/balanced', effort: 'high', tier: 'balanced-coder', family: 'a', key: 'a/balanced@high' },
|
|
304
|
+
{ id: 'b/balanced', effort: 'high', tier: 'balanced-coder', family: 'b', key: 'b/balanced@high' },
|
|
305
|
+
{ id: 'c/deep', effort: 'high', tier: 'deep-reasoner', family: 'c', key: 'c/deep@high' },
|
|
306
|
+
]
|
|
307
|
+
const implementer = recommendImplementer(many, 'routine')
|
|
308
|
+
const explorer = fillExplorer(many, 'pi')
|
|
309
|
+
const reviewer = rankReviewers(many, implementer, 'pi')
|
|
310
|
+
const ask = buildAsk(implementer, many, reviewer, { risk: 'routine', explorer })
|
|
311
|
+
assert.equal(ask.options[0].modelId, implementer.id)
|
|
312
|
+
assert.equal(ask.recommended, implementer.key)
|
|
313
|
+
assert.ok(ask.options.length <= 4)
|
|
314
|
+
assert.equal(ask.reviewerFill.modelId, reviewer.fill.id)
|
|
315
|
+
assert.equal(ask.explorerFill.modelId, explorer.fill.id)
|
|
316
|
+
assert.ok(
|
|
317
|
+
ask.options.some((o) => o.tier === 'frontier') || ask.alternates.some((o) => o.tier === 'frontier'),
|
|
318
|
+
'frontier stays visible',
|
|
319
|
+
)
|
|
320
|
+
})
|
|
321
|
+
|
|
322
|
+
test('buildPreflightModels is injectable and defers sync/bank', () => {
|
|
323
|
+
const result = buildPreflightModels({
|
|
324
|
+
harness: 'pi',
|
|
325
|
+
risk: 'routine',
|
|
326
|
+
available: ['openai/gpt-5.5', 'xai/grok-4.5', 'openai/gpt-5.6-sol', 'openai/gpt-5.6-terra', 'openai/gpt-6-astra'],
|
|
327
|
+
discoverFn: () => ({
|
|
328
|
+
harness: 'pi',
|
|
329
|
+
completeness: 'full',
|
|
330
|
+
discovered: [
|
|
331
|
+
{ id: 'openai/gpt-6-astra', source: 'current-turn' },
|
|
332
|
+
{ id: 'openai/gpt-5.5', source: 'current-turn' },
|
|
333
|
+
{ id: 'xai/grok-4.5', source: 'current-turn' },
|
|
334
|
+
{ id: 'openai/gpt-5.6-sol', source: 'current-turn' },
|
|
335
|
+
{ id: 'openai/gpt-5.6-terra', source: 'current-turn' },
|
|
336
|
+
],
|
|
337
|
+
notes: [],
|
|
338
|
+
}),
|
|
339
|
+
tierFn: () => ({
|
|
340
|
+
models: [
|
|
341
|
+
{ id: 'openai/gpt-6-astra', tiers: { high: 'frontier' }, source: 'team-seed', citation: null },
|
|
342
|
+
{ id: 'openai/gpt-5.5', tiers: { high: 'deep-reasoner' }, source: 'team-seed', citation: null },
|
|
343
|
+
{ id: 'xai/grok-4.5', tiers: { high: 'balanced-coder' }, source: 'team-seed', citation: null },
|
|
344
|
+
{ id: 'openai/gpt-5.6-sol', tiers: { high: 'frontier' }, source: 'team-seed', citation: null },
|
|
345
|
+
{ id: 'openai/gpt-5.6-terra', tiers: { high: 'balanced-coder' }, source: 'team-seed', citation: null },
|
|
346
|
+
],
|
|
347
|
+
seedDisagreements: [],
|
|
348
|
+
staleClassifications: [],
|
|
349
|
+
}),
|
|
350
|
+
})
|
|
351
|
+
assert.equal(result.ok, true)
|
|
352
|
+
assert.equal(result.implementer.tier, 'balanced-coder')
|
|
353
|
+
assert.equal(result.explorer.reason, 'balanced-coder')
|
|
354
|
+
assert.equal(result.explorer.fill.id, 'openai/gpt-5.6-terra')
|
|
355
|
+
assert.equal(result.reviewer.reason, 'team-default')
|
|
356
|
+
assert.equal(result.reviewer.fill.id, TEAM_REVIEWER_POLICY.pi.primary.id)
|
|
357
|
+
assert.equal(result.reviewer.fill.effort, 'medium')
|
|
358
|
+
assert.ok(result.deferred.afterConfirm.some((s) => s.includes('sync')))
|
|
359
|
+
assert.ok(result.deferred.doNotReadBeforeAsk.length >= 1)
|
|
360
|
+
assert.equal(result.ask.options[0].modelId, result.implementer.id)
|
|
361
|
+
assert.equal(result.ask.explorerFill.modelId, 'openai/gpt-5.6-terra')
|
|
362
|
+
assert.equal(result.diagnoser.fill.key, 'openai/gpt-6-astra@high')
|
|
363
|
+
assert.equal(result.ask.diagnoserFill.id, 'openai/gpt-6-astra@high')
|
|
364
|
+
})
|
|
365
|
+
|
|
366
|
+
test('CLI --ask-only prints only the ask block', () => {
|
|
367
|
+
const script = fileURLToPath(new URL('./preflight-models.mjs', import.meta.url))
|
|
368
|
+
const run = spawnSync(process.execPath, [
|
|
369
|
+
script,
|
|
370
|
+
'--harness', 'pi',
|
|
371
|
+
'--risk', 'routine',
|
|
372
|
+
'--available', 'openai/gpt-5.5,xai/grok-4.5,openai/gpt-5.6-sol,openai/gpt-5.6-terra,openai/gpt-6-astra',
|
|
373
|
+
'--ask-only',
|
|
374
|
+
], { encoding: 'utf8' })
|
|
375
|
+
assert.equal(run.status, 0, run.stderr)
|
|
376
|
+
const ask = JSON.parse(run.stdout)
|
|
377
|
+
assert.ok(ask.question)
|
|
378
|
+
assert.ok(Array.isArray(ask.options))
|
|
379
|
+
assert.ok(ask.reviewerFill)
|
|
380
|
+
assert.ok(ask.explorerFill)
|
|
381
|
+
assert.equal(ask.options[0].description.includes('recommended'), true)
|
|
382
|
+
})
|
|
383
|
+
|
|
384
|
+
const pinTiered = [
|
|
385
|
+
{ id: 'gpt-6-astra', tiers: { high: 'frontier' }, source: 'team-seed' },
|
|
386
|
+
{ id: 'gpt-6.1-sol', tiers: { high: 'frontier' }, source: 'team-seed' },
|
|
387
|
+
{ id: 'gpt-5.6-terra', tiers: { high: 'balanced-coder' }, source: 'team-seed' },
|
|
388
|
+
]
|
|
389
|
+
const pinRun = (implementer, tiered = pinTiered) => buildPreflightModels({
|
|
390
|
+
harness: 'codex',
|
|
391
|
+
risk: 'routine',
|
|
392
|
+
implementer,
|
|
393
|
+
discoverFn: () => ({
|
|
394
|
+
harness: 'codex',
|
|
395
|
+
completeness: 'full',
|
|
396
|
+
discovered: tiered.map(({ id }) => ({ id, source: 'current-turn' })),
|
|
397
|
+
notes: [],
|
|
398
|
+
}),
|
|
399
|
+
tierFn: () => ({ models: tiered, seedDisagreements: [], staleClassifications: [] }),
|
|
400
|
+
})
|
|
401
|
+
|
|
402
|
+
test('a pinned implementer is kept over the recommendation and is the only option', () => {
|
|
403
|
+
const result = pinRun('gpt-5.6-terra')
|
|
404
|
+
assert.equal(result.ok, true)
|
|
405
|
+
assert.equal(result.pinned, 'gpt-5.6-terra')
|
|
406
|
+
assert.equal(result.implementer.key, 'gpt-5.6-terra@high', 'the ladder would have picked gpt-6.1-sol@high')
|
|
407
|
+
assert.deepEqual(result.ask.options.map((o) => o.id), ['gpt-5.6-terra@high'])
|
|
408
|
+
assert.deepEqual(result.ask.alternates, [])
|
|
409
|
+
assert.equal(result.ask.pinned, true)
|
|
410
|
+
assert.notEqual(result.reviewer.fill.id, 'gpt-5.6-terra', 'the reviewer still differs from the pinned implementer')
|
|
411
|
+
})
|
|
412
|
+
|
|
413
|
+
test('a pin with an effort matches that operating point only', () => {
|
|
414
|
+
assert.equal(pinRun('gpt-6-astra@high').implementer.key, 'gpt-6-astra@high')
|
|
415
|
+
const wrongEffort = pinRun('gpt-6-astra@max')
|
|
416
|
+
assert.equal(wrongEffort.ok, false)
|
|
417
|
+
assert.equal(wrongEffort.code, 'pinned-implementer-unavailable')
|
|
418
|
+
})
|
|
419
|
+
|
|
420
|
+
test('an unavailable pinned implementer blocks instead of falling back', () => {
|
|
421
|
+
const result = pinRun('cory-model')
|
|
422
|
+
assert.equal(result.ok, false)
|
|
423
|
+
assert.equal(result.code, 'pinned-implementer-unavailable')
|
|
424
|
+
assert.equal(result.implementer, null)
|
|
425
|
+
assert.equal(result.ask, null)
|
|
426
|
+
assert.match(result.reason, /never substitute/)
|
|
427
|
+
})
|
|
428
|
+
|
|
429
|
+
test('no pin keeps the recommendation and the full option list', () => {
|
|
430
|
+
const result = pinRun(null)
|
|
431
|
+
assert.equal(result.pinned, undefined)
|
|
432
|
+
assert.equal(result.implementer.key, 'gpt-6.1-sol@high')
|
|
433
|
+
assert.ok(result.ask.options.length > 1)
|
|
434
|
+
assert.equal(result.ask.pinned, undefined)
|
|
435
|
+
})
|
|
436
|
+
|
|
437
|
+
test('CLI exits 0 with the unavailable pin in the output and rejects an empty one with 2', () => {
|
|
438
|
+
const script = fileURLToPath(new URL('./preflight-models.mjs', import.meta.url))
|
|
439
|
+
const run = (extra) => spawnSync(process.execPath, [
|
|
440
|
+
script, '--harness', 'pi', '--available', 'openai/gpt-5.6-terra', ...extra,
|
|
441
|
+
], { encoding: 'utf8' })
|
|
442
|
+
const missing = run(['--implementer', 'openai/not-a-model'])
|
|
443
|
+
assert.equal(missing.status, 0, missing.stderr)
|
|
444
|
+
assert.equal(JSON.parse(missing.stdout).ok, false)
|
|
445
|
+
assert.equal(JSON.parse(missing.stdout).code, 'pinned-implementer-unavailable')
|
|
446
|
+
assert.equal(run(['--implementer=']).status, 2)
|
|
447
|
+
assert.equal(run(['--implementer']).status, 2, 'a flag with no value never falls back to the recommendation')
|
|
448
|
+
})
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Check that every GitHub issue a resolve-issues run will work on is marked ready for an
|
|
3
|
+
// agent, before anything else in preflight.
|
|
4
|
+
//
|
|
5
|
+
// ready-check.mjs --issue <ref> [--issue <ref> ...] [--repo <owner/name>] [--label ready-for-agent] [--exit-outcome]
|
|
6
|
+
//
|
|
7
|
+
// A <ref> is `owner/name#123`, an issue URL, or a bare `123` with --repo. Every
|
|
8
|
+
// issue the run will implement must carry the label: each named issue and every
|
|
9
|
+
// OPEN sub-issue beneath it (any depth, any repository). Closed sub-issues are
|
|
10
|
+
// skipped: they are already done and will not be implemented. An open issue with an
|
|
11
|
+
// open sub-issue is a container (an epic): it needs no label, its open sub-issues carry
|
|
12
|
+
// it instead. An open issue whose sub-issues are all closed is a unit again, because open
|
|
13
|
+
// means work remains: it needs the label, and the verdict lists it under `rollups` so the
|
|
14
|
+
// explorer records what is left or that nothing is (then the human closes it).
|
|
15
|
+
// An issue that cannot be read counts as not ready, and a parent with an
|
|
16
|
+
// unreadable sub-issue is a unit, so the check fails closed rather than letting
|
|
17
|
+
// unreviewed work through.
|
|
18
|
+
//
|
|
19
|
+
// A missing label does not stop a run: resolve-issues asks the human whether to
|
|
20
|
+
// proceed, and rush-issues warns and proceeds. An unreadable issue does stop it.
|
|
21
|
+
//
|
|
22
|
+
// Prints a JSON verdict whose `outcome` is `ready` (every checked issue is ready), `unlabeled`
|
|
23
|
+
// (every issue is readable but some lack the label), or `unreadable` (any issue could not be
|
|
24
|
+
// read; it outranks unlabeled). Exit 0 whenever it ran and answered; 2 for bad input or a crash.
|
|
25
|
+
// --exit-outcome restores the old exit codes for one release (deprecated): 0 ready, 3 unlabeled,
|
|
26
|
+
// 1 unreadable. Each `missing` entry names its `parent`: null for an issue the human named, the
|
|
27
|
+
// parent's ref for a sub-issue.
|
|
28
|
+
|
|
29
|
+
import { execFileSync } from 'node:child_process'
|
|
30
|
+
|
|
31
|
+
import { isMainModule } from '../../engineering-runtime/scripts/main-module.mjs'
|
|
32
|
+
import { fetchTree } from './issue-tree.mjs'
|
|
33
|
+
|
|
34
|
+
export const DEFAULT_LABEL = 'ready-for-agent'
|
|
35
|
+
|
|
36
|
+
export class ReadyCheckError extends Error {}
|
|
37
|
+
|
|
38
|
+
export function parseIssueRef(text, defaultRepo = null) {
|
|
39
|
+
const value = String(text ?? '').trim()
|
|
40
|
+
let match = /^https?:\/\/github\.com\/([^/\s]+)\/([^/\s]+)\/issues\/(\d+)\b/.exec(value)
|
|
41
|
+
if (match) return { repo: `${match[1]}/${match[2]}`, number: Number(match[3]) }
|
|
42
|
+
match = /^([^/\s#]+)\/([^/\s#]+)#(\d+)$/.exec(value)
|
|
43
|
+
if (match) return { repo: `${match[1]}/${match[2]}`, number: Number(match[3]) }
|
|
44
|
+
match = /^#?(\d+)$/.exec(value)
|
|
45
|
+
if (match && defaultRepo) return { repo: defaultRepo, number: Number(match[1]) }
|
|
46
|
+
throw new ReadyCheckError(`cannot read issue reference "${value}" (use owner/name#123, an issue URL, or a number with --repo)`)
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const refKey = ({ repo, number }) => `${repo.toLowerCase()}#${number}`
|
|
50
|
+
|
|
51
|
+
function repoFromApiUrl(url) {
|
|
52
|
+
const match = /\/repos\/([^/]+)\/([^/]+)(?:\/|$)/.exec(String(url || ''))
|
|
53
|
+
return match ? `${match[1]}/${match[2]}` : null
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// Walk the named issues and their open sub-issues. `fetchIssue` and
|
|
57
|
+
// `fetchSubIssues` are injected so the rules can be tested without GitHub.
|
|
58
|
+
export function readyCheck(refs, { label = DEFAULT_LABEL, fetchIssue, fetchSubIssues } = {}) {
|
|
59
|
+
const wanted = String(label).toLowerCase()
|
|
60
|
+
const checked = []
|
|
61
|
+
const visited = new Set()
|
|
62
|
+
const childKeys = new Map()
|
|
63
|
+
const queue = refs.map((ref) => ({ ref, parent: null, depth: 0 }))
|
|
64
|
+
while (queue.length > 0) {
|
|
65
|
+
const { ref, parent, depth } = queue.shift()
|
|
66
|
+
const key = refKey(ref)
|
|
67
|
+
if (visited.has(key)) continue
|
|
68
|
+
visited.add(key)
|
|
69
|
+
const at = `${ref.repo}#${ref.number}`
|
|
70
|
+
let issue
|
|
71
|
+
try {
|
|
72
|
+
issue = fetchIssue(ref)
|
|
73
|
+
} catch (error) {
|
|
74
|
+
checked.push({ ref: at, parent, depth, state: null, title: null, labels: [], ready: false, error: String(error?.message || error) })
|
|
75
|
+
continue
|
|
76
|
+
}
|
|
77
|
+
const state = String(issue?.state || '').toLowerCase()
|
|
78
|
+
const labels = (issue?.labels || []).map((l) => (typeof l === 'string' ? l : l?.name)).filter(Boolean)
|
|
79
|
+
// A named issue is always checked; a closed sub-issue is done and skipped.
|
|
80
|
+
if (depth > 0 && state === 'closed') {
|
|
81
|
+
checked.push({ ref: at, parent, depth, state, title: issue?.title ?? null, labels, ready: true, skipped: 'closed' })
|
|
82
|
+
continue
|
|
83
|
+
}
|
|
84
|
+
const ready = labels.some((name) => name.toLowerCase() === wanted)
|
|
85
|
+
const entry = { ref: at, parent, depth, state, title: issue?.title ?? null, labels, ready }
|
|
86
|
+
checked.push(entry)
|
|
87
|
+
let children
|
|
88
|
+
try {
|
|
89
|
+
children = fetchSubIssues(ref)
|
|
90
|
+
} catch (error) {
|
|
91
|
+
entry.subIssuesUnreadable = true
|
|
92
|
+
checked.push({ ref: `${at} (sub-issues)`, parent: at, depth: depth + 1, state: null, title: null, labels: [], ready: false, error: String(error?.message || error) })
|
|
93
|
+
continue
|
|
94
|
+
}
|
|
95
|
+
for (const child of children || []) {
|
|
96
|
+
const repo = repoFromApiUrl(child?.repository_url) || ref.repo
|
|
97
|
+
if (!Number.isInteger(child?.number)) continue
|
|
98
|
+
const childRef = { repo, number: child.number }
|
|
99
|
+
childKeys.set(entry, [...(childKeys.get(entry) || []), refKey(childRef)])
|
|
100
|
+
queue.push({ ref: childRef, parent: at, depth: depth + 1 })
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
// Classify each open issue from its sub-issues: `container` when one is open, `unit`
|
|
104
|
+
// otherwise (none, all closed, or unreadable). Only a unit needs the label.
|
|
105
|
+
const byKey = new Map(checked.filter((entry) => !entry.error).map((entry) => [entry.ref.toLowerCase(), entry]))
|
|
106
|
+
for (const entry of byKey.values()) {
|
|
107
|
+
if (entry.skipped) continue
|
|
108
|
+
const keys = childKeys.get(entry) || []
|
|
109
|
+
const open = keys.map((key) => byKey.get(key)).some((child) => child && child.state !== 'closed')
|
|
110
|
+
if (open && !entry.subIssuesUnreadable) {
|
|
111
|
+
if (!entry.ready) Object.assign(entry, { ready: true, exempt: 'has open sub-issues' })
|
|
112
|
+
} else if (keys.length > 0 && !entry.subIssuesUnreadable && entry.state !== 'closed' && keys.every((key) => byKey.has(key))) entry.rollup = true
|
|
113
|
+
}
|
|
114
|
+
const notReady = checked.filter((entry) => !entry.ready)
|
|
115
|
+
const unreadable = notReady.filter((entry) => entry.error).map(({ ref, error }) => ({ ref, error }))
|
|
116
|
+
return {
|
|
117
|
+
label,
|
|
118
|
+
outcome: unreadable.length > 0 ? 'unreadable' : notReady.length > 0 ? 'unlabeled' : 'ready',
|
|
119
|
+
ready: notReady.length === 0,
|
|
120
|
+
rollups: checked.filter((entry) => entry.rollup).map(({ ref, title, state }) => ({ ref, title, state })),
|
|
121
|
+
missing: notReady.filter((entry) => !entry.error).map(({ ref, title, labels, parent }) => ({ ref, title, labels, parent })),
|
|
122
|
+
unreadable,
|
|
123
|
+
checked,
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function gh(args) {
|
|
128
|
+
return JSON.parse(execFileSync('gh', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: 30_000 }))
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
export const githubFetchers = {
|
|
132
|
+
fetchIssue: ({ repo, number }) => gh(['api', `repos/${repo}/issues/${number}`]),
|
|
133
|
+
// `--slurp` wraps each page in an outer array; flatten it back to one list.
|
|
134
|
+
fetchSubIssues: ({ repo, number }) => gh(['api', '--paginate', '--slurp', `repos/${repo}/issues/${number}/sub_issues`]).flat(),
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// Answer from trees fetched by issue-tree.mjs (one GraphQL query per named issue), and
|
|
138
|
+
// fall back to the REST fetchers only for an issue the trees do not fully cover.
|
|
139
|
+
export function treeFetchers(trees, fallback = githubFetchers) {
|
|
140
|
+
const nodes = new Map()
|
|
141
|
+
for (const tree of trees) for (const node of Object.values(tree?.nodes ?? {})) if (node.state) nodes.set(node.key.toLowerCase(), node)
|
|
142
|
+
const find = ({ repo, number }) => nodes.get(`${repo}#${number}`.toLowerCase())
|
|
143
|
+
return {
|
|
144
|
+
fetchIssue: (ref) => {
|
|
145
|
+
const node = find(ref)
|
|
146
|
+
return node ? { state: node.state, title: node.title, labels: node.labels.map((name) => ({ name })) } : fallback.fetchIssue(ref)
|
|
147
|
+
},
|
|
148
|
+
fetchSubIssues: (ref) => {
|
|
149
|
+
const node = find(ref)
|
|
150
|
+
if (!node?.children) return fallback.fetchSubIssues(ref)
|
|
151
|
+
return node.children.map((key) => {
|
|
152
|
+
const at = key.lastIndexOf('#')
|
|
153
|
+
return { number: Number(key.slice(at + 1)), repository_url: `https://api.github.com/repos/${key.slice(0, at)}` }
|
|
154
|
+
})
|
|
155
|
+
},
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
export function parseArgs(argv) {
|
|
160
|
+
const args = { issues: [], repo: null, label: DEFAULT_LABEL }
|
|
161
|
+
for (let i = 0; i < argv.length; i += 1) {
|
|
162
|
+
const arg = argv[i]
|
|
163
|
+
if (arg === '--issue') args.issues.push(argv[++i])
|
|
164
|
+
else if (arg === '--repo') args.repo = argv[++i]
|
|
165
|
+
else if (arg === '--label') args.label = argv[++i]
|
|
166
|
+
else if (arg === '--exit-outcome') args.exitOutcome = true
|
|
167
|
+
else throw new ReadyCheckError(`unknown argument ${arg}`)
|
|
168
|
+
}
|
|
169
|
+
if (args.issues.length === 0) throw new ReadyCheckError('at least one --issue is required')
|
|
170
|
+
if (!args.label) throw new ReadyCheckError('--label needs a value')
|
|
171
|
+
return args
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
if (isMainModule(import.meta.url)) {
|
|
175
|
+
try {
|
|
176
|
+
const args = parseArgs(process.argv.slice(2))
|
|
177
|
+
const refs = args.issues.map((text) => parseIssueRef(text, args.repo))
|
|
178
|
+
const result = readyCheck(refs, { label: args.label, ...treeFetchers(refs.map((ref) => fetchTree(ref))) })
|
|
179
|
+
console.log(JSON.stringify(result, null, 2))
|
|
180
|
+
process.exitCode = args.exitOutcome ? { ready: 0, unlabeled: 3, unreadable: 1 }[result.outcome] : 0
|
|
181
|
+
} catch (error) {
|
|
182
|
+
console.error(error instanceof ReadyCheckError ? error.message : String(error?.stack || error))
|
|
183
|
+
process.exitCode = 2
|
|
184
|
+
}
|
|
185
|
+
}
|