@1aboveio/skills 1.0.8 → 1.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.codex-plugin/plugin.json +13 -0
- package/.codex-plugin/resources.json +1827 -0
- package/README.md +22 -0
- package/package.json +9 -2
- package/skills/app-dev/airflow/SKILL.md +1 -1
- package/skills/app-dev/airflow/release-check.mjs +3 -1
- package/skills/app-dev/android-foundation/SKILL.md +1 -1
- package/skills/app-dev/android-foundation/release-check.mjs +3 -1
- package/skills/app-dev/app-debug/SKILL.md +1 -1
- package/skills/app-dev/app-debug/release-check.mjs +3 -1
- package/skills/app-dev/better-auth/SKILL.md +1 -1
- package/skills/app-dev/better-auth/release-check.mjs +3 -1
- package/skills/app-dev/data-catalog/SKILL.md +1 -1
- package/skills/app-dev/data-catalog/release-check.mjs +3 -1
- package/skills/app-dev/data-quality/SKILL.md +1 -1
- package/skills/app-dev/data-quality/release-check.mjs +3 -1
- package/skills/app-dev/ios-foundation/SKILL.md +1 -1
- package/skills/app-dev/ios-foundation/release-check.mjs +3 -1
- package/skills/app-dev/monorepo/SKILL.md +1 -1
- package/skills/app-dev/monorepo/release-check.mjs +3 -1
- package/skills/app-dev/nextjs-fullstack/SKILL.md +1 -1
- package/skills/app-dev/nextjs-fullstack/release-check.mjs +3 -1
- package/skills/app-dev/prisma-setup/SKILL.md +1 -1
- package/skills/app-dev/prisma-setup/release-check.mjs +3 -1
- package/skills/app-dev/pyspark/SKILL.md +1 -1
- package/skills/app-dev/pyspark/release-check.mjs +3 -1
- package/skills/app-dev/python-backend/SKILL.md +1 -1
- package/skills/app-dev/python-backend/release-check.mjs +3 -1
- package/skills/app-dev/shadcn/SKILL.md +1 -1
- package/skills/app-dev/shadcn/release-check.mjs +3 -1
- package/skills/app-dev/zod-v4/SKILL.md +1 -1
- package/skills/app-dev/zod-v4/release-check.mjs +3 -1
- package/skills/cicd/cloud-build/SKILL.md +1 -1
- package/skills/cicd/cloud-build/release-check.mjs +3 -1
- package/skills/cicd/cloud-debug/SKILL.md +1 -1
- package/skills/cicd/cloud-debug/release-check.mjs +3 -1
- package/skills/cicd/cloud-deploy/SKILL.md +1 -1
- package/skills/cicd/cloud-deploy/release-check.mjs +3 -1
- package/skills/cicd/docker/SKILL.md +1 -1
- package/skills/cicd/docker/release-check.mjs +3 -1
- package/skills/cicd/github-actions/SKILL.md +1 -1
- package/skills/cicd/github-actions/release-check.mjs +3 -1
- package/skills/cicd/github-flow/SKILL.md +1 -1
- package/skills/cicd/github-flow/release-check.mjs +3 -1
- package/skills/cicd/google-cloud/SKILL.md +1 -1
- package/skills/cicd/google-cloud/release-check.mjs +3 -1
- package/skills/cicd/mergify/SKILL.md +1 -1
- package/skills/cicd/mergify/release-check.mjs +3 -1
- package/skills/cicd/podman/SKILL.md +1 -1
- package/skills/cicd/podman/release-check.mjs +3 -1
- package/skills/content/article-review/SKILL.md +1 -1
- package/skills/content/article-review/release-check.mjs +3 -1
- package/skills/content/article-to-image/SKILL.md +1 -1
- package/skills/content/article-to-image/release-check.mjs +3 -1
- package/skills/content/flowchart-svg/SKILL.md +1 -1
- package/skills/content/flowchart-svg/release-check.mjs +3 -1
- package/skills/content/image-gen/SKILL.md +1 -1
- package/skills/content/image-gen/release-check.mjs +3 -1
- package/skills/content/mermaid/SKILL.md +1 -1
- package/skills/content/mermaid/release-check.mjs +3 -1
- package/skills/content/wechat-article-writer/SKILL.md +1 -1
- package/skills/content/wechat-article-writer/release-check.mjs +3 -1
- package/skills/content/wechat-publisher/SKILL.md +1 -1
- package/skills/content/wechat-publisher/release-check.mjs +3 -1
- package/skills/content/wechat-search/SKILL.md +1 -1
- package/skills/content/wechat-search/release-check.mjs +3 -1
- package/skills/content/xai-search/SKILL.md +88 -13
- package/skills/content/xai-search/release-check.mjs +3 -1
- package/skills/content/xai-search/scripts/xai_search.sh +125 -149
- package/skills/core/distill-writing/SKILL.md +1 -1
- package/skills/core/distill-writing/release-check.mjs +3 -1
- package/skills/core/doc-to-md/SKILL.md +1 -1
- package/skills/core/doc-to-md/release-check.mjs +3 -1
- package/skills/core/editorial-review/SKILL.md +1 -1
- package/skills/core/editorial-review/release-check.mjs +3 -1
- package/skills/core/house-style/SKILL.md +1 -1
- package/skills/core/house-style/release-check.mjs +3 -1
- package/skills/core/plain-language/SKILL.md +1 -1
- package/skills/core/plain-language/release-check.mjs +3 -1
- package/skills/core/polish-cn/SKILL.md +1 -1
- package/skills/core/polish-cn/release-check.mjs +3 -1
- package/skills/core/reg-parser/SKILL.md +1 -1
- package/skills/core/reg-parser/release-check.mjs +3 -1
- package/skills/core/terminology/SKILL.md +1 -1
- package/skills/core/terminology/release-check.mjs +3 -1
- package/skills/core/write-doc/SKILL.md +1 -1
- package/skills/core/write-doc/release-check.mjs +3 -1
- package/skills/engineering/better-review/SKILL.md +1 -1
- package/skills/engineering/better-review/release-check.mjs +3 -1
- package/skills/engineering/contributing/SKILL.md +1 -1
- package/skills/engineering/contributing/release-check.mjs +3 -1
- package/skills/engineering/e2e-test/SKILL.md +1 -1
- package/skills/engineering/e2e-test/release-check.mjs +3 -1
- package/skills/engineering/engineering-runtime/SKILL.md +1 -1
- package/skills/engineering/engineering-runtime/coherence/workflow.json +13 -13
- package/skills/engineering/engineering-runtime/release-check.mjs +3 -1
- package/skills/engineering/engineering-runtime/scripts/installation-resolver.mjs +191 -0
- package/skills/engineering/engineering-runtime/scripts/workflow-coherence.mjs +51 -10
- package/skills/engineering/ensure-coverage/SKILL.md +1 -1
- package/skills/engineering/ensure-coverage/release-check.mjs +3 -1
- package/skills/engineering/golden-path/SKILL.md +1 -1
- package/skills/engineering/golden-path/release-check.mjs +3 -1
- package/skills/engineering/harness-runtime/SKILL.md +1 -1
- package/skills/engineering/harness-runtime/release-check.mjs +3 -1
- package/skills/engineering/resolve-issues/SKILL.md +1 -1
- package/skills/engineering/resolve-issues/references/preflight.md +7 -0
- package/skills/engineering/resolve-issues/references/shared-modules.md +27 -0
- package/skills/engineering/resolve-issues/references/task-plan.md +5 -1
- package/skills/engineering/resolve-issues/release-check.mjs +3 -1
- package/skills/engineering/resolve-issues/scripts/active-installation.mjs +32 -0
- package/skills/engineering/resolve-issues/scripts/install-hooks.mjs +3 -0
- package/skills/engineering/resolve-issues/scripts/rehydrate.mjs +17 -1
- package/skills/engineering/resolve-release/SKILL.md +1 -1
- package/skills/engineering/resolve-release/release-check.mjs +3 -1
- package/skills/engineering/rush-issues/SKILL.md +1 -1
- package/skills/engineering/rush-issues/release-check.mjs +3 -1
- package/skills/engineering/rush-release/SKILL.md +1 -1
- package/skills/engineering/rush-release/release-check.mjs +3 -1
- package/skills/engineering/test-quality/SKILL.md +1 -1
- package/skills/engineering/test-quality/release-check.mjs +3 -1
- package/skills/risk-compliance/compliance-audit/SKILL.md +1 -1
- package/skills/risk-compliance/compliance-audit/release-check.mjs +3 -1
- package/skills/risk-compliance/contract-compare/SKILL.md +1 -1
- package/skills/risk-compliance/contract-compare/release-check.mjs +3 -1
- package/skills/risk-compliance/contract-review/SKILL.md +1 -1
- package/skills/risk-compliance/contract-review/release-check.mjs +3 -1
- package/skills/risk-compliance/doc-authenticity/SKILL.md +1 -1
- package/skills/risk-compliance/doc-authenticity/release-check.mjs +3 -1
- package/skills/risk-compliance/fraud-analysis/SKILL.md +1 -1
- package/skills/risk-compliance/fraud-analysis/release-check.mjs +3 -1
- package/skills/risk-compliance/payment-analysis/SKILL.md +1 -1
- package/skills/risk-compliance/payment-analysis/release-check.mjs +3 -1
- package/skills/risk-compliance/similar-domain/SKILL.md +1 -1
- package/skills/risk-compliance/similar-domain/release-check.mjs +3 -1
- package/skills/risk-compliance/tld-detection/SKILL.md +1 -1
- package/skills/risk-compliance/tld-detection/release-check.mjs +3 -1
- package/skills/app-dev/android-foundation/scripts/device-run.test.mjs +0 -436
- package/skills/app-dev/data-catalog/scripts/data-catalog.test.mjs +0 -36
- package/skills/app-dev/data-quality/scripts/data-quality.test.mjs +0 -12
- package/skills/cicd/github-actions/scripts/ci-run-audit.test.mjs +0 -637
- package/skills/cicd/github-flow/scripts/flow.test.mjs +0 -114
- package/skills/cicd/github-flow/scripts/persist-conventions.test.mjs +0 -108
- package/skills/cicd/mergify/scripts/audit.test.mjs +0 -201
- package/skills/cicd/mergify/scripts/entrypoints.test.mjs +0 -106
- package/skills/cicd/mergify/scripts/pr-620-mergify-comments.fixture.json +0 -30
- package/skills/cicd/mergify/scripts/stack-push.test.mjs +0 -233
- package/skills/cicd/mergify/scripts/watch-pr-delivery.test.mjs +0 -1826
- package/skills/content/image-gen/scripts/credentials.test.mjs +0 -212
- package/skills/content/image-gen/scripts/options.test.mjs +0 -245
- package/skills/content/xai-search/tests/smoke_test.sh +0 -36
- package/skills/core/polish-cn/scripts/polish-cn.test.mjs +0 -244
- package/skills/core/reg-parser/scripts/reg-parser.test.mjs +0 -165
- package/skills/engineering/better-review/scripts/pr-context.test.mjs +0 -168
- package/skills/engineering/better-review/scripts/rereview-scope.test.mjs +0 -587
- package/skills/engineering/better-review/scripts/scan-diff.test.mjs +0 -414
- package/skills/engineering/better-review/scripts/vendor-sync.test.mjs +0 -96
- package/skills/engineering/contributing/scripts/persist-conventions.test.mjs +0 -118
- package/skills/engineering/e2e-test/scripts/presentation-checks.test.mjs +0 -106
- package/skills/engineering/e2e-test/scripts/presentation-perceivability.test.mjs +0 -117
- package/skills/engineering/e2e-test/scripts/presentation-reachability.test.mjs +0 -69
- package/skills/engineering/e2e-test/scripts/presentation-render-health.test.mjs +0 -98
- package/skills/engineering/e2e-test/scripts/presentation-sweep.test.mjs +0 -71
- package/skills/engineering/e2e-test/scripts/presentation-temporal.test.mjs +0 -59
- package/skills/engineering/e2e-test/scripts/project-route-manifest.test.mjs +0 -76
- package/skills/engineering/engineering-runtime/engineering-runtime.test.mjs +0 -68
- package/skills/engineering/engineering-runtime/scripts/invocation-policy.test.mjs +0 -95
- package/skills/engineering/engineering-runtime/scripts/main-module.test.mjs +0 -288
- package/skills/engineering/engineering-runtime/scripts/required-check-plan.test.mjs +0 -96
- package/skills/engineering/ensure-coverage/scripts/adapters/nextjs-inventory.test.mjs +0 -96
- package/skills/engineering/ensure-coverage/scripts/check-quarantine-expiry.test.mjs +0 -47
- package/skills/engineering/ensure-coverage/scripts/ci-audit.test.mjs +0 -86
- package/skills/engineering/ensure-coverage/scripts/coverage-ledger.test.mjs +0 -447
- package/skills/engineering/ensure-coverage/scripts/design-parity.test.mjs +0 -552
- package/skills/engineering/ensure-coverage/scripts/lint-tests.test.mjs +0 -289
- package/skills/engineering/ensure-coverage/scripts/mock-policy-config.test.mjs +0 -119
- package/skills/engineering/golden-path/scripts/golden-path.test.mjs +0 -1457
- package/skills/engineering/harness-runtime/discover-models.test.mjs +0 -363
- package/skills/engineering/harness-runtime/harness-runtime.test.mjs +0 -967
- package/skills/engineering/harness-runtime/model-catalog-seed.test.mjs +0 -829
- package/skills/engineering/harness-runtime/model-catalog.test.mjs +0 -366
- package/skills/engineering/resolve-issues/scripts/cleanup-runs.test.mjs +0 -280
- package/skills/engineering/resolve-issues/scripts/dropbox.test.mjs +0 -346
- package/skills/engineering/resolve-issues/scripts/findings-reuse.test.mjs +0 -88
- package/skills/engineering/resolve-issues/scripts/github-reviews.test.mjs +0 -264
- package/skills/engineering/resolve-issues/scripts/install-hooks.test.mjs +0 -79
- package/skills/engineering/resolve-issues/scripts/issue-tree.test.mjs +0 -107
- package/skills/engineering/resolve-issues/scripts/manual-invocation.test.mjs +0 -14
- package/skills/engineering/resolve-issues/scripts/next-actions-capacity.test.mjs +0 -149
- package/skills/engineering/resolve-issues/scripts/next-actions.test.mjs +0 -915
- package/skills/engineering/resolve-issues/scripts/plan-sync.test.mjs +0 -156
- package/skills/engineering/resolve-issues/scripts/preflight-models.test.mjs +0 -448
- package/skills/engineering/resolve-issues/scripts/ready-check.test.mjs +0 -217
- package/skills/engineering/resolve-issues/scripts/rehydrate.test.mjs +0 -91
- package/skills/engineering/resolve-issues/scripts/review-verdict.test.mjs +0 -1060
- package/skills/engineering/resolve-issues/scripts/size-check.test.mjs +0 -458
- package/skills/engineering/resolve-release/scripts/apply.test.mjs +0 -99
- package/skills/engineering/resolve-release/scripts/dark-flags.test.mjs +0 -100
- package/skills/engineering/resolve-release/scripts/flow.test.mjs +0 -340
- package/skills/engineering/resolve-release/scripts/green-head.test.mjs +0 -101
- package/skills/engineering/resolve-release/scripts/npm-version-absence.test.mjs +0 -426
- package/skills/engineering/resolve-release/scripts/plan-npm.test.mjs +0 -659
- package/skills/engineering/resolve-release/scripts/plan-toml-boundary.test.mjs +0 -317
- package/skills/engineering/resolve-release/scripts/plan-uv.test.mjs +0 -291
- package/skills/engineering/resolve-release/scripts/plan.test.mjs +0 -428
- package/skills/engineering/test-quality/scripts/lint-tests.test.mjs +0 -591
- package/skills/engineering/test-quality/scripts/mock-policy-config.test.mjs +0 -119
- package/skills/engineering/test-quality/scripts/persist-conventions.test.mjs +0 -104
- package/skills/risk-compliance/compliance-audit/scripts/compliance-audit.test.mjs +0 -14
- package/skills/risk-compliance/doc-authenticity/manual-invocation.test.mjs +0 -12
- package/skills/risk-compliance/doc-authenticity/scripts/validate_verdict.test.mjs +0 -212
- package/skills/risk-compliance/tld-detection/scripts/detect_ladders.test.mjs +0 -14
- package/skills/risk-compliance/tld-detection/scripts/detect_streams.test.mjs +0 -14
- package/skills/risk-compliance/tld-detection/tests/test_detect_ladders.py +0 -134
- package/skills/risk-compliance/tld-detection/tests/test_detect_streams.py +0 -520
|
@@ -1,829 +0,0 @@
|
|
|
1
|
-
// Tests for the team seed (model-catalog.md rule 7): seed read path, alias
|
|
2
|
-
// resolution, precedence, the read-only guarantee, and the rule-3
|
|
3
|
-
// never-proposes-an-undiscovered-model property. Separate file from
|
|
4
|
-
// model-catalog.test.mjs (which covers the per-user banked catalog) because
|
|
5
|
-
// the seed is a distinct, read-only artifact with its own invariants.
|
|
6
|
-
import { test } from 'node:test'
|
|
7
|
-
import assert from 'node:assert/strict'
|
|
8
|
-
import fs from 'node:fs'
|
|
9
|
-
import os from 'node:os'
|
|
10
|
-
import path from 'node:path'
|
|
11
|
-
import { fileURLToPath } from 'node:url'
|
|
12
|
-
import { spawnSync } from 'node:child_process'
|
|
13
|
-
|
|
14
|
-
import {
|
|
15
|
-
readSeed, findSeedEntry, seedTiersFor, seedTierAt, seedProposals,
|
|
16
|
-
resolveModelTiers, tierProposals, entryFromSeedConfirmation, defaultSeedPath,
|
|
17
|
-
readCatalog, writeCatalog, EFFORTS, TIERS, SLOT_ELIGIBLE_TIERS,
|
|
18
|
-
crossHarnessProposals, plainlyMatches, operatingPoints, CLASSIFY_STALE_AFTER_DAYS,
|
|
19
|
-
} from './model-catalog.mjs'
|
|
20
|
-
|
|
21
|
-
const SCRIPT = fileURLToPath(new URL('./bin/model-catalog.mjs', import.meta.url))
|
|
22
|
-
const tmpFile = (name = 'seed.json') => path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'model-catalog-seed-test-')), name)
|
|
23
|
-
|
|
24
|
-
function run(args, input) {
|
|
25
|
-
return spawnSync('node', [SCRIPT, ...args], { encoding: 'utf8', input })
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
// A small fixture mirroring the shipped seed's shape: two models that AGREE
|
|
29
|
-
// across harnesses, and one seeded from a single harness only.
|
|
30
|
-
const FIXTURE_SEED = {
|
|
31
|
-
seededAt: '2026-07-20',
|
|
32
|
-
models: [
|
|
33
|
-
{
|
|
34
|
-
canonical: 'gpt-5.6-sol',
|
|
35
|
-
family: 'openai',
|
|
36
|
-
ids: { 'claude-code': 'gpt-5.6-sol', pi: 'openai/gpt-5.6-sol' },
|
|
37
|
-
tiers: { 'claude-code': { high: 'frontier' }, pi: { high: 'frontier' } },
|
|
38
|
-
note: 'agrees everywhere',
|
|
39
|
-
},
|
|
40
|
-
{
|
|
41
|
-
canonical: 'kimi-k2.7-code',
|
|
42
|
-
family: 'moonshot',
|
|
43
|
-
ids: { 'claude-code': 'kimi-k2.7-code', pi: 'kimi-coding/kimi-for-coding' },
|
|
44
|
-
tiers: { 'claude-code': { high: 'scoped-coder' }, pi: { high: 'scoped-coder' } },
|
|
45
|
-
note: 'agrees everywhere',
|
|
46
|
-
},
|
|
47
|
-
{
|
|
48
|
-
canonical: 'opus',
|
|
49
|
-
family: 'anthropic',
|
|
50
|
-
ids: { 'claude-code': 'opus' },
|
|
51
|
-
tiers: { 'claude-code': { high: 'deep-reasoner' } },
|
|
52
|
-
note: 'claude-code only',
|
|
53
|
-
},
|
|
54
|
-
],
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
// ---- seed read path ----
|
|
58
|
-
|
|
59
|
-
test('readSeed returns null for an absent file (absent is normal, same contract as readCatalog)', () => {
|
|
60
|
-
assert.equal(readSeed(tmpFile()), null)
|
|
61
|
-
})
|
|
62
|
-
|
|
63
|
-
test('readSeed parses the seed JSON', () => {
|
|
64
|
-
const file = tmpFile()
|
|
65
|
-
fs.writeFileSync(file, JSON.stringify(FIXTURE_SEED))
|
|
66
|
-
assert.deepEqual(readSeed(file), FIXTURE_SEED)
|
|
67
|
-
})
|
|
68
|
-
|
|
69
|
-
// @covers package:harness-runtime
|
|
70
|
-
// @level integration
|
|
71
|
-
test('defaultSeedPath resolves to the shipped references/model-catalog-seed.json, and it parses', () => {
|
|
72
|
-
const p = defaultSeedPath()
|
|
73
|
-
assert.ok(p.endsWith(path.join('references', 'model-catalog-seed.json')))
|
|
74
|
-
const seed = readSeed(p)
|
|
75
|
-
assert.ok(seed, 'the shipped seed must exist and parse')
|
|
76
|
-
assert.ok(seed.seededAt)
|
|
77
|
-
assert.ok(Array.isArray(seed.models) && seed.models.length > 0)
|
|
78
|
-
})
|
|
79
|
-
|
|
80
|
-
test('the shipped seed keeps gpt-5.6-sol frontier at its measured high point only', () => {
|
|
81
|
-
const seed = readSeed(defaultSeedPath())
|
|
82
|
-
const sol = seed.models.find((m) => m.canonical === 'gpt-5.6-sol')
|
|
83
|
-
assert.ok(sol, 'seed must carry gpt-5.6-sol')
|
|
84
|
-
|
|
85
|
-
for (const [harness, id] of Object.entries({
|
|
86
|
-
'claude-code': 'gpt-5.6-sol',
|
|
87
|
-
codex: 'gpt-5.6-sol',
|
|
88
|
-
pi: 'openai/gpt-5.6-sol',
|
|
89
|
-
})) {
|
|
90
|
-
assert.deepEqual(sol.tiers[harness], { high: 'frontier' }, `gpt-5.6-sol@${harness}`)
|
|
91
|
-
const { proposals } = seedProposals(seed, harness, [id])
|
|
92
|
-
assert.deepEqual(proposals.map((p) => p.id), [id])
|
|
93
|
-
assert.deepEqual(proposals[0].points, [{ effort: 'high', tier: 'frontier', slotEligible: true }])
|
|
94
|
-
}
|
|
95
|
-
})
|
|
96
|
-
|
|
97
|
-
test('the shipped seed banks gpt-6-astra at its measured high point only', () => {
|
|
98
|
-
const seed = readSeed(defaultSeedPath())
|
|
99
|
-
const astra = seed.models.find((m) => m.canonical === 'gpt-6-astra')
|
|
100
|
-
assert.ok(astra, 'seed must carry the canonical gpt-6-astra id')
|
|
101
|
-
assert.ok(!seed.models.some((m) => m.canonical === 'gpt-6-astro'), 'the astro typo must not become an alias')
|
|
102
|
-
|
|
103
|
-
for (const [harness, id] of Object.entries({
|
|
104
|
-
'claude-code': 'gpt-6-astra',
|
|
105
|
-
codex: 'gpt-6-astra',
|
|
106
|
-
pi: 'openai/gpt-6-astra',
|
|
107
|
-
})) {
|
|
108
|
-
assert.equal(astra.ids[harness], id, `gpt-6-astra@${harness} id`)
|
|
109
|
-
assert.deepEqual(astra.tiers[harness], { high: 'frontier' }, `gpt-6-astra@${harness}`)
|
|
110
|
-
const { proposals } = seedProposals(seed, harness, [id])
|
|
111
|
-
assert.deepEqual(proposals[0].points, [{ effort: 'high', tier: 'frontier', slotEligible: true }])
|
|
112
|
-
}
|
|
113
|
-
})
|
|
114
|
-
|
|
115
|
-
test('the shipped seed banks gpt-6-sol as deep-reasoner at its vendor-measured high point only', () => {
|
|
116
|
-
const seed = readSeed(defaultSeedPath())
|
|
117
|
-
const sol6 = seed.models.find((m) => m.canonical === 'gpt-6-sol')
|
|
118
|
-
assert.ok(sol6, 'seed must carry gpt-6-sol')
|
|
119
|
-
// 65.3 @high from OpenAI's 2026-09-22 launch chart lands in the deep-reasoner band with
|
|
120
|
-
// gpt-5.5 (64.4±3.1) and grok-4.6 (65.2±1.5); its xhigh/max points are not banked, and it
|
|
121
|
-
// does not inherit gpt-5.6-sol's frontier seat (68.8 vs 72.7 at max).
|
|
122
|
-
for (const [harness, id] of Object.entries({ 'claude-code': 'gpt-6-sol', codex: 'gpt-6-sol', pi: 'openai/gpt-6-sol' })) {
|
|
123
|
-
assert.equal(sol6.ids[harness], id, `gpt-6-sol@${harness} id`)
|
|
124
|
-
assert.deepEqual(sol6.tiers[harness], { high: 'deep-reasoner' }, `gpt-6-sol@${harness}`)
|
|
125
|
-
const { proposals } = seedProposals(seed, harness, [id])
|
|
126
|
-
assert.deepEqual(proposals[0].points, [{ effort: 'high', tier: 'deep-reasoner', slotEligible: true }])
|
|
127
|
-
}
|
|
128
|
-
const sol56 = seed.models.find((m) => m.canonical === 'gpt-5.6-sol')
|
|
129
|
-
assert.deepEqual(sol56.tiers.pi, { high: 'frontier' }, 'gpt-5.6-sol keeps its frontier seat')
|
|
130
|
-
})
|
|
131
|
-
|
|
132
|
-
test('the shipped seed banks gpt-6.1-sol as frontier at its vendor-measured high point only', () => {
|
|
133
|
-
const seed = readSeed(defaultSeedPath())
|
|
134
|
-
const sol61 = seed.models.find((m) => m.canonical === 'gpt-6.1-sol')
|
|
135
|
-
assert.ok(sol61, 'seed must carry gpt-6.1-sol')
|
|
136
|
-
// 75.2 @high from OpenAI's 2026-09-29 launch lands in the frontier band with gpt-6-astra
|
|
137
|
-
// (73.2±3.4) and gpt-5.6-sol (69.4±1.4); its medium point (73) is the explorer's policy
|
|
138
|
-
// point, not banked.
|
|
139
|
-
for (const [harness, id] of Object.entries({ 'claude-code': 'gpt-6.1-sol', codex: 'gpt-6.1-sol', pi: 'openai/gpt-6.1-sol' })) {
|
|
140
|
-
assert.equal(sol61.ids[harness], id, `gpt-6.1-sol@${harness} id`)
|
|
141
|
-
assert.deepEqual(sol61.tiers[harness], { high: 'frontier' }, `gpt-6.1-sol@${harness}`)
|
|
142
|
-
const { proposals } = seedProposals(seed, harness, [id])
|
|
143
|
-
assert.deepEqual(proposals[0].points, [{ effort: 'high', tier: 'frontier', slotEligible: true }])
|
|
144
|
-
}
|
|
145
|
-
})
|
|
146
|
-
|
|
147
|
-
test('the shipped seed banks gpt-6-luna at max (measured) and at high (human classification)', () => {
|
|
148
|
-
const seed = readSeed(defaultSeedPath())
|
|
149
|
-
const luna6 = seed.models.find((m) => m.canonical === 'gpt-6-luna')
|
|
150
|
-
assert.ok(luna6, 'seed must carry gpt-6-luna')
|
|
151
|
-
// OpenAI published low/medium/max only (66.6 @max); rule 6 forbids reading max down to high, so the
|
|
152
|
-
// high point is a 2026-09-23 human classification (the rush routine ladder runs Luna at high because
|
|
153
|
-
// max is too slow). Both points are balanced-coder; the note says which one is measured.
|
|
154
|
-
for (const [harness, id] of Object.entries({ 'claude-code': 'gpt-6-luna', codex: 'gpt-6-luna', pi: 'openai/gpt-6-luna' })) {
|
|
155
|
-
assert.equal(luna6.ids[harness], id, `gpt-6-luna@${harness} id`)
|
|
156
|
-
assert.deepEqual(luna6.tiers[harness], { high: 'balanced-coder', max: 'balanced-coder' }, `gpt-6-luna@${harness}`)
|
|
157
|
-
}
|
|
158
|
-
assert.match(luna6.note, /HUMAN CLASSIFICATION/, 'the note must say the high point is not benchmark-backed')
|
|
159
|
-
})
|
|
160
|
-
|
|
161
|
-
// ---- alias resolution (both directions: id -> entry, entry -> per-harness tier) ----
|
|
162
|
-
|
|
163
|
-
test('findSeedEntry resolves a harness-specific alias id to the shared canonical entry', () => {
|
|
164
|
-
// The confirmed alias (issue #370): kimi-k2.7-code (claude-code) = kimi-coding/kimi-for-coding (pi).
|
|
165
|
-
const viaClaudeCode = findSeedEntry(FIXTURE_SEED, 'claude-code', 'kimi-k2.7-code')
|
|
166
|
-
const viaPi = findSeedEntry(FIXTURE_SEED, 'pi', 'kimi-coding/kimi-for-coding')
|
|
167
|
-
assert.equal(viaClaudeCode.canonical, 'kimi-k2.7-code')
|
|
168
|
-
assert.equal(viaPi.canonical, 'kimi-k2.7-code')
|
|
169
|
-
assert.equal(viaClaudeCode, viaPi) // same entry object, reached from either harness's id
|
|
170
|
-
})
|
|
171
|
-
|
|
172
|
-
test('findSeedEntry returns null for an id the seed does not know, or a harness the entry does not cover', () => {
|
|
173
|
-
assert.equal(findSeedEntry(FIXTURE_SEED, 'claude-code', 'no-such-model'), null)
|
|
174
|
-
assert.equal(findSeedEntry(FIXTURE_SEED, 'codex', 'kimi-k2.7-code'), null) // no codex id on this entry
|
|
175
|
-
})
|
|
176
|
-
|
|
177
|
-
test('a plainly-normalizable id that was NOT given a confirmed alias does not resolve — no guessing', () => {
|
|
178
|
-
// kimi-k3 (claude-code) vs kimi-coding/k3 (pi) do not plainly normalize (stripping
|
|
179
|
-
// "kimi-coding/" gives "k3", not "kimi-k3") and were never confirmed as a pair
|
|
180
|
-
// (issue #370) — the seed keeps them as two separate entries, so a lookup for
|
|
181
|
-
// one harness's id must never resolve to the other harness's entry.
|
|
182
|
-
const seed = {
|
|
183
|
-
seededAt: '2026-07-20',
|
|
184
|
-
models: [
|
|
185
|
-
{ canonical: 'kimi-k3', family: 'moonshot', ids: { 'claude-code': 'kimi-k3' }, tiers: { 'claude-code': { max: 'frontier' } } },
|
|
186
|
-
{ canonical: 'kimi-coding/k3', family: 'moonshot', ids: { pi: 'kimi-coding/k3' }, tiers: { pi: { high: 'deep-reasoner' } } },
|
|
187
|
-
],
|
|
188
|
-
}
|
|
189
|
-
assert.equal(findSeedEntry(seed, 'pi', 'kimi-k3'), null)
|
|
190
|
-
assert.equal(findSeedEntry(seed, 'claude-code', 'kimi-coding/k3'), null)
|
|
191
|
-
})
|
|
192
|
-
|
|
193
|
-
// ---- per-harness tiers stay PER-HARNESS, never collapsed ----
|
|
194
|
-
|
|
195
|
-
test('seedTiersFor/seedTierAt resolve the tier for the RIGHT harness', () => {
|
|
196
|
-
assert.deepEqual(seedTiersFor(FIXTURE_SEED, 'claude-code', 'kimi-k2.7-code'), { high: 'scoped-coder' })
|
|
197
|
-
assert.deepEqual(seedTiersFor(FIXTURE_SEED, 'pi', 'kimi-coding/kimi-for-coding'), { high: 'scoped-coder' })
|
|
198
|
-
assert.equal(seedTierAt(FIXTURE_SEED, 'claude-code', 'kimi-k2.7-code', 'high'), 'scoped-coder')
|
|
199
|
-
assert.equal(seedTierAt(FIXTURE_SEED, 'pi', 'kimi-coding/kimi-for-coding', 'high'), 'scoped-coder')
|
|
200
|
-
})
|
|
201
|
-
|
|
202
|
-
test('seedTierAt returns null for an effort the entry does not cover, or a harness it was not seeded from', () => {
|
|
203
|
-
assert.equal(seedTierAt(FIXTURE_SEED, 'claude-code', 'kimi-k2.7-code', 'max'), null)
|
|
204
|
-
assert.equal(seedTierAt(FIXTURE_SEED, 'codex', 'opus', 'high'), null) // opus is claude-code only
|
|
205
|
-
})
|
|
206
|
-
|
|
207
|
-
test('the shipped seed keeps its remaining per-harness disagreements (not collapsed)', () => {
|
|
208
|
-
const seed = readSeed(defaultSeedPath())
|
|
209
|
-
// Zero shared models now disagree per harness. deepseek-v4-pro was the last
|
|
210
|
-
// (claude-code high:scoped-coder vs pi high:balanced-coder) and left on
|
|
211
|
-
// 2026-08-13 when 0813 GA collapsed it to an agreeing max:deep-reasoner point.
|
|
212
|
-
// Pinned so a silent drift back into disagreement has to be deliberate.
|
|
213
|
-
const shared = seed.models.filter((m) => Object.keys(m.tiers || {}).length > 1)
|
|
214
|
-
for (const entry of shared) {
|
|
215
|
-
const maps = Object.values(entry.tiers)
|
|
216
|
-
assert.ok(maps.every((t) => JSON.stringify(t) === JSON.stringify(maps[0])), `${entry.canonical} must agree across harnesses`)
|
|
217
|
-
}
|
|
218
|
-
const pro = seed.models.find((m) => m.canonical === 'deepseek-v4-pro')
|
|
219
|
-
assert.ok(pro, 'seed must carry deepseek-v4-pro')
|
|
220
|
-
for (const harness of ['claude-code', 'pi']) {
|
|
221
|
-
assert.deepEqual(pro.tiers[harness], { max: 'deep-reasoner' }, `deepseek-v4-pro@${harness}`)
|
|
222
|
-
}
|
|
223
|
-
// grok-4.5 left this set on 2026-07-22 (pi demoted deep-reasoner -> balanced-coder);
|
|
224
|
-
// pinned so a re-promotion has to be a deliberate edit, not a silent drift back.
|
|
225
|
-
const grok = seed.models.find((m) => m.canonical === 'grok-4.5')
|
|
226
|
-
assert.equal(grok.tiers['claude-code'].high, 'balanced-coder')
|
|
227
|
-
assert.equal(grok.tiers.pi.high, 'balanced-coder')
|
|
228
|
-
|
|
229
|
-
// grok-4.6 added 2026-08-13 from vendor DeepSWE v1.1 @high 65.9. Separate entry,
|
|
230
|
-
// not a reband of 4.5. Pinned so a silent collapse into 4.5 or a drop to
|
|
231
|
-
// balanced-coder has to be deliberate.
|
|
232
|
-
const grok46 = seed.models.find((m) => m.canonical === 'grok-4.6')
|
|
233
|
-
assert.ok(grok46, 'seed must carry grok-4.6')
|
|
234
|
-
for (const harness of ['claude-code', 'pi']) {
|
|
235
|
-
assert.deepEqual(grok46.tiers[harness], { high: 'deep-reasoner' }, `grok-4.6@${harness}`)
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
// grok-4.7 added 2026-09-23 from SpaceXAI's launch chart: vendor DeepSWE v1.1 71.0 @high
|
|
239
|
-
// (asterisked "high effort"), not yet on the official board. Above gpt-5.6-sol's official
|
|
240
|
-
// 69.4±1.4 high point and inside gpt-6-astra's 73.2±3.4, so frontier at high; a separate
|
|
241
|
-
// entry that does not re-tier grok-4.6. Pinned so a silent drop to deep-reasoner or a
|
|
242
|
-
// score-chasing xhigh point has to be a deliberate edit when Datacurve publishes the run.
|
|
243
|
-
const grok47 = seed.models.find((m) => m.canonical === 'grok-4.7')
|
|
244
|
-
assert.ok(grok47, 'seed must carry grok-4.7')
|
|
245
|
-
assert.equal(grok47.family, 'xai')
|
|
246
|
-
for (const [harness, id] of Object.entries({ 'claude-code': 'grok-4.7', pi: 'xai/grok-4.7' })) {
|
|
247
|
-
assert.equal(grok47.ids[harness], id, `grok-4.7@${harness} id`)
|
|
248
|
-
assert.deepEqual(grok47.tiers[harness], { high: 'frontier' }, `grok-4.7@${harness}`)
|
|
249
|
-
const { proposals } = seedProposals(seed, harness, [id])
|
|
250
|
-
assert.deepEqual(proposals[0].points, [{ effort: 'high', tier: 'frontier', slotEligible: true }])
|
|
251
|
-
}
|
|
252
|
-
assert.ok(!grok47.ids.codex, 'codex holds no grok entry')
|
|
253
|
-
assert.deepEqual(grok46.tiers.pi, { high: 'deep-reasoner' }, 'grok-4.7 does not re-tier grok-4.6')
|
|
254
|
-
|
|
255
|
-
// deepseek-flash moved 2026-08-13 from high:balanced-coder to max:balanced-coder.
|
|
256
|
-
// The 55.4 figure previously cited as @high is the vendor max point (official board:
|
|
257
|
-
// [max] 53%±4). Pinned so a silent restore of the high point has to be deliberate.
|
|
258
|
-
const flash = seed.models.find((m) => m.canonical === 'deepseek-flash')
|
|
259
|
-
assert.ok(flash, 'seed must carry deepseek-flash')
|
|
260
|
-
for (const harness of ['claude-code', 'codex', 'pi']) {
|
|
261
|
-
assert.deepEqual(flash.tiers[harness], { max: 'balanced-coder' }, `deepseek-flash@${harness}`)
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
// DeepSWE measured luna at high (44.2%±2.9), so its higher-effort results are not
|
|
265
|
-
// banked. Pinned so a score-chasing xhigh/max point cannot quietly return.
|
|
266
|
-
// mimo-v2.5-pro left this set on 2026-08-02: pi downgraded balanced-coder -> scoped-coder,
|
|
267
|
-
// so both its harnesses now agree. Pinned like grok-4.5 and luna, because it was this file's
|
|
268
|
-
// worked example of a harness disagreement and a silent drift back would be easy to miss.
|
|
269
|
-
const mimo = seed.models.find((m) => m.canonical === 'mimo-v2.5-pro')
|
|
270
|
-
for (const harness of ['claude-code', 'pi']) {
|
|
271
|
-
assert.deepEqual(mimo.tiers[harness], { high: 'scoped-coder' }, `mimo-v2.5-pro@${harness}`)
|
|
272
|
-
}
|
|
273
|
-
|
|
274
|
-
// mimo-v2.6-pro added 2026-09-23 from Xiaomi's release table: vendor DeepSWE v1.1 71.9
|
|
275
|
-
// (own run of the public harness), not on the official board. Above gpt-5.6-sol's official
|
|
276
|
-
// 69.4±1.4 high point and inside gpt-6-astra's 73.2±3.4, so frontier at high — `high` being
|
|
277
|
-
// the label for the model's single thinking-on point (the API has no graded efforts). A
|
|
278
|
-
// separate entry that does not re-tier mimo-v2.5-pro. Pinned so a silent drop to
|
|
279
|
-
// deep-reasoner, or a collapse into 2.5-pro, has to be a deliberate edit.
|
|
280
|
-
const mimo26 = seed.models.find((m) => m.canonical === 'mimo-v2.6-pro')
|
|
281
|
-
assert.ok(mimo26, 'seed must carry mimo-v2.6-pro')
|
|
282
|
-
assert.equal(mimo26.family, 'xiaomi')
|
|
283
|
-
for (const [harness, id] of Object.entries({ 'claude-code': 'mimo-v2.6-pro', pi: 'xiaomi/mimo-v2.6-pro' })) {
|
|
284
|
-
assert.equal(mimo26.ids[harness], id, `mimo-v2.6-pro@${harness} id`)
|
|
285
|
-
assert.deepEqual(mimo26.tiers[harness], { high: 'frontier' }, `mimo-v2.6-pro@${harness}`)
|
|
286
|
-
const { proposals } = seedProposals(seed, harness, [id])
|
|
287
|
-
assert.deepEqual(proposals[0].points, [{ effort: 'high', tier: 'frontier', slotEligible: true }])
|
|
288
|
-
}
|
|
289
|
-
assert.ok(!mimo26.ids.codex, 'codex holds no MiMo entry')
|
|
290
|
-
assert.deepEqual(mimo.tiers.pi, { high: 'scoped-coder' }, 'mimo-v2.6-pro does not re-tier mimo-v2.5-pro')
|
|
291
|
-
|
|
292
|
-
const luna = seed.models.find((m) => m.canonical === 'gpt-5.6-luna')
|
|
293
|
-
for (const harness of ['claude-code', 'codex', 'pi']) {
|
|
294
|
-
assert.deepEqual(
|
|
295
|
-
luna.tiers[harness],
|
|
296
|
-
{ high: 'scoped-coder' },
|
|
297
|
-
`gpt-5.6-luna@${harness}`,
|
|
298
|
-
)
|
|
299
|
-
}
|
|
300
|
-
|
|
301
|
-
// Opus 4.8's 51.8%±4.6 high point overlaps the balanced-coder reference band;
|
|
302
|
-
// its separate max result must not promote the default operating point.
|
|
303
|
-
const opus = seed.models.find((m) => m.canonical === 'opus')
|
|
304
|
-
assert.deepEqual(opus.tiers['claude-code'], { high: 'balanced-coder' })
|
|
305
|
-
})
|
|
306
|
-
|
|
307
|
-
// ---- rule 3, mechanized: never proposes a model this run's discovery didn't yield ----
|
|
308
|
-
|
|
309
|
-
test('seedProposals only proposes ids present in discoveredIds — the seed carrying an id is not enough', () => {
|
|
310
|
-
const { proposals } = seedProposals(FIXTURE_SEED, 'claude-code', ['gpt-5.6-sol']) // opus, kimi-k2.7-code NOT discovered
|
|
311
|
-
assert.deepEqual(proposals.map((p) => p.id), ['gpt-5.6-sol'])
|
|
312
|
-
})
|
|
313
|
-
|
|
314
|
-
test('seedProposals with an EMPTY discovery set proposes nothing, however much the seed covers', () => {
|
|
315
|
-
assert.deepEqual(seedProposals(FIXTURE_SEED, 'claude-code', []).proposals, [])
|
|
316
|
-
})
|
|
317
|
-
|
|
318
|
-
test('seedProposals never proposes an id under the WRONG harness even if discovered there', () => {
|
|
319
|
-
// pi's discovery of "kimi-coding/kimi-for-coding" must not surface the claude-code
|
|
320
|
-
// id "kimi-k2.7-code", and vice versa — availability is per-harness (rule 3).
|
|
321
|
-
const { proposals } = seedProposals(FIXTURE_SEED, 'claude-code', ['kimi-coding/kimi-for-coding'])
|
|
322
|
-
assert.deepEqual(proposals, [])
|
|
323
|
-
})
|
|
324
|
-
|
|
325
|
-
test('seedProposals skips an entry that has no tiers for the queried harness even if the id somehow matched', () => {
|
|
326
|
-
const seed = { seededAt: '2026-07-20', models: [{ canonical: 'x', family: 'f', ids: { 'claude-code': 'x' }, tiers: {} }] }
|
|
327
|
-
assert.deepEqual(seedProposals(seed, 'claude-code', ['x']).proposals, [])
|
|
328
|
-
})
|
|
329
|
-
|
|
330
|
-
test('CLI seed-proposals: an undiscovered id never appears in the output', () => {
|
|
331
|
-
const seedFile = tmpFile()
|
|
332
|
-
fs.writeFileSync(seedFile, JSON.stringify(FIXTURE_SEED))
|
|
333
|
-
const res = run(['seed-proposals', '--seed-file', seedFile, '--harness', 'claude-code', '--seen', 'opus,definitely-not-real', '--file', tmpFile('catalog.json')])
|
|
334
|
-
assert.equal(res.status, 0)
|
|
335
|
-
const { proposals } = JSON.parse(res.stdout)
|
|
336
|
-
assert.deepEqual(proposals.map((p) => p.id), ['opus'])
|
|
337
|
-
assert.equal(proposals[0].citation, 'tier from team seed: opus')
|
|
338
|
-
})
|
|
339
|
-
|
|
340
|
-
// ---- rule 7 precedence, mechanized: a banked entry suppresses the seed PER MODEL ----
|
|
341
|
-
|
|
342
|
-
test('seedProposals: a model already banked in this harness gets NO seed proposal — the catalog outranks the seed', () => {
|
|
343
|
-
const catalog = { harness: 'claude-code', models: [{ id: 'opus', family: 'anthropic', tiers: { high: 'frontier' } }] }
|
|
344
|
-
const { proposals, suppressed } = seedProposals(FIXTURE_SEED, 'claude-code', ['opus', 'gpt-5.6-sol'], catalog)
|
|
345
|
-
assert.deepEqual(proposals.map((p) => p.id), ['gpt-5.6-sol'])
|
|
346
|
-
assert.deepEqual(suppressed.map((s) => s.id), ['opus'])
|
|
347
|
-
})
|
|
348
|
-
|
|
349
|
-
test('seedProposals: a DELETED operating point is not restored from the seed — suppression is per model, never per effort', () => {
|
|
350
|
-
// The terra regression: the human banked {high} after deciding the max point
|
|
351
|
-
// wasn't real. A per-effort merge would see no banked tier at `max`, read the
|
|
352
|
-
// deletion as silence, and hand `max: deep-reasoner` back every run.
|
|
353
|
-
const seed = {
|
|
354
|
-
seededAt: '2026-07-20',
|
|
355
|
-
models: [{
|
|
356
|
-
canonical: 'terra',
|
|
357
|
-
family: 'openai',
|
|
358
|
-
ids: { pi: 'openai/terra' },
|
|
359
|
-
tiers: { pi: { high: 'balanced-coder', max: 'deep-reasoner' } },
|
|
360
|
-
}],
|
|
361
|
-
}
|
|
362
|
-
const catalog = { harness: 'pi', models: [{ id: 'openai/terra', family: 'openai', tiers: { high: 'balanced-coder' } }] }
|
|
363
|
-
const { proposals, suppressed } = seedProposals(seed, 'pi', ['openai/terra'], catalog)
|
|
364
|
-
|
|
365
|
-
assert.deepEqual(proposals, [], 'no proposal at all for a banked model')
|
|
366
|
-
assert.equal(suppressed[0].disagrees, true, 'the seed claiming a point the human dropped is a disagreement')
|
|
367
|
-
assert.deepEqual(suppressed[0].banked, { high: 'balanced-coder' })
|
|
368
|
-
assert.deepEqual(suppressed[0].seed, { high: 'balanced-coder', max: 'deep-reasoner' })
|
|
369
|
-
})
|
|
370
|
-
|
|
371
|
-
test('seedProposals: suppression where seed and catalog AGREE is not flagged as a disagreement', () => {
|
|
372
|
-
const catalog = { models: [{ id: 'opus', family: 'anthropic', tiers: { high: 'deep-reasoner' } }] }
|
|
373
|
-
const { suppressed } = seedProposals(FIXTURE_SEED, 'claude-code', ['opus'], catalog)
|
|
374
|
-
assert.equal(suppressed[0].disagrees, false)
|
|
375
|
-
})
|
|
376
|
-
|
|
377
|
-
test('seedProposals: an entry banked with NO operating points does not suppress the seed pre-fill', () => {
|
|
378
|
-
// tiers:{} passes a truthiness guard but classifies nothing — check the points.
|
|
379
|
-
const catalog = { models: [{ id: 'opus', family: 'anthropic', tiers: {} }] }
|
|
380
|
-
const { proposals } = seedProposals(FIXTURE_SEED, 'claude-code', ['opus'], catalog)
|
|
381
|
-
assert.deepEqual(proposals.map((p) => p.id), ['opus'])
|
|
382
|
-
})
|
|
383
|
-
|
|
384
|
-
test('seedProposals: a legacy single-point {tier} entry still suppresses (banked is read through toTiers)', () => {
|
|
385
|
-
const catalog = { models: [{ id: 'opus', family: 'anthropic', tier: 'prior-frontier' }] }
|
|
386
|
-
const { proposals, suppressed } = seedProposals(FIXTURE_SEED, 'claude-code', ['opus'], catalog)
|
|
387
|
-
assert.deepEqual(proposals, [])
|
|
388
|
-
assert.deepEqual(suppressed[0].banked, { high: 'deep-reasoner' }) // legacy name normalized
|
|
389
|
-
})
|
|
390
|
-
|
|
391
|
-
test('seedProposals with NO catalog argument still proposes — an absent catalog suppresses nothing', () => {
|
|
392
|
-
const { proposals, suppressed } = seedProposals(FIXTURE_SEED, 'claude-code', ['opus'])
|
|
393
|
-
assert.deepEqual(proposals.map((p) => p.id), ['opus'])
|
|
394
|
-
assert.deepEqual(suppressed, [])
|
|
395
|
-
})
|
|
396
|
-
|
|
397
|
-
// ---- build 1: date-aware disagreement (seedNewer) — the grok-4.5 class ----
|
|
398
|
-
// FIXTURE_SEED.seededAt is 2026-07-20; seed says opus=deep-reasoner.
|
|
399
|
-
test('seedProposals: a disagreement where the seed is NEWER than the banked confirmation sets seedNewer', () => {
|
|
400
|
-
const catalog = { models: [{ id: 'opus', family: 'anthropic', confirmedAt: '2026-07-19', tiers: { high: 'balanced-coder' } }] }
|
|
401
|
-
const { suppressed } = seedProposals(FIXTURE_SEED, 'claude-code', ['opus'], catalog)
|
|
402
|
-
assert.equal(suppressed[0].disagrees, true)
|
|
403
|
-
assert.equal(suppressed[0].seedNewer, true, 'seed seededAt 07-20 is newer than banked confirmedAt 07-19')
|
|
404
|
-
assert.equal(suppressed[0].seededAt, '2026-07-20')
|
|
405
|
-
assert.equal(suppressed[0].bankedConfirmedAt, '2026-07-19')
|
|
406
|
-
})
|
|
407
|
-
|
|
408
|
-
test('seedProposals: seedNewer is false when the banked confirmation is at or after the seed date', () => {
|
|
409
|
-
const catalog = { models: [{ id: 'opus', family: 'anthropic', confirmedAt: '2026-07-25', tiers: { high: 'balanced-coder' } }] }
|
|
410
|
-
const { suppressed } = seedProposals(FIXTURE_SEED, 'claude-code', ['opus'], catalog)
|
|
411
|
-
assert.equal(suppressed[0].disagrees, true)
|
|
412
|
-
assert.equal(suppressed[0].seedNewer, false, 'a banked confirmation after the seed date is not overruled')
|
|
413
|
-
})
|
|
414
|
-
|
|
415
|
-
test('seedProposals: seedNewer is false when seed and banked AGREE, however old the banked entry', () => {
|
|
416
|
-
const catalog = { models: [{ id: 'opus', family: 'anthropic', confirmedAt: '2020-01-01', tiers: { high: 'deep-reasoner' } }] }
|
|
417
|
-
const { suppressed } = seedProposals(FIXTURE_SEED, 'claude-code', ['opus'], catalog)
|
|
418
|
-
assert.equal(suppressed[0].disagrees, false)
|
|
419
|
-
assert.equal(suppressed[0].seedNewer, false)
|
|
420
|
-
})
|
|
421
|
-
|
|
422
|
-
test('CLI seed-proposals escalates the citation to RE-CONFIRM when the seed is newer than the banked entry', () => {
|
|
423
|
-
const seedFile = tmpFile('seed.json')
|
|
424
|
-
fs.writeFileSync(seedFile, JSON.stringify(FIXTURE_SEED)) // seededAt 2026-07-20, opus=deep-reasoner
|
|
425
|
-
const catalogFile = tmpFile('catalog.json')
|
|
426
|
-
writeCatalog(catalogFile, { harness: 'claude-code', models: [{ id: 'opus', family: 'anthropic', confirmedAt: '2026-07-19', tiers: { high: 'balanced-coder' } }] })
|
|
427
|
-
const res = run(['seed-proposals', '--seed-file', seedFile, '--file', catalogFile, '--harness', 'claude-code', '--seen', 'opus'])
|
|
428
|
-
assert.equal(res.status, 0)
|
|
429
|
-
const { suppressed } = JSON.parse(res.stdout)
|
|
430
|
-
assert.match(suppressed[0].citation, /NEWER than your banked opus/)
|
|
431
|
-
assert.match(suppressed[0].citation, /RE-CONFIRM/)
|
|
432
|
-
assert.doesNotMatch(suppressed[0].citation, /Refresh the seed if yours is the better answer/, 'the stale-banked case must not print the default catalog-wins nudge')
|
|
433
|
-
})
|
|
434
|
-
|
|
435
|
-
test('CLI seed-proposals reads the harness catalog itself and reports the disagreement in the citation', () => {
|
|
436
|
-
const seedFile = tmpFile('seed.json')
|
|
437
|
-
fs.writeFileSync(seedFile, JSON.stringify(FIXTURE_SEED))
|
|
438
|
-
const catalogFile = tmpFile('catalog.json')
|
|
439
|
-
writeCatalog(catalogFile, { harness: 'claude-code', models: [{ id: 'opus', family: 'anthropic', tiers: { high: 'balanced-coder' } }] })
|
|
440
|
-
|
|
441
|
-
const res = run(['seed-proposals', '--seed-file', seedFile, '--file', catalogFile, '--harness', 'claude-code', '--seen', 'opus'])
|
|
442
|
-
assert.equal(res.status, 0)
|
|
443
|
-
const { proposals, suppressed } = JSON.parse(res.stdout)
|
|
444
|
-
assert.deepEqual(proposals, [])
|
|
445
|
-
assert.match(suppressed[0].citation, /team seed disagrees with your banked opus/)
|
|
446
|
-
assert.match(suppressed[0].citation, /your catalog wins/)
|
|
447
|
-
})
|
|
448
|
-
|
|
449
|
-
// ---- the shipped seed's own invariants ----
|
|
450
|
-
|
|
451
|
-
test('the shipped seed is internally well-formed: known efforts and tiers, ids covering every tiered harness', () => {
|
|
452
|
-
const seed = readSeed(defaultSeedPath())
|
|
453
|
-
for (const entry of seed.models) {
|
|
454
|
-
assert.ok(entry.canonical, 'every entry needs a canonical name')
|
|
455
|
-
assert.ok(entry.family, `${entry.canonical}: family is required`)
|
|
456
|
-
assert.ok(entry.ids && Object.keys(entry.ids).length, `${entry.canonical}: ids map is required`)
|
|
457
|
-
for (const [harness, tiers] of Object.entries(entry.tiers || {})) {
|
|
458
|
-
assert.ok(entry.ids[harness], `${entry.canonical}: tiers for "${harness}" with no id under that harness`)
|
|
459
|
-
assert.ok(Object.keys(tiers).length, `${entry.canonical}: empty tiers for "${harness}" says nothing`)
|
|
460
|
-
for (const [effort, tier] of Object.entries(tiers)) {
|
|
461
|
-
assert.ok(EFFORTS.includes(effort), `${entry.canonical}@${harness}: effort "${effort}" not in ${EFFORTS.join('|')}`)
|
|
462
|
-
assert.ok(TIERS.includes(tier), `${entry.canonical}@${harness}: tier "${tier}" not in ${TIERS.join('|')}`)
|
|
463
|
-
}
|
|
464
|
-
}
|
|
465
|
-
}
|
|
466
|
-
})
|
|
467
|
-
|
|
468
|
-
test('the shipped seed never maps one harness id to two entries — an ambiguous alias would make lookup order-dependent', () => {
|
|
469
|
-
const seed = readSeed(defaultSeedPath())
|
|
470
|
-
const seen = new Map()
|
|
471
|
-
for (const entry of seed.models) {
|
|
472
|
-
for (const [harness, id] of Object.entries(entry.ids || {})) {
|
|
473
|
-
const key = `${harness}\u0000${id}`
|
|
474
|
-
assert.equal(seen.get(key), undefined, `${harness}:${id} is claimed by both "${seen.get(key)}" and "${entry.canonical}"`)
|
|
475
|
-
seen.set(key, entry.canonical)
|
|
476
|
-
}
|
|
477
|
-
}
|
|
478
|
-
})
|
|
479
|
-
|
|
480
|
-
test('CLI seed: exits 3 with a clear message when the seed file is absent', () => {
|
|
481
|
-
const res = run(['seed', '--seed-file', tmpFile()])
|
|
482
|
-
assert.equal(res.status, 3)
|
|
483
|
-
assert.match(res.stderr, /no seed at/)
|
|
484
|
-
})
|
|
485
|
-
|
|
486
|
-
test('CLI seed: prints the shipped seed by default (no --seed-file needed)', () => {
|
|
487
|
-
const res = run(['seed'])
|
|
488
|
-
assert.equal(res.status, 0)
|
|
489
|
-
const seed = JSON.parse(res.stdout)
|
|
490
|
-
assert.ok(seed.seededAt)
|
|
491
|
-
assert.ok(seed.models.length >= 16) // issue #370 shipped 15; grok-4.6 added 2026-08-13
|
|
492
|
-
})
|
|
493
|
-
|
|
494
|
-
// ---- precedence: own banked entry -> cross-harness -> team seed -> discovery -> unconfirmed ----
|
|
495
|
-
//
|
|
496
|
-
// resolveTierPrecedence (scalar tier per source, resolved per model+effort) was
|
|
497
|
-
// replaced by resolveModelTiers (whole tiers maps, resolved per MODEL) on
|
|
498
|
-
// 2026-07-20. Its five tests went with it: they proved the old shape computed
|
|
499
|
-
// correctly, which was never in doubt — nothing called it, and its signature
|
|
500
|
-
// invited exactly the per-effort merge its own comment forbade.
|
|
501
|
-
|
|
502
|
-
test('resolveModelTiers: a banked entry wins over everything, including the seed', () => {
|
|
503
|
-
const r = resolveModelTiers({
|
|
504
|
-
banked: { high: 'frontier' },
|
|
505
|
-
crossHarness: { tiers: { high: 'balanced-coder' }, citation: 'x' },
|
|
506
|
-
seed: { tiers: { high: 'scoped-coder' }, citation: 'y' },
|
|
507
|
-
discoveryTiers: { high: 'deep-reasoner' },
|
|
508
|
-
})
|
|
509
|
-
assert.deepEqual(r.tiers, { high: 'frontier' })
|
|
510
|
-
assert.equal(r.source, 'banked')
|
|
511
|
-
})
|
|
512
|
-
|
|
513
|
-
test('resolveModelTiers: cross-harness (rule 5) beats the team seed, and carries its citation', () => {
|
|
514
|
-
const r = resolveModelTiers({
|
|
515
|
-
crossHarness: { tiers: { high: 'balanced-coder' }, citation: 'tier from codex catalog: x' },
|
|
516
|
-
seed: { tiers: { high: 'scoped-coder' }, citation: 'tier from team seed: x' },
|
|
517
|
-
})
|
|
518
|
-
assert.equal(r.source, 'cross-harness')
|
|
519
|
-
assert.equal(r.citation, 'tier from codex catalog: x')
|
|
520
|
-
})
|
|
521
|
-
|
|
522
|
-
test('resolveModelTiers: the seed pre-fills only when nothing stronger exists', () => {
|
|
523
|
-
const r = resolveModelTiers({ seed: { tiers: { high: 'scoped-coder' }, citation: 'c' }, discoveryTiers: { high: 'deep-reasoner' } })
|
|
524
|
-
assert.equal(r.source, 'team-seed')
|
|
525
|
-
})
|
|
526
|
-
|
|
527
|
-
test('resolveModelTiers: discovery evidence is the fallback below the seed; nothing at all is unconfirmed', () => {
|
|
528
|
-
assert.equal(resolveModelTiers({ discoveryTiers: { high: 'balanced-coder' } }).source, 'discovery')
|
|
529
|
-
assert.equal(resolveModelTiers({}).source, 'unconfirmed')
|
|
530
|
-
assert.equal(resolveModelTiers().source, 'unconfirmed')
|
|
531
|
-
assert.deepEqual(resolveModelTiers().tiers, {})
|
|
532
|
-
})
|
|
533
|
-
|
|
534
|
-
test('resolveModelTiers: an EMPTY tiers map does not win — it classifies nothing', () => {
|
|
535
|
-
// The trap `bank` already guards: `tiers: {}` passes a truthiness check while
|
|
536
|
-
// saying nothing, so a banked-but-empty entry must fall through, not shadow
|
|
537
|
-
// the seed with a blank answer.
|
|
538
|
-
const r = resolveModelTiers({ banked: {}, seed: { tiers: { high: 'frontier' }, citation: 'c' } })
|
|
539
|
-
assert.equal(r.source, 'team-seed')
|
|
540
|
-
})
|
|
541
|
-
|
|
542
|
-
test('resolveModelTiers cannot express a per-effort merge — the shape is the safeguard', () => {
|
|
543
|
-
// The regression that motivated the replacement: a human banked {high} after
|
|
544
|
-
// dropping a max point. Whatever the seed claims at max, the banked map wins
|
|
545
|
-
// WHOLE, so the deleted point cannot come back.
|
|
546
|
-
const r = resolveModelTiers({
|
|
547
|
-
banked: { high: 'balanced-coder' },
|
|
548
|
-
seed: { tiers: { high: 'balanced-coder', max: 'deep-reasoner' }, citation: 'c' },
|
|
549
|
-
})
|
|
550
|
-
assert.deepEqual(r.tiers, { high: 'balanced-coder' })
|
|
551
|
-
assert.equal(r.tiers.max, undefined, 'a dropped operating point must not reappear from a weaker source')
|
|
552
|
-
})
|
|
553
|
-
|
|
554
|
-
// ---- a seed entry is never confirmedAt; confirming banks a NORMAL entry into the user's own file ----
|
|
555
|
-
|
|
556
|
-
test('entryFromSeedConfirmation produces a normal bankable entry, stamped fresh, never carrying seed provenance as confirmedAt', () => {
|
|
557
|
-
const out = entryFromSeedConfirmation(FIXTURE_SEED, 'claude-code', 'kimi-k2.7-code', '2026-07-20')
|
|
558
|
-
assert.deepEqual(out, { id: 'kimi-k2.7-code', family: 'moonshot', tiers: { high: 'scoped-coder' }, confirmedAt: '2026-07-20', lastSeenAt: '2026-07-20' })
|
|
559
|
-
})
|
|
560
|
-
|
|
561
|
-
test('entryFromSeedConfirmation resolves the PI-side alias id to the same canonical entry, banking the PI tier', () => {
|
|
562
|
-
const out = entryFromSeedConfirmation(FIXTURE_SEED, 'pi', 'kimi-coding/kimi-for-coding', '2026-07-20')
|
|
563
|
-
assert.deepEqual(out.tiers, { high: 'scoped-coder' }) // pi's tier, not claude-code's
|
|
564
|
-
})
|
|
565
|
-
|
|
566
|
-
test('entryFromSeedConfirmation throws for an id the seed does not cover under that harness', () => {
|
|
567
|
-
assert.throws(() => entryFromSeedConfirmation(FIXTURE_SEED, 'codex', 'kimi-k2.7-code', '2026-07-20'), /no seed entry/)
|
|
568
|
-
assert.throws(() => entryFromSeedConfirmation(FIXTURE_SEED, 'codex', 'opus', '2026-07-20'), /no seed entry/) // opus has no codex id at all
|
|
569
|
-
})
|
|
570
|
-
|
|
571
|
-
// ---- read-only: bank/sync never write to the seed, sync --seen never prunes it ----
|
|
572
|
-
|
|
573
|
-
test('CLI bank refuses to write to the seed file by path (the default seed location)', () => {
|
|
574
|
-
const before = fs.readFileSync(defaultSeedPath(), 'utf8')
|
|
575
|
-
const res = run(['bank', JSON.stringify({ id: 'new-model', family: 'x', tier: 'frontier' }), '--file', defaultSeedPath()])
|
|
576
|
-
assert.equal(res.status, 2)
|
|
577
|
-
assert.match(res.stderr, /read-only/)
|
|
578
|
-
assert.equal(fs.readFileSync(defaultSeedPath(), 'utf8'), before, 'the shipped seed must be byte-identical after a refused write')
|
|
579
|
-
})
|
|
580
|
-
|
|
581
|
-
test('CLI sync refuses to write to the seed file by path', () => {
|
|
582
|
-
const before = fs.readFileSync(defaultSeedPath(), 'utf8')
|
|
583
|
-
const res = run(['sync', '--file', defaultSeedPath(), '--seen', 'gpt-5.6-sol'])
|
|
584
|
-
assert.equal(res.status, 2)
|
|
585
|
-
assert.match(res.stderr, /read-only/)
|
|
586
|
-
assert.equal(fs.readFileSync(defaultSeedPath(), 'utf8'), before)
|
|
587
|
-
})
|
|
588
|
-
|
|
589
|
-
test('CLI bank/sync refuse ANY file shaped like a seed (carrying seededAt), not just the default path', () => {
|
|
590
|
-
// A copy of the seed at a scratch path must still be caught — the guard checks
|
|
591
|
-
// shape, not only the well-known location.
|
|
592
|
-
const copy = tmpFile('seed-copy.json')
|
|
593
|
-
fs.writeFileSync(copy, JSON.stringify(FIXTURE_SEED))
|
|
594
|
-
const before = fs.readFileSync(copy, 'utf8')
|
|
595
|
-
|
|
596
|
-
const bankRes = run(['bank', JSON.stringify({ id: 'x', family: 'f', tier: 'frontier' }), '--file', copy])
|
|
597
|
-
assert.equal(bankRes.status, 2)
|
|
598
|
-
assert.match(bankRes.stderr, /read-only/)
|
|
599
|
-
|
|
600
|
-
const syncRes = run(['sync', '--file', copy, '--seen', 'gpt-5.6-sol'])
|
|
601
|
-
assert.equal(syncRes.status, 2)
|
|
602
|
-
assert.match(syncRes.stderr, /read-only/)
|
|
603
|
-
|
|
604
|
-
assert.equal(fs.readFileSync(copy, 'utf8'), before, 'a seed-shaped file must be untouched by a refused bank/sync')
|
|
605
|
-
})
|
|
606
|
-
|
|
607
|
-
test('sync --seen pruning logic never applies to seed entries — the seed has no lastSeenAt/confirmedAt to prune by', () => {
|
|
608
|
-
// Structural proof, not just the write-refusal above: seed entries carry no
|
|
609
|
-
// lastSeenAt/confirmedAt fields at all (that is a per-user-catalog concept),
|
|
610
|
-
// so even if syncCatalog's pure prune predicate were run over seed-shaped
|
|
611
|
-
// data it could not classify a seed entry as stale — there is nothing to
|
|
612
|
-
// stamp `pruned` against. seedProposals (the actual seed read path) has no
|
|
613
|
-
// pruning step whatsoever.
|
|
614
|
-
for (const entry of FIXTURE_SEED.models) {
|
|
615
|
-
assert.equal(entry.lastSeenAt, undefined)
|
|
616
|
-
assert.equal(entry.confirmedAt, undefined)
|
|
617
|
-
}
|
|
618
|
-
})
|
|
619
|
-
|
|
620
|
-
test('confirming a seed-backed proposal banks into the USER catalog file, leaving the seed itself untouched', () => {
|
|
621
|
-
const seedFile = tmpFile('seed.json')
|
|
622
|
-
fs.writeFileSync(seedFile, JSON.stringify(FIXTURE_SEED))
|
|
623
|
-
const userCatalogFile = tmpFile('catalog.json')
|
|
624
|
-
const seedBefore = fs.readFileSync(seedFile, 'utf8')
|
|
625
|
-
|
|
626
|
-
// Simulate pre-flight step 4: the human confirmed the seed-backed proposal for
|
|
627
|
-
// kimi-k2.7-code, so it's banked into the user's OWN per-harness catalog.
|
|
628
|
-
const entry = entryFromSeedConfirmation(FIXTURE_SEED, 'claude-code', 'kimi-k2.7-code', '2026-07-20')
|
|
629
|
-
const res = run(['bank', JSON.stringify(entry), '--file', userCatalogFile, '--today', '2026-07-20'])
|
|
630
|
-
assert.equal(res.status, 0)
|
|
631
|
-
|
|
632
|
-
assert.deepEqual(readCatalog(userCatalogFile).models[0].tiers, { high: 'scoped-coder' })
|
|
633
|
-
assert.ok(readCatalog(userCatalogFile).models[0].confirmedAt, 'a banked entry carries a real confirmedAt — never seed provenance')
|
|
634
|
-
assert.equal(fs.readFileSync(seedFile, 'utf8'), seedBefore, 'the seed file itself is never written by a confirmation')
|
|
635
|
-
})
|
|
636
|
-
|
|
637
|
-
// ---- rule 5 (cross-harness), which had no implementation at all until 2026-07-20 ----
|
|
638
|
-
|
|
639
|
-
const XH_CATS = {
|
|
640
|
-
'claude-code': { harness: 'claude-code', models: [{ id: 'gpt-5.6-terra', family: 'openai', tiers: { high: 'balanced-coder' } }, { id: 'kimi-k3', family: 'moonshot', tiers: { max: 'frontier' } }] },
|
|
641
|
-
codex: { harness: 'codex', models: [{ id: 'gpt-5.6-sol', family: 'openai', tiers: { high: 'frontier' } }] },
|
|
642
|
-
pi: { harness: 'pi', models: [{ id: 'kimi-coding/k3', family: 'moonshot', tiers: { high: 'deep-reasoner' } }] },
|
|
643
|
-
}
|
|
644
|
-
const XH_PATHS = { 'claude-code': 1, codex: 1, pi: 1 }
|
|
645
|
-
const xhRead = (p) => XH_CATS[Object.keys(XH_PATHS).find((h) => p.includes(h === 'claude-code' ? '.claude' : `.${h}`))] || null
|
|
646
|
-
const xh = (harness, seen) => crossHarnessProposals(harness, seen, { readAt: xhRead, paths: XH_PATHS })
|
|
647
|
-
|
|
648
|
-
test('plainlyMatches accepts a provider/ prefix difference in either direction, and exact equality', () => {
|
|
649
|
-
assert.equal(plainlyMatches('openai/gpt-5.6-sol', 'gpt-5.6-sol'), true)
|
|
650
|
-
assert.equal(plainlyMatches('gpt-5.6-sol', 'openai/gpt-5.6-sol'), true)
|
|
651
|
-
assert.equal(plainlyMatches('gpt-5.5', 'gpt-5.5'), true)
|
|
652
|
-
})
|
|
653
|
-
|
|
654
|
-
test('plainlyMatches REFUSES the near-misses — this is the half of rule 5 that matters', () => {
|
|
655
|
-
// A harness-local alias normalizes to nothing.
|
|
656
|
-
assert.equal(plainlyMatches('sonnet', 'anthropic/claude-sonnet-5'), false)
|
|
657
|
-
// The trap the shipped seed documents: stripping `kimi-coding/` gives `k3`, not
|
|
658
|
-
// `kimi-k3`. The seed's authors declined this merge pending human confirmation;
|
|
659
|
-
// an agent hand-applying rule 5 makes it anyway. The refusal must be mechanical.
|
|
660
|
-
assert.equal(plainlyMatches('kimi-coding/k3', 'kimi-k3'), false)
|
|
661
|
-
assert.equal(plainlyMatches('openai/gpt-5.6-sol', 'openai/gpt-5.6-luna'), false)
|
|
662
|
-
})
|
|
663
|
-
|
|
664
|
-
test('crossHarnessProposals pre-fills from another harness with the citation rule 5 specifies', () => {
|
|
665
|
-
const out = xh('pi', ['openai/gpt-5.6-terra'])
|
|
666
|
-
assert.equal(out.length, 1)
|
|
667
|
-
assert.deepEqual(out[0].tiers, { high: 'balanced-coder' })
|
|
668
|
-
assert.equal(out[0].fromHarness, 'claude-code')
|
|
669
|
-
assert.equal(out[0].citation, 'tier from claude-code catalog: gpt-5.6-terra')
|
|
670
|
-
})
|
|
671
|
-
|
|
672
|
-
test('crossHarnessProposals never transfers a tier across an unconfirmed alias', () => {
|
|
673
|
-
// pi's `kimi-coding/k3` and claude-code's `kimi-k3` are plausibly one model.
|
|
674
|
-
// Rule 5 must not merge them, so a claude-code run discovering kimi-k3 gets nothing.
|
|
675
|
-
assert.deepEqual(xh('claude-code', ['kimi-k3']), [])
|
|
676
|
-
})
|
|
677
|
-
|
|
678
|
-
test('crossHarnessProposals skips a model THIS harness already banked — rule 1 outranks rule 5', () => {
|
|
679
|
-
assert.deepEqual(xh('claude-code', ['gpt-5.6-terra']), [])
|
|
680
|
-
})
|
|
681
|
-
|
|
682
|
-
test('crossHarnessProposals never leaks availability — only discovered ids appear (rule 3)', () => {
|
|
683
|
-
// codex has gpt-5.6-sol banked; a pi run that did not discover it gets nothing.
|
|
684
|
-
assert.deepEqual(xh('pi', []), [])
|
|
685
|
-
assert.deepEqual(xh('pi', ['something-else']), [])
|
|
686
|
-
})
|
|
687
|
-
|
|
688
|
-
// ---- SLOT_ELIGIBLE_TIERS, wired at last ----
|
|
689
|
-
|
|
690
|
-
test('operatingPoints marks scoped-coder ineligible rather than hiding it', () => {
|
|
691
|
-
const catalog = { models: [{ id: 'x', family: 'f', tiers: { high: 'scoped-coder' } }] }
|
|
692
|
-
const [pt] = operatingPoints(catalog, 'x')
|
|
693
|
-
assert.equal(pt.tier, 'scoped-coder')
|
|
694
|
-
assert.equal(pt.slotEligible, false, 'marked, so pre-flight can show it without offering it')
|
|
695
|
-
assert.equal(operatingPoints(catalog, 'x').length, 1, 'still listed — banking it is what stops pre-flight re-asking')
|
|
696
|
-
})
|
|
697
|
-
|
|
698
|
-
test('every SLOT_ELIGIBLE_TIERS band is marked eligible and every other band is not', () => {
|
|
699
|
-
for (const tier of TIERS) {
|
|
700
|
-
const catalog = { models: [{ id: 'x', family: 'f', tiers: { high: tier } }] }
|
|
701
|
-
assert.equal(operatingPoints(catalog, 'x')[0].slotEligible, SLOT_ELIGIBLE_TIERS.includes(tier), tier)
|
|
702
|
-
}
|
|
703
|
-
})
|
|
704
|
-
|
|
705
|
-
test('the shipped seed cannot hand a fresh install a slot-eligible scoped-coder', () => {
|
|
706
|
-
const seed = readSeed(defaultSeedPath())
|
|
707
|
-
const ids = seed.models.filter((m) => m.ids['claude-code']).map((m) => m.ids['claude-code'])
|
|
708
|
-
const { proposals } = seedProposals(seed, 'claude-code', ids, null) // null catalog = fresh install
|
|
709
|
-
const scoped = proposals.flatMap((p) => p.points).filter((pt) => pt.tier === 'scoped-coder')
|
|
710
|
-
assert.ok(scoped.length > 0, 'the seed does ship scoped-coder entries — that is the case under test')
|
|
711
|
-
assert.ok(scoped.every((pt) => pt.slotEligible === false), 'a scoped-coder point must never be marked slot-eligible')
|
|
712
|
-
})
|
|
713
|
-
|
|
714
|
-
// ---- bank-from-seed: the confirmation path, previously hand-written ----
|
|
715
|
-
|
|
716
|
-
test('CLI bank-from-seed banks THIS harness tiers with a fresh confirmedAt, never the seed provenance', () => {
|
|
717
|
-
const seedFile = tmpFile('seed.json')
|
|
718
|
-
fs.writeFileSync(seedFile, JSON.stringify(FIXTURE_SEED))
|
|
719
|
-
const catalogFile = tmpFile('catalog.json')
|
|
720
|
-
const res = run(['bank-from-seed', 'kimi-coding/kimi-for-coding', '--seed-file', seedFile, '--file', catalogFile, '--harness', 'pi', '--today', '2026-07-20'])
|
|
721
|
-
assert.equal(res.status, 0)
|
|
722
|
-
const [entry] = readCatalog(catalogFile).models
|
|
723
|
-
assert.deepEqual(entry.tiers, { high: 'scoped-coder' }, "pi's tier, not claude-code's")
|
|
724
|
-
assert.equal(entry.confirmedAt, '2026-07-20')
|
|
725
|
-
assert.equal(entry.seededAt, undefined, 'no seed provenance may ride along as a confirmation')
|
|
726
|
-
})
|
|
727
|
-
|
|
728
|
-
test('CLI bank-from-seed refuses an id the seed does not cover under that harness', () => {
|
|
729
|
-
const seedFile = tmpFile('seed.json')
|
|
730
|
-
fs.writeFileSync(seedFile, JSON.stringify(FIXTURE_SEED))
|
|
731
|
-
const res = run(['bank-from-seed', 'kimi-k2.7-code', '--seed-file', seedFile, '--file', tmpFile('c.json'), '--harness', 'codex'])
|
|
732
|
-
assert.equal(res.status, 2)
|
|
733
|
-
assert.match(res.stderr, /no seed entry/)
|
|
734
|
-
})
|
|
735
|
-
|
|
736
|
-
// ---- tierProposals: the composed answer pre-flight step 2 actually consumes ----
|
|
737
|
-
|
|
738
|
-
test('tierProposals decides each discovered model and names the winning source', () => {
|
|
739
|
-
const cats = {
|
|
740
|
-
'claude-code': { harness: 'claude-code', models: [{ id: 'opus', family: 'anthropic', tiers: { high: 'deep-reasoner' } }] },
|
|
741
|
-
codex: { harness: 'codex', models: [{ id: 'gpt-5.6-sol', family: 'openai', tiers: { high: 'frontier' } }] },
|
|
742
|
-
pi: { harness: 'pi', models: [] },
|
|
743
|
-
}
|
|
744
|
-
const readAt = (p) => cats[Object.keys(cats).find((h) => p.includes(h === 'claude-code' ? '.claude' : `.${h}`))] || null
|
|
745
|
-
const seedFile = tmpFile('seed.json')
|
|
746
|
-
fs.writeFileSync(seedFile, JSON.stringify(FIXTURE_SEED))
|
|
747
|
-
|
|
748
|
-
const { models } = tierProposals('claude-code', ['opus', 'gpt-5.6-sol', 'kimi-k2.7-code', 'never-heard-of-it'], { seedFile, readAt })
|
|
749
|
-
const by = Object.fromEntries(models.map((m) => [m.id, m]))
|
|
750
|
-
|
|
751
|
-
assert.equal(by.opus.source, 'banked', 'own catalog wins')
|
|
752
|
-
assert.equal(by['gpt-5.6-sol'].source, 'cross-harness', "codex's entry, plainly matching")
|
|
753
|
-
assert.equal(by['kimi-k2.7-code'].source, 'team-seed', 'nothing banked or cross-harness')
|
|
754
|
-
assert.equal(by['never-heard-of-it'].source, 'unconfirmed')
|
|
755
|
-
assert.deepEqual(by['never-heard-of-it'].tiers, {})
|
|
756
|
-
})
|
|
757
|
-
|
|
758
|
-
test('tierProposals marks slot eligibility on the resolved answer, so a scoped-coder cannot reach a slot list', () => {
|
|
759
|
-
const readAt = () => null // fresh install: nothing banked anywhere
|
|
760
|
-
const seedFile = tmpFile('seed.json')
|
|
761
|
-
fs.writeFileSync(seedFile, JSON.stringify(FIXTURE_SEED))
|
|
762
|
-
const { models } = tierProposals('claude-code', ['kimi-k2.7-code'], { seedFile, readAt })
|
|
763
|
-
assert.equal(models[0].tiers.high, 'scoped-coder')
|
|
764
|
-
assert.deepEqual(models[0].points, [{ effort: 'high', tier: 'scoped-coder', slotEligible: false }])
|
|
765
|
-
})
|
|
766
|
-
|
|
767
|
-
test('tierProposals surfaces a stale-seed disagreement instead of swallowing it', () => {
|
|
768
|
-
const cats = { 'claude-code': { harness: 'claude-code', models: [{ id: 'kimi-k2.7-code', family: 'moonshot', tiers: { high: 'balanced-coder' } }] } }
|
|
769
|
-
const readAt = (p) => (p.includes('.claude') ? cats['claude-code'] : null)
|
|
770
|
-
const seedFile = tmpFile('seed.json')
|
|
771
|
-
fs.writeFileSync(seedFile, JSON.stringify(FIXTURE_SEED)) // seed says scoped-coder
|
|
772
|
-
const { models, seedDisagreements } = tierProposals('claude-code', ['kimi-k2.7-code'], { seedFile, readAt })
|
|
773
|
-
assert.equal(models[0].source, 'banked')
|
|
774
|
-
assert.equal(seedDisagreements.length, 1)
|
|
775
|
-
assert.deepEqual(seedDisagreements[0].banked, { high: 'balanced-coder' })
|
|
776
|
-
assert.deepEqual(seedDisagreements[0].seed, { high: 'scoped-coder' })
|
|
777
|
-
})
|
|
778
|
-
|
|
779
|
-
// ---- build 2: classification-staleness (confirmedAt age), distinct from prune (lastSeenAt) ----
|
|
780
|
-
test('tierProposals flags a banked classification older than CLASSIFY_STALE_AFTER_DAYS, and only when past it', () => {
|
|
781
|
-
assert.equal(CLASSIFY_STALE_AFTER_DAYS, 30)
|
|
782
|
-
const cats = { 'claude-code': { harness: 'claude-code', models: [{ id: 'grok-4.5', family: 'xai', confirmedAt: '2026-07-19', lastSeenAt: '2026-08-30', tiers: { high: 'deep-reasoner' } }] } }
|
|
783
|
-
const readAt = (p) => (p.includes('.claude') ? cats['claude-code'] : null)
|
|
784
|
-
const seedFile = tmpFile('seed.json')
|
|
785
|
-
fs.writeFileSync(seedFile, JSON.stringify(FIXTURE_SEED))
|
|
786
|
-
|
|
787
|
-
// 42 days after confirmedAt — recently SEEN (never pruned) yet the classification is stale.
|
|
788
|
-
const stale = tierProposals('claude-code', ['grok-4.5'], { seedFile, readAt, today: '2026-08-30' })
|
|
789
|
-
assert.deepEqual(stale.staleClassifications, [{ id: 'grok-4.5', confirmedAt: '2026-07-19', ageDays: 42 }])
|
|
790
|
-
|
|
791
|
-
// 6 days after — not stale.
|
|
792
|
-
const fresh = tierProposals('claude-code', ['grok-4.5'], { seedFile, readAt, today: '2026-07-25' })
|
|
793
|
-
assert.deepEqual(fresh.staleClassifications, [])
|
|
794
|
-
})
|
|
795
|
-
|
|
796
|
-
test('tierProposals only flags stale classifications for DISCOVERED ids (rule 3)', () => {
|
|
797
|
-
const cats = { 'claude-code': { harness: 'claude-code', models: [{ id: 'old-model', family: 'x', confirmedAt: '2020-01-01', tiers: { high: 'frontier' } }] } }
|
|
798
|
-
const readAt = (p) => (p.includes('.claude') ? cats['claude-code'] : null)
|
|
799
|
-
const seedFile = tmpFile('seed.json')
|
|
800
|
-
fs.writeFileSync(seedFile, JSON.stringify(FIXTURE_SEED))
|
|
801
|
-
const out = tierProposals('claude-code', ['definitely-not-discovered'], { seedFile, readAt, today: '2026-08-30' })
|
|
802
|
-
assert.deepEqual(out.staleClassifications, [], 'an undiscovered banked model is not surfaced')
|
|
803
|
-
})
|
|
804
|
-
|
|
805
|
-
test('CLI tiers composes the three arms in one call', () => {
|
|
806
|
-
const seedFile = tmpFile('seed.json')
|
|
807
|
-
fs.writeFileSync(seedFile, JSON.stringify(FIXTURE_SEED))
|
|
808
|
-
const res = run(['tiers', '--harness', 'claude-code', '--seed-file', seedFile, '--seen', 'definitely-not-real'])
|
|
809
|
-
assert.equal(res.status, 0)
|
|
810
|
-
const { models } = JSON.parse(res.stdout)
|
|
811
|
-
assert.equal(models[0].source, 'unconfirmed')
|
|
812
|
-
})
|
|
813
|
-
|
|
814
|
-
test('CLI tiers reads the catalog from --file — the pure-path fix exercised through the CLI, not injected readAt', () => {
|
|
815
|
-
// The staleClassifications/seedDisagreements surfaces are only truthful if the CLI reads the
|
|
816
|
-
// catalog it was pointed at. Before the fix `tiers --file` read the default per-harness path,
|
|
817
|
-
// so a stale banked model on an explicit path came back `unconfirmed` with no staleClassifications.
|
|
818
|
-
const seedFile = tmpFile('seed.json')
|
|
819
|
-
fs.writeFileSync(seedFile, JSON.stringify(FIXTURE_SEED))
|
|
820
|
-
const catalogFile = tmpFile('catalog.json')
|
|
821
|
-
writeCatalog(catalogFile, { harness: 'claude-code', models: [{ id: 'old-model', family: 'x', confirmedAt: '2026-07-19', lastSeenAt: '2026-08-30', tiers: { high: 'deep-reasoner' } }] })
|
|
822
|
-
|
|
823
|
-
const res = run(['tiers', '--file', catalogFile, '--harness', 'claude-code', '--seed-file', seedFile, '--seen', 'old-model', '--today', '2026-08-30'])
|
|
824
|
-
assert.equal(res.status, 0)
|
|
825
|
-
const out = JSON.parse(res.stdout)
|
|
826
|
-
assert.equal(out.models[0].source, 'banked', 'the --file catalog is read, so the model is banked, not unconfirmed')
|
|
827
|
-
assert.equal(out.models[0].tiers.high, 'deep-reasoner')
|
|
828
|
-
assert.deepEqual(out.staleClassifications, [{ id: 'old-model', confirmedAt: '2026-07-19', ageDays: 42 }])
|
|
829
|
-
})
|