@1aboveio/skills 1.0.8 → 1.0.10

package/skills/engineering/harness-runtime/discover-models.test.mjs

@@ -1,363 +0,0 @@
-// discover-models tests — real fixture config files under a temp HOME, NEVER the
-// user's real ~/.pi / ~/.codex / ~/.claude. Every case writes its own config tree
-// into a scratch dir and runs the script with `HOME=<tmp>` and an INJECTED env for
-// the apiKey `$VAR` resolution, so no test reads or depends on the real machine's
-// config or environment. A test that touched the real config would be a defect —
-// the whole point of this unit is a trustworthy, isolated discovery read.
-
-import { test } from 'node:test'
-import assert from 'node:assert/strict'
-import fs from 'node:fs'
-import os from 'node:os'
-import path from 'node:path'
-import { fileURLToPath } from 'node:url'
-import { execFileSync } from 'node:child_process'
-
-import { discoverPi, discoverCodex, discoverClaudeCode, apiKeyResolves } from './discover-models.mjs'
-
-const scriptPath = path.join(path.dirname(fileURLToPath(import.meta.url)), 'bin', 'discover-models.mjs')
-
-// ---- temp-HOME fixture scaffolding ----
-
-function mkHome() {
-  return fs.mkdtempSync(path.join(os.tmpdir(), 'discover-models-'))
-}
-
-function writeJson(file, obj) {
-  fs.mkdirSync(path.dirname(file), { recursive: true })
-  fs.writeFileSync(file, JSON.stringify(obj, null, 2))
-}
-
-function writeText(file, text) {
-  fs.mkdirSync(path.dirname(file), { recursive: true })
-  fs.writeFileSync(file, text)
-}
-
-// Run the script with a fully-controlled env: an explicit HOME (the fixture tree)
-// and only the env vars a case injects — so `$VAR` resolution is tested against
-// injected values, never the real environment. Returns { code, stdout, json }.
-function run(args, { home, extraEnv = {} } = {}) {
-  const env = { PATH: process.env.PATH, HOME: home, ...extraEnv }
-  let stdout = ''
-  let code = 0
-  try {
-    stdout = execFileSync('node', [scriptPath, ...args], { encoding: 'utf8', env })
-  } catch (err) {
-    code = err.status ?? 1
-    stdout = err.stdout ?? ''
-  }
-  let json = null
-  try {
-    json = JSON.parse(stdout)
-  } catch {
-    /* non-JSON (help/usage) — leave json null */
-  }
-  return { code, stdout, json }
-}
-
-const ids = (json) => json.discovered.map((d) => d.id)
-
-// ---- Pi: the full credentialed-remainder fixture ----
-
-// Builds the Pi fixture the task spec asks for, under a fresh temp HOME:
-//   - enabledModels: the curated picker set
-//   - a default provider/model
-//   - a models.json custom entry whose provider apiKey is a LITERAL (resolves)
-//   - a models.json custom entry whose provider apiKey is $SET_VAR (resolves)
-//   - a models.json custom entry whose provider apiKey is $UNSET_VAR (does NOT resolve)
-//   - an auth.json-credentialed provider with NO enabledModels entry (provider-level)
-function buildPiHome() {
-  const home = mkHome()
-  writeJson(path.join(home, '.pi', 'agent', 'settings.json'), {
-    defaultProvider: 'openai',
-    defaultModel: 'gpt-x-terra',
-    enabledModels: ['openai/gpt-x-terra', 'openai/gpt-x-sol', 'acme/curated-1'],
-  })
-  writeJson(path.join(home, '.pi', 'agent', 'models.json'), {
-    providers: {
-      // openai is curated (has enabledModels entries) — a proxy provider, no models[]
-      openai: { baseUrl: 'https://example/v1', apiKey: '$OPENAI_KEY' },
-      // literal apiKey → resolves; custom models emitted at model level
-      litprov: { apiKey: 'sk-literal-abc', models: [{ id: 'lit-model-1' }, { id: 'lit-model-2' }] },
-      // $SET_VAR apiKey → resolves (env injected)
-      setprov: { apiKey: '$SET_VAR', models: [{ id: 'set-model-1' }] },
-      // $UNSET_VAR apiKey → does NOT resolve; its custom model is a KNOWN definition
-      // the user cannot run — the never-a-source / credential-gate case.
-      unsetprov: { apiKey: '$UNSET_VAR', models: [{ id: 'unset-model-1' }] },
-    },
-  })
-  // A provider credentialed via auth.json with no enabledModels entry and no
-  // models[] detail — listed at PROVIDER level only.
-  writeJson(path.join(home, '.pi', 'agent', 'auth.json'), {
-    google: { type: 'apiKey', key: 'stored-google-key' },
-  })
-  return home
-}
-
-// @covers package:harness-runtime
-// @level integration
-test('pi: discovers enabledModels, the default, and credentialed custom models', () => {
-  const home = buildPiHome()
-  const { code, json } = run(['--harness', 'pi'], { home, extraEnv: { OPENAI_KEY: 'x', SET_VAR: 'yes' } })
-  assert.equal(code, 0)
-  assert.equal(json.harness, 'pi')
-  assert.equal(json.completeness, 'full')
-  const discovered = ids(json)
-  // curated picker set
-  assert.ok(discovered.includes('openai/gpt-x-terra'))
-  assert.ok(discovered.includes('openai/gpt-x-sol'))
-  assert.ok(discovered.includes('acme/curated-1'))
-  // literal-apiKey custom models
-  assert.ok(discovered.includes('litprov/lit-model-1'))
-  assert.ok(discovered.includes('litprov/lit-model-2'))
-  // $SET_VAR-apiKey custom model
-  assert.ok(discovered.includes('setprov/set-model-1'))
-})
-
-test('pi: the default provider/model is present and sourced as the default', () => {
-  const home = buildPiHome()
-  const { json } = run(['--harness', 'pi'], { home, extraEnv: { OPENAI_KEY: 'x', SET_VAR: 'yes' } })
-  const entry = json.discovered.find((d) => d.id === 'openai/gpt-x-terra')
-  assert.ok(entry, 'default model must be discovered')
-  // It is already in enabledModels, so first-source-wins keeps the enabledModels source.
-  assert.equal(entry.source, 'settings.json:enabledModels')
-})
-
-test('pi: an auth.json-credentialed provider with no enabledModels entry is listed at PROVIDER level only', () => {
-  const home = buildPiHome()
-  const { json } = run(['--harness', 'pi'], { home, extraEnv: { OPENAI_KEY: 'x', SET_VAR: 'yes' } })
-  const discovered = ids(json)
-  assert.ok(discovered.includes('google/*'), 'expected provider-level google/*')
-  // NEVER an expanded per-model registry for the credentialed-but-uncurated provider.
-  assert.ok(!discovered.some((id) => id.startsWith('google/') && id !== 'google/*'),
-    'google must appear only at provider level, never expanded into concrete model ids')
-  const entry = json.discovered.find((d) => d.id === 'google/*')
-  assert.match(entry.source, /auth\.json entry/)
-})
-
-test('pi: NEVER-A-SOURCE — a config-known model whose provider key does not resolve is NOT emitted', () => {
-  const home = buildPiHome()
-  // $UNSET_VAR is deliberately absent from the injected env.
-  const { json } = run(['--harness', 'pi'], { home, extraEnv: { OPENAI_KEY: 'x', SET_VAR: 'yes' } })
-  const discovered = ids(json)
-  assert.ok(
-    !discovered.includes('unsetprov/unset-model-1'),
-    'a custom model whose provider apiKey ($UNSET_VAR) does not resolve must NOT be discovered — ' +
-      'it is a definition the config knows but the user cannot run',
-  )
-  // And the provider itself is not credentialed, so no provider-level entry either.
-  assert.ok(!discovered.includes('unsetprov/*'))
-})
-
-test('pi: records the never-a-source rule as a note', () => {
-  const home = buildPiHome()
-  const { json } = run(['--harness', 'pi'], { home, extraEnv: { OPENAI_KEY: 'x', SET_VAR: 'yes' } })
-  assert.ok(json.notes.some((n) => /never a discovery source/.test(n) && /pi --models/.test(n)))
-})
-
-test('pi: missing config is graceful — empty set + note, exit 0, no crash', () => {
-  const home = mkHome() // no .pi tree at all
-  const { code, json } = run(['--harness', 'pi'], { home })
-  assert.equal(code, 0)
-  assert.equal(json.completeness, 'full')
-  assert.deepEqual(json.discovered, [])
-  assert.ok(json.notes.some((n) => /not found/.test(n)))
-})
-
-// ---- Codex ----
-
-// @covers package:harness-runtime
-// @level integration
-test('codex: discovers configured defaults without claiming a complete runnable inventory', () => {
-  const home = mkHome()
-  writeText(path.join(home, '.codex', 'config.toml'),
-    'model = "gpt-x-sol"\n' +
-    'model_reasoning_effort = "high"\n' +      // must NOT be read as a model
-    'model_context_window = 1000000\n' +
-    '\n' +
-    '[profiles.deep]\n' +
-    'model = "gpt-x-max"\n')
-  const { code, json } = run(['--harness', 'codex'], { home })
-  assert.equal(code, 0)
-  assert.equal(json.harness, 'codex')
-  assert.equal(json.completeness, 'partial')
-  const discovered = ids(json)
-  assert.deepEqual(discovered.sort(), ['gpt-x-max', 'gpt-x-sol'].sort())
-  // The `model_*` keys must not leak in as models.
-  assert.ok(!discovered.includes('high'))
-  assert.ok(!discovered.some((id) => /^\d+$/.test(id)))
-  // The top-level model is sourced as config.toml:model; the profile one carries its table.
-  assert.equal(json.discovered.find((d) => d.id === 'gpt-x-sol').source, 'config.toml:model')
-  assert.match(json.discovered.find((d) => d.id === 'gpt-x-max').source, /\[profiles\.deep\]/)
-  assert.ok(json.notes.some((note) => /partial/.test(note)))
-})
-
-test('codex: current-turn advertised ids are preserved, unioned, and make discovery full', () => {
-  const home = mkHome()
-  writeText(path.join(home, '.codex', 'config.toml'), 'model = "configured-default"\n')
-  const { code, json } = run([
-    '--harness', 'codex',
-    '--available', 'advertised/model-a,configured-default,advertised:model-b',
-  ], { home })
-
-  assert.equal(code, 0)
-  assert.equal(json.completeness, 'full')
-  assert.deepEqual(ids(json), ['advertised/model-a', 'configured-default', 'advertised:model-b'])
-  assert.equal(json.discovered[0].source, 'current-turn')
-  assert.equal(json.discovered[2].source, 'current-turn')
-})
-
-test('codex: a human-named runnable id is evidence but does not turn partial discovery into full', () => {
-  const home = mkHome()
-  const { code, json } = run([
-    '--harness', 'codex',
-    '--human-named', 'operator/model-exact',
-  ], { home })
-
-  assert.equal(code, 0)
-  assert.equal(json.completeness, 'partial')
-  assert.deepEqual(json.discovered, [{ id: 'operator/model-exact', source: 'human-named' }])
-})
-
-test('codex: missing config is graceful — empty set + note, exit 0', () => {
-  const home = mkHome()
-  const { code, json } = run(['--harness', 'codex'], { home })
-  assert.equal(code, 0)
-  assert.equal(json.completeness, 'partial')
-  assert.deepEqual(json.discovered, [])
-  assert.ok(json.notes.some((n) => /not found/.test(n)))
-})
-
-// ---- Claude Code (honest-partial) ----
-
-// @covers package:harness-runtime
-// @level integration
-test('claude-code: is honest-partial — records that discovery stays manual and does NOT read the banked catalog', () => {
-  const home = mkHome()
-  writeJson(path.join(home, '.claude', 'settings.json'), { effortLevel: 'high', permissions: {} })
-  // A banked catalog is present — it must NOT be treated as an availability source.
-  writeJson(path.join(home, '.claude', 'model-catalog.json'), {
-    harness: 'claude-code',
-    models: [{ id: 'should-not-appear', tiers: { high: 'frontier' } }],
-  })
-  const { code, json } = run(['--harness', 'claude-code'], { home })
-  assert.equal(code, 0)
-  assert.equal(json.harness, 'claude-code')
-  assert.equal(json.completeness, 'partial')
-  assert.ok(!ids(json).includes('should-not-appear'),
-    'the banked model-catalog.json is a cache of confirmations, never an availability source')
-  assert.ok(json.notes.some((n) => /Agent-tool model enum/.test(n) && /stays manual/.test(n)))
-})
-
-test('claude-code: current-turn advertised ids complete discovery and union with settings', () => {
-  const home = mkHome()
-  writeJson(path.join(home, '.claude', 'settings.json'), { model: 'configured-model' })
-  const { code, json } = run([
-    '--harness', 'claude-code',
-    '--available', 'advertised-model,configured-model',
-  ], { home })
-
-  assert.equal(code, 0)
-  assert.equal(json.completeness, 'full')
-  assert.deepEqual(ids(json), ['advertised-model', 'configured-model'])
-  assert.ok(json.discovered.every((model) => model.source === 'current-turn'))
-})
-
-// ---- CLI contract: help, unknown flags, auto-detect ----
-
-test('--help exits 0 with non-empty usage naming the flags and exit codes', () => {
-  const { code, stdout } = run(['--help'], { home: mkHome() })
-  assert.equal(code, 0)
-  assert.ok(stdout.includes('--harness'))
-  assert.ok(stdout.includes('--available'))
-  assert.ok(stdout.includes('--human-named'))
-  assert.ok(stdout.includes('Exit codes'))
-  assert.ok(/pi\|codex\|claude-code/.test(stdout))
-})
-
-test('-h is an alias for --help (exit 0)', () => {
-  const { code, stdout } = run(['-h'], { home: mkHome() })
-  assert.equal(code, 0)
-  assert.ok(stdout.length > 0)
-})
-
-test('an unknown flag exits 2 with usage', () => {
-  const { code } = run(['--bogus'], { home: mkHome() })
-  assert.equal(code, 2)
-})
-
-test('an unknown --harness value exits 2', () => {
-  const { code } = run(['--harness', 'nope'], { home: mkHome() })
-  assert.equal(code, 2)
-})
-
-test('no --harness with an ambiguous/empty HOME exits 2 (never guesses)', () => {
-  const { code } = run([], { home: mkHome() }) // empty HOME: no harness present
-  assert.equal(code, 2)
-})
-
-test('no --harness auto-detects the sole present harness', () => {
-  const home = mkHome()
-  writeText(path.join(home, '.codex', 'config.toml'), 'model = "solo-model"\n')
-  const { code, json } = run([], { home })
-  assert.equal(code, 0)
-  assert.equal(json.harness, 'codex')
-  assert.ok(ids(json).includes('solo-model'))
-})
-
-// ---- pure-core unit checks (fast, no spawn) ----
-
-test('apiKeyResolves: literal resolves; $SET resolves; $UNSET does not; empty does not', () => {
-  assert.equal(apiKeyResolves('sk-abc', {}).resolved, true)
-  assert.equal(apiKeyResolves('$K', { K: 'v' }).resolved, true)
-  assert.equal(apiKeyResolves('$K', {}).resolved, false)
-  assert.equal(apiKeyResolves('', {}).resolved, false)
-  assert.equal(apiKeyResolves(undefined, {}).resolved, false)
-})
-
-test('discoverPi (pure): credential gate excludes the unresolved custom model', () => {
-  const { discovered } = discoverPi({
-    settings: { enabledModels: ['a/one'] },
-    models: {
-      providers: {
-        ok: { apiKey: 'lit', models: [{ id: 'runnable' }] },
-        no: { apiKey: '$MISSING', models: [{ id: 'notrunnable' }] },
-      },
-    },
-    auth: {},
-    env: {},
-  })
-  const set = discovered.map((d) => d.id)
-  assert.ok(set.includes('a/one'))
-  assert.ok(set.includes('ok/runnable'))
-  assert.ok(!set.includes('no/notrunnable'))
-})
-
-// @covers package:harness-runtime
-// @level unit
-test('discoverCodex (pure): explicit current-turn ids pass unchanged and take source precedence', () => {
-  const advertised = ['provider/model-a', 'model:b']
-  const { discovered, completeness } = discoverCodex({
-    configText: 'model = "provider/model-a"\nmodel = "config-only"\n',
-    spawnModelIds: advertised,
-  })
-  assert.equal(completeness, 'full')
-  assert.deepEqual(discovered, [
-    { id: 'provider/model-a', source: 'current-turn' },
-    { id: 'model:b', source: 'current-turn' },
-    { id: 'config-only', source: 'config.toml:model' },
-  ])
-  assert.deepEqual(advertised, ['provider/model-a', 'model:b'])
-})
-
-test('discoverCodex (pure): empty text yields empty + note, not a throw', () => {
-  const { discovered, notes, completeness } = discoverCodex({ configText: null })
-  assert.deepEqual(discovered, [])
-  assert.equal(completeness, 'partial')
-  assert.ok(notes.length > 0)
-})
-
-test('discoverClaudeCode (pure): always carries the honest-manual note', () => {
-  const { notes } = discoverClaudeCode({ settings: {} })
-  assert.ok(notes.some((n) => /stays manual/.test(n)))
-})