@1aboveio/skills 0.0.0-stage → 1.0.0

package/skills/engineering/harness-runtime/discover-models.test.mjs

@@ -0,0 +1,363 @@
+// discover-models tests — real fixture config files under a temp HOME, NEVER the
+// user's real ~/.pi / ~/.codex / ~/.claude. Every case writes its own config tree
+// into a scratch dir and runs the script with `HOME=<tmp>` and an INJECTED env for
+// the apiKey `$VAR` resolution, so no test reads or depends on the real machine's
+// config or environment. A test that touched the real config would be a defect —
+// the whole point of this unit is a trustworthy, isolated discovery read.
+
+import { test } from 'node:test'
+import assert from 'node:assert/strict'
+import fs from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { execFileSync } from 'node:child_process'
+
+import { discoverPi, discoverCodex, discoverClaudeCode, apiKeyResolves } from './discover-models.mjs'
+
+const scriptPath = path.join(path.dirname(fileURLToPath(import.meta.url)), 'bin', 'discover-models.mjs')
+
+// ---- temp-HOME fixture scaffolding ----
+
+function mkHome() {
+  return fs.mkdtempSync(path.join(os.tmpdir(), 'discover-models-'))
+}
+
+function writeJson(file, obj) {
+  fs.mkdirSync(path.dirname(file), { recursive: true })
+  fs.writeFileSync(file, JSON.stringify(obj, null, 2))
+}
+
+function writeText(file, text) {
+  fs.mkdirSync(path.dirname(file), { recursive: true })
+  fs.writeFileSync(file, text)
+}
+
+// Run the script with a fully-controlled env: an explicit HOME (the fixture tree)
+// and only the env vars a case injects — so `$VAR` resolution is tested against
+// injected values, never the real environment. Returns { code, stdout, json }.
+function run(args, { home, extraEnv = {} } = {}) {
+  const env = { PATH: process.env.PATH, HOME: home, ...extraEnv }
+  let stdout = ''
+  let code = 0
+  try {
+    stdout = execFileSync('node', [scriptPath, ...args], { encoding: 'utf8', env })
+  } catch (err) {
+    code = err.status ?? 1
+    stdout = err.stdout ?? ''
+  }
+  let json = null
+  try {
+    json = JSON.parse(stdout)
+  } catch {
+    /* non-JSON (help/usage) — leave json null */
+  }
+  return { code, stdout, json }
+}
+
+const ids = (json) => json.discovered.map((d) => d.id)
+
+// ---- Pi: the full credentialed-remainder fixture ----
+
+// Builds the Pi fixture the task spec asks for, under a fresh temp HOME:
+//   - enabledModels: the curated picker set
+//   - a default provider/model
+//   - a models.json custom entry whose provider apiKey is a LITERAL (resolves)
+//   - a models.json custom entry whose provider apiKey is $SET_VAR (resolves)
+//   - a models.json custom entry whose provider apiKey is $UNSET_VAR (does NOT resolve)
+//   - an auth.json-credentialed provider with NO enabledModels entry (provider-level)
+function buildPiHome() {
+  const home = mkHome()
+  writeJson(path.join(home, '.pi', 'agent', 'settings.json'), {
+    defaultProvider: 'openai',
+    defaultModel: 'gpt-x-terra',
+    enabledModels: ['openai/gpt-x-terra', 'openai/gpt-x-sol', 'acme/curated-1'],
+  })
+  writeJson(path.join(home, '.pi', 'agent', 'models.json'), {
+    providers: {
+      // openai is curated (has enabledModels entries) — a proxy provider, no models[]
+      openai: { baseUrl: 'https://example/v1', apiKey: '$OPENAI_KEY' },
+      // literal apiKey → resolves; custom models emitted at model level
+      litprov: { apiKey: 'sk-literal-abc', models: [{ id: 'lit-model-1' }, { id: 'lit-model-2' }] },
+      // $SET_VAR apiKey → resolves (env injected)
+      setprov: { apiKey: '$SET_VAR', models: [{ id: 'set-model-1' }] },
+      // $UNSET_VAR apiKey → does NOT resolve; its custom model is a KNOWN definition
+      // the user cannot run — the never-a-source / credential-gate case.
+      unsetprov: { apiKey: '$UNSET_VAR', models: [{ id: 'unset-model-1' }] },
+    },
+  })
+  // A provider credentialed via auth.json with no enabledModels entry and no
+  // models[] detail — listed at PROVIDER level only.
+  writeJson(path.join(home, '.pi', 'agent', 'auth.json'), {
+    google: { type: 'apiKey', key: 'stored-google-key' },
+  })
+  return home
+}
+
+// @covers package:harness-runtime
+// @level integration
+test('pi: discovers enabledModels, the default, and credentialed custom models', () => {
+  const home = buildPiHome()
+  const { code, json } = run(['--harness', 'pi'], { home, extraEnv: { OPENAI_KEY: 'x', SET_VAR: 'yes' } })
+  assert.equal(code, 0)
+  assert.equal(json.harness, 'pi')
+  assert.equal(json.completeness, 'full')
+  const discovered = ids(json)
+  // curated picker set
+  assert.ok(discovered.includes('openai/gpt-x-terra'))
+  assert.ok(discovered.includes('openai/gpt-x-sol'))
+  assert.ok(discovered.includes('acme/curated-1'))
+  // literal-apiKey custom models
+  assert.ok(discovered.includes('litprov/lit-model-1'))
+  assert.ok(discovered.includes('litprov/lit-model-2'))
+  // $SET_VAR-apiKey custom model
+  assert.ok(discovered.includes('setprov/set-model-1'))
+})
+
+test('pi: the default provider/model is present and sourced as the default', () => {
+  const home = buildPiHome()
+  const { json } = run(['--harness', 'pi'], { home, extraEnv: { OPENAI_KEY: 'x', SET_VAR: 'yes' } })
+  const entry = json.discovered.find((d) => d.id === 'openai/gpt-x-terra')
+  assert.ok(entry, 'default model must be discovered')
+  // It is already in enabledModels, so first-source-wins keeps the enabledModels source.
+  assert.equal(entry.source, 'settings.json:enabledModels')
+})
+
+test('pi: an auth.json-credentialed provider with no enabledModels entry is listed at PROVIDER level only', () => {
+  const home = buildPiHome()
+  const { json } = run(['--harness', 'pi'], { home, extraEnv: { OPENAI_KEY: 'x', SET_VAR: 'yes' } })
+  const discovered = ids(json)
+  assert.ok(discovered.includes('google/*'), 'expected provider-level google/*')
+  // NEVER an expanded per-model registry for the credentialed-but-uncurated provider.
+  assert.ok(!discovered.some((id) => id.startsWith('google/') && id !== 'google/*'),
+    'google must appear only at provider level, never expanded into concrete model ids')
+  const entry = json.discovered.find((d) => d.id === 'google/*')
+  assert.match(entry.source, /auth\.json entry/)
+})
+
+test('pi: NEVER-A-SOURCE — a config-known model whose provider key does not resolve is NOT emitted', () => {
+  const home = buildPiHome()
+  // $UNSET_VAR is deliberately absent from the injected env.
+  const { json } = run(['--harness', 'pi'], { home, extraEnv: { OPENAI_KEY: 'x', SET_VAR: 'yes' } })
+  const discovered = ids(json)
+  assert.ok(
+    !discovered.includes('unsetprov/unset-model-1'),
+    'a custom model whose provider apiKey ($UNSET_VAR) does not resolve must NOT be discovered — ' +
+      'it is a definition the config knows but the user cannot run',
+  )
+  // And the provider itself is not credentialed, so no provider-level entry either.
+  assert.ok(!discovered.includes('unsetprov/*'))
+})
+
+test('pi: records the never-a-source rule as a note', () => {
+  const home = buildPiHome()
+  const { json } = run(['--harness', 'pi'], { home, extraEnv: { OPENAI_KEY: 'x', SET_VAR: 'yes' } })
+  assert.ok(json.notes.some((n) => /never a discovery source/.test(n) && /pi --models/.test(n)))
+})
+
+test('pi: missing config is graceful — empty set + note, exit 0, no crash', () => {
+  const home = mkHome() // no .pi tree at all
+  const { code, json } = run(['--harness', 'pi'], { home })
+  assert.equal(code, 0)
+  assert.equal(json.completeness, 'full')
+  assert.deepEqual(json.discovered, [])
+  assert.ok(json.notes.some((n) => /not found/.test(n)))
+})
+
+// ---- Codex ----
+
+// @covers package:harness-runtime
+// @level integration
+test('codex: discovers configured defaults without claiming a complete runnable inventory', () => {
+  const home = mkHome()
+  writeText(path.join(home, '.codex', 'config.toml'),
+    'model = "gpt-x-sol"\n' +
+    'model_reasoning_effort = "high"\n' +      // must NOT be read as a model
+    'model_context_window = 1000000\n' +
+    '\n' +
+    '[profiles.deep]\n' +
+    'model = "gpt-x-max"\n')
+  const { code, json } = run(['--harness', 'codex'], { home })
+  assert.equal(code, 0)
+  assert.equal(json.harness, 'codex')
+  assert.equal(json.completeness, 'partial')
+  const discovered = ids(json)
+  assert.deepEqual(discovered.sort(), ['gpt-x-max', 'gpt-x-sol'].sort())
+  // The `model_*` keys must not leak in as models.
+  assert.ok(!discovered.includes('high'))
+  assert.ok(!discovered.some((id) => /^\d+$/.test(id)))
+  // The top-level model is sourced as config.toml:model; the profile one carries its table.
+  assert.equal(json.discovered.find((d) => d.id === 'gpt-x-sol').source, 'config.toml:model')
+  assert.match(json.discovered.find((d) => d.id === 'gpt-x-max').source, /\[profiles\.deep\]/)
+  assert.ok(json.notes.some((note) => /partial/.test(note)))
+})
+
+test('codex: current-turn advertised ids are preserved, unioned, and make discovery full', () => {
+  const home = mkHome()
+  writeText(path.join(home, '.codex', 'config.toml'), 'model = "configured-default"\n')
+  const { code, json } = run([
+    '--harness', 'codex',
+    '--available', 'advertised/model-a,configured-default,advertised:model-b',
+  ], { home })
+
+  assert.equal(code, 0)
+  assert.equal(json.completeness, 'full')
+  assert.deepEqual(ids(json), ['advertised/model-a', 'configured-default', 'advertised:model-b'])
+  assert.equal(json.discovered[0].source, 'current-turn')
+  assert.equal(json.discovered[2].source, 'current-turn')
+})
+
+test('codex: a human-named runnable id is evidence but does not turn partial discovery into full', () => {
+  const home = mkHome()
+  const { code, json } = run([
+    '--harness', 'codex',
+    '--human-named', 'operator/model-exact',
+  ], { home })
+
+  assert.equal(code, 0)
+  assert.equal(json.completeness, 'partial')
+  assert.deepEqual(json.discovered, [{ id: 'operator/model-exact', source: 'human-named' }])
+})
+
+test('codex: missing config is graceful — empty set + note, exit 0', () => {
+  const home = mkHome()
+  const { code, json } = run(['--harness', 'codex'], { home })
+  assert.equal(code, 0)
+  assert.equal(json.completeness, 'partial')
+  assert.deepEqual(json.discovered, [])
+  assert.ok(json.notes.some((n) => /not found/.test(n)))
+})
+
+// ---- Claude Code (honest-partial) ----
+
+// @covers package:harness-runtime
+// @level integration
+test('claude-code: is honest-partial — records that discovery stays manual and does NOT read the banked catalog', () => {
+  const home = mkHome()
+  writeJson(path.join(home, '.claude', 'settings.json'), { effortLevel: 'high', permissions: {} })
+  // A banked catalog is present — it must NOT be treated as an availability source.
+  writeJson(path.join(home, '.claude', 'model-catalog.json'), {
+    harness: 'claude-code',
+    models: [{ id: 'should-not-appear', tiers: { high: 'frontier' } }],
+  })
+  const { code, json } = run(['--harness', 'claude-code'], { home })
+  assert.equal(code, 0)
+  assert.equal(json.harness, 'claude-code')
+  assert.equal(json.completeness, 'partial')
+  assert.ok(!ids(json).includes('should-not-appear'),
+    'the banked model-catalog.json is a cache of confirmations, never an availability source')
+  assert.ok(json.notes.some((n) => /Agent-tool model enum/.test(n) && /stays manual/.test(n)))
+})
+
+test('claude-code: current-turn advertised ids complete discovery and union with settings', () => {
+  const home = mkHome()
+  writeJson(path.join(home, '.claude', 'settings.json'), { model: 'configured-model' })
+  const { code, json } = run([
+    '--harness', 'claude-code',
+    '--available', 'advertised-model,configured-model',
+  ], { home })
+
+  assert.equal(code, 0)
+  assert.equal(json.completeness, 'full')
+  assert.deepEqual(ids(json), ['advertised-model', 'configured-model'])
+  assert.ok(json.discovered.every((model) => model.source === 'current-turn'))
+})
+
+// ---- CLI contract: help, unknown flags, auto-detect ----
+
+test('--help exits 0 with non-empty usage naming the flags and exit codes', () => {
+  const { code, stdout } = run(['--help'], { home: mkHome() })
+  assert.equal(code, 0)
+  assert.ok(stdout.includes('--harness'))
+  assert.ok(stdout.includes('--available'))
+  assert.ok(stdout.includes('--human-named'))
+  assert.ok(stdout.includes('Exit codes'))
+  assert.ok(/pi\|codex\|claude-code/.test(stdout))
+})
+
+test('-h is an alias for --help (exit 0)', () => {
+  const { code, stdout } = run(['-h'], { home: mkHome() })
+  assert.equal(code, 0)
+  assert.ok(stdout.length > 0)
+})
+
+test('an unknown flag exits 2 with usage', () => {
+  const { code } = run(['--bogus'], { home: mkHome() })
+  assert.equal(code, 2)
+})
+
+test('an unknown --harness value exits 2', () => {
+  const { code } = run(['--harness', 'nope'], { home: mkHome() })
+  assert.equal(code, 2)
+})
+
+test('no --harness with an ambiguous/empty HOME exits 2 (never guesses)', () => {
+  const { code } = run([], { home: mkHome() }) // empty HOME: no harness present
+  assert.equal(code, 2)
+})
+
+test('no --harness auto-detects the sole present harness', () => {
+  const home = mkHome()
+  writeText(path.join(home, '.codex', 'config.toml'), 'model = "solo-model"\n')
+  const { code, json } = run([], { home })
+  assert.equal(code, 0)
+  assert.equal(json.harness, 'codex')
+  assert.ok(ids(json).includes('solo-model'))
+})
+
+// ---- pure-core unit checks (fast, no spawn) ----
+
+test('apiKeyResolves: literal resolves; $SET resolves; $UNSET does not; empty does not', () => {
+  assert.equal(apiKeyResolves('sk-abc', {}).resolved, true)
+  assert.equal(apiKeyResolves('$K', { K: 'v' }).resolved, true)
+  assert.equal(apiKeyResolves('$K', {}).resolved, false)
+  assert.equal(apiKeyResolves('', {}).resolved, false)
+  assert.equal(apiKeyResolves(undefined, {}).resolved, false)
+})
+
+test('discoverPi (pure): credential gate excludes the unresolved custom model', () => {
+  const { discovered } = discoverPi({
+    settings: { enabledModels: ['a/one'] },
+    models: {
+      providers: {
+        ok: { apiKey: 'lit', models: [{ id: 'runnable' }] },
+        no: { apiKey: '$MISSING', models: [{ id: 'notrunnable' }] },
+      },
+    },
+    auth: {},
+    env: {},
+  })
+  const set = discovered.map((d) => d.id)
+  assert.ok(set.includes('a/one'))
+  assert.ok(set.includes('ok/runnable'))
+  assert.ok(!set.includes('no/notrunnable'))
+})
+
+// @covers package:harness-runtime
+// @level unit
+test('discoverCodex (pure): explicit current-turn ids pass unchanged and take source precedence', () => {
+  const advertised = ['provider/model-a', 'model:b']
+  const { discovered, completeness } = discoverCodex({
+    configText: 'model = "provider/model-a"\nmodel = "config-only"\n',
+    spawnModelIds: advertised,
+  })
+  assert.equal(completeness, 'full')
+  assert.deepEqual(discovered, [
+    { id: 'provider/model-a', source: 'current-turn' },
+    { id: 'model:b', source: 'current-turn' },
+    { id: 'config-only', source: 'config.toml:model' },
+  ])
+  assert.deepEqual(advertised, ['provider/model-a', 'model:b'])
+})
+
+test('discoverCodex (pure): empty text yields empty + note, not a throw', () => {
+  const { discovered, notes, completeness } = discoverCodex({ configText: null })
+  assert.deepEqual(discovered, [])
+  assert.equal(completeness, 'partial')
+  assert.ok(notes.length > 0)
+})
+
+test('discoverClaudeCode (pure): always carries the honest-manual note', () => {
+  const { notes } = discoverClaudeCode({ settings: {} })
+  assert.ok(notes.some((n) => /stays manual/.test(n)))
+})

package/skills/engineering/harness-runtime/fixtures/native-question-schemas.json

@@ -0,0 +1,33 @@
+{
+  "pi": {
+    "toolName": "ask_user_question",
+    "source": "@juicesharp/rpiv-ask-user-question@2.4.0 tool/types.ts and docs/tool-schema.md",
+    "questions": { "minimum": 1, "maximum": 4 },
+    "headerMaximum": 16,
+    "options": { "minimum": 2, "maximum": 4 },
+    "optionLabelMaximum": 60,
+    "multiSelect": true,
+    "secret": false,
+    "reservedLabels": ["Other", "Type something.", "Next"]
+  },
+  "codex": {
+    "toolName": "request_user_input",
+    "source": "openai/codex@3d4d253f8f4a812c595cd59e2c114c2c3696c293 codex-rs/core/src/tools/handlers/request_user_input_spec.rs",
+    "questions": { "minimum": 1, "maximum": 3 },
+    "headerMaximum": 12,
+    "options": { "minimum": 2, "maximum": 3 },
+    "questionIdFormat": "snake_case",
+    "recommendationLabelSuffix": " (Recommended)",
+    "multiSelect": false,
+    "secret": false
+  },
+  "claude-code": {
+    "toolName": "AskUserQuestion",
+    "source": "anthropics/claude-code@54cc51a08a5d3900e5abd02ad75a2ce46f3f008c plugins/plugin-dev/skills/command-development/references/interactive-commands.md",
+    "questions": { "minimum": 1, "maximum": 4 },
+    "headerMaximum": 12,
+    "options": { "minimum": 2, "maximum": 4 },
+    "multiSelect": true,
+    "secret": false
+  }
+}

package/skills/engineering/harness-runtime/fixtures/question-responses.json

@@ -0,0 +1,54 @@
+{
+  "pi": {
+    "success": {
+      "content": [{ "type": "text", "text": "User has answered your questions." }],
+      "details": {
+        "answers": [
+          { "questionIndex": 0, "question": "How should delivery proceed?", "kind": "option", "answer": "Queue" },
+          { "questionIndex": 1, "question": "How should context proceed?", "kind": "custom", "answer": "After lunch" }
+        ],
+        "cancelled": false
+      }
+    },
+    "multi": {
+      "content": [{ "type": "text", "text": "User has answered your questions." }],
+      "details": {
+        "answers": [
+          { "questionIndex": 0, "question": "How should features proceed?", "kind": "multi", "answer": null, "selected": ["Queue", "Direct"] }
+        ],
+        "cancelled": false
+      }
+    },
+    "cancelled": {
+      "content": [{ "type": "text", "text": "User declined to answer questions" }],
+      "details": { "answers": [], "cancelled": true }
+    }
+  },
+  "codex": {
+    "success": {
+      "answers": {
+        "delivery": { "answers": ["Queue (Recommended)"] },
+        "context": { "answers": ["After lunch"] }
+      }
+    },
+    "cancelled": { "cancelled": true }
+  },
+  "claude-code": {
+    "success": {
+      "questions": [],
+      "answers": {
+        "How should delivery proceed?": "Queue",
+        "How should context proceed?": "After lunch"
+      },
+      "annotations": {}
+    },
+    "multi": {
+      "questions": [],
+      "answers": {
+        "How should features proceed?": "Queue, Direct"
+      },
+      "annotations": {}
+    },
+    "cancelled": { "cancelled": true }
+  }
+}