@1aboveio/skills 0.10.0

package/skills/engineering/harness-runtime/discover-models.mjs

@@ -0,0 +1,392 @@
+// discover-models — canonical harness-runtime model availability discovery.
+// It unions explicit current-turn and human-named runnable ids with configured or
+// credentialed evidence without overstating the completeness of static config.
+// The discovered id set fed to `model-catalog.mjs tiers --seen` stops being produced
+// by an agent reading prose (which drifts into proposing models that don't exist,
+// or expanding a built-in registry into training-data noise).
+//
+// Why a script and not prose: pre-flight step 1 forbids proposing any model that
+// didn't appear in a discovery source. Described as prose, "discover the available
+// models" degrades to the agent listing plausible names from memory — the exact
+// failure the no-silent-fallback guard exists to catch (a confirmed-but-unrunnable
+// model resurfaces mid-run). Mechanized, discovery is a deterministic read of the
+// user's real config plus explicit active-turn evidence, and the never-a-source
+// rule is enforced by construction: no known-model registry is an input.
+//
+//   node bin/discover-models.mjs --harness pi
+//   node bin/discover-models.mjs --harness codex --available gpt-a,gpt-b
+//   node bin/discover-models.mjs --harness claude-code --available opus,sonnet
+//   node discover-models.mjs                         # auto-detect the sole present harness
+//   node discover-models.mjs --help                  # usage + exit 0
+//
+// OUTPUT (stdout, always JSON):
+//   { harness, completeness, discovered: [{ id, source }], notes: [<string>...] }
+// Feed the ids to `model-catalog.mjs tiers --seen <id,id,...> --harness <h>`.
+//
+// EXIT CODES:
+//   0  discovery ran (JSON emitted) — including an empty set with an explanatory
+//      note (missing config, or an honest-partial harness). Empty-but-honest is a
+//      correct outcome, not a failure.
+//   2  usage error: unknown flag, unknown --harness value, or no --harness given
+//      and the present harness could not be disambiguated.
+//
+// HARNESS COMPLETENESS (honesty over false coverage):
+//   - pi — FULL from credential-aware ~/.pi/agent/{settings,models,auth}.json.
+//   - codex — PARTIAL from config.toml defaults; FULL with current-turn spawn ids.
+//   - claude-code — PARTIAL from settings.json; FULL with current-turn Agent ids.
+// Neither Codex nor Claude Code reads model-catalog.json for availability. The
+// catalog is a cache of classification confirmations, never a runnable inventory.
+//
+// THE NEVER-A-SOURCE RULE, encoded as behavior: this script never shells out to a
+// model registry / `pi --models` expansion / a models-API dump, and never reads a
+// harness's full known-model catalog. A model the config "knows a definition for"
+// but that the user has neither enabled nor holds resolving credentials for is NOT
+// emitted — see the credential gate in discoverPi and its test.
+
+import fs from 'node:fs'
+import path from 'node:path'
+
+const HARNESSES = ['pi', 'codex', 'claude-code']
+
+// Resolve HOME from the environment first so a test can point discovery at a
+// fixture tree via `HOME=<tmp>` — os.homedir() honors $HOME on POSIX, but reading
+// it explicitly makes the test contract obvious and the injection unambiguous.
+function homeDir(env) {
+  return env.HOME || env.USERPROFILE || ''
+}
+
+// Read + JSON.parse a file, returning null when it is missing and throwing only on
+// a genuinely malformed file (a corrupt config should be loud, not silently empty).
+function readJson(file) {
+  if (!fs.existsSync(file)) return null
+  return JSON.parse(fs.readFileSync(file, 'utf8'))
+}
+
+function readText(file) {
+  if (!fs.existsSync(file)) return null
+  return fs.readFileSync(file, 'utf8')
+}
+
+function validIds(ids) {
+  return Array.isArray(ids) ? ids.filter((id) => typeof id === 'string' && id.length > 0) : []
+}
+
+function composeEvidence(configured, { spawnModelIds = [], humanNamedModelIds = [] } = {}) {
+  const discovered = []
+  const seen = new Set()
+  const emit = (id, source) => {
+    if (seen.has(id)) return
+    seen.add(id)
+    discovered.push({ id, source })
+  }
+  for (const id of validIds(spawnModelIds)) emit(id, 'current-turn')
+  for (const id of validIds(humanNamedModelIds)) emit(id, 'human-named')
+  for (const model of configured) emit(model.id, model.source)
+  return discovered
+}
+
+// ---- pure, unit-testable cores (no filesystem; env passed in) ----
+
+// Does a provider's apiKey field resolve to a usable credential?
+//   - a literal non-empty string  → resolves (how: "literal apiKey")
+//   - "$VAR"                        → resolves iff env[VAR] is non-empty
+//   - missing/empty                → does not resolve
+// This is the credential gate that makes the never-a-source rule real: a custom
+// model whose provider key does not resolve is a definition the config knows but
+// the user cannot run, so it is never emitted.
+export function apiKeyResolves(apiKey, env) {
+  if (!apiKey || typeof apiKey !== 'string') return { resolved: false, how: 'no apiKey' }
+  if (apiKey.startsWith('$')) {
+    const varName = apiKey.slice(1)
+    const val = env[varName]
+    return { resolved: !!val, how: `${varName} ${val ? 'set' : 'unset'}` }
+  }
+  return { resolved: true, how: 'literal apiKey' }
+}
+
+// Pi discovery from already-parsed config objects. Sources, in emission order:
+//   1. settings.json enabledModels[]         (the curated picker set)
+//   2. settings.json default{Provider,Model} (the active default, if not already listed)
+//   3. models.json custom models[] whose provider apiKey resolves (credentialed remainder)
+//   4. credentialed providers with NO enabledModels entry and no per-model detail,
+//      listed at PROVIDER level only (`provider/*`) — you name a model to use it.
+// Dedupe by id (first source wins). Uncredentialed custom models and the built-in
+// registry are never emitted.
+export function discoverPi({ settings, models, auth, env }) {
+  const discovered = []
+  const notes = []
+  const seen = new Set()
+  const emit = (id, source) => {
+    if (seen.has(id)) return
+    seen.add(id)
+    discovered.push({ id, source })
+  }
+
+  if (!settings && !models) {
+    notes.push('~/.pi/agent/settings.json and models.json not found — nothing to discover for pi')
+  }
+
+  const enabled = Array.isArray(settings?.enabledModels) ? settings.enabledModels : []
+  for (const id of enabled) {
+    if (typeof id === 'string' && id.length) emit(id, 'settings.json:enabledModels')
+  }
+
+  // The active default is a runnable model even if it were somehow absent from
+  // enabledModels; emit it (deduped) so the default can never be dropped.
+  if (settings?.defaultProvider && settings?.defaultModel) {
+    emit(`${settings.defaultProvider}/${settings.defaultModel}`, 'settings.json:default')
+  }
+
+  // The credentialed-but-uncurated remainder from models.json + auth.json.
+  const providers = (models && typeof models.providers === 'object' && models.providers) || {}
+  const authMap = (auth && typeof auth === 'object' && auth) || {}
+  const providerNames = new Set([...Object.keys(providers), ...Object.keys(authMap)])
+
+  for (const provider of providerNames) {
+    const cfg = providers[provider] || {}
+    const key = apiKeyResolves(cfg.apiKey, env)
+    const authCredentialed = !!authMap[provider]
+    const credentialed = key.resolved || authCredentialed
+    if (!credentialed) continue // credential gate: never emit an unrunnable definition
+
+    const credHow = key.resolved ? key.how : 'auth.json entry'
+    const custom = Array.isArray(cfg.models) ? cfg.models : []
+    const hasEnabledEntry = enabled.some((e) => typeof e === 'string' && e.split('/')[0] === provider)
+
+    if (custom.length) {
+      // Per-model detail available: emit each custom model (deduped against enabled).
+      for (const m of custom) {
+        if (m && typeof m.id === 'string' && m.id.length) {
+          emit(`${provider}/${m.id}`, `models.json:custom (${credHow})`)
+        }
+      }
+    } else if (!hasEnabledEntry) {
+      // Credentialed provider with no curated entry and no per-model detail: you can
+      // run it, but only by naming a model. List at PROVIDER level only — NEVER
+      // expand the provider's registry.
+      emit(`${provider}/*`, `credentialed provider, no enabledModels entry (${credHow})`)
+    }
+  }
+
+  notes.push(
+    'Pi\'s built-in model registry / `pi --models` expansion is never a discovery source — ' +
+      'only enabledModels, the default, credentialed models.json custom entries, and credentialed ' +
+      'providers are read. A model the config defines but the user has not enabled or credentialed is omitted.',
+  )
+
+  return { discovered, notes }
+}
+
+// Codex discovery unions current-turn spawn ids with the `model = "..."` defaults
+// in config.toml. Config alone is partial: it does not enumerate every model the
+// current runtime can launch. Human-named runnable ids are evidence but likewise
+// cannot prove that the resulting set is complete.
+export function discoverCodex({ configText, spawnModelIds = [], humanNamedModelIds = [] }) {
+  const discovered = []
+  const notes = []
+  const seen = new Set()
+
+  if (configText == null) {
+    notes.push('~/.codex/config.toml not found — no configured Codex defaults to add')
+    if (spawnModelIds.length === 0) {
+      notes.push('Codex discovery is partial without current-turn spawn model ids or runtime enumeration.')
+    }
+    return {
+      discovered: composeEvidence(discovered, { spawnModelIds, humanNamedModelIds }),
+      notes,
+      completeness: spawnModelIds.length > 0 ? 'full' : 'partial',
+    }
+  }
+
+  let table = ''
+  for (const rawLine of configText.split('\n')) {
+    const line = rawLine.replace(/#.*$/, '') // strip trailing comments
+    const header = /^\s*\[([^\]]+)\]/.exec(line)
+    if (header) {
+      table = header[1]
+      continue
+    }
+    // Match the `model` key exactly — `model_reasoning_effort`, `model_context_window`
+    // etc. have an underscore before `=` and are excluded by the `\s*=`.
+    const m = /^\s*model\s*=\s*["']([^"']+)["']/.exec(line)
+    if (m) {
+      const id = m[1]
+      if (!seen.has(id)) {
+        seen.add(id)
+        discovered.push({ id, source: table ? `config.toml:[${table}]` : 'config.toml:model' })
+      }
+    }
+  }
+
+  if (discovered.length === 0) {
+    notes.push('~/.codex/config.toml has no `model = "..."` key — no configured Codex defaults to add')
+  }
+  if (spawnModelIds.length === 0) {
+    notes.push('Codex discovery is partial without current-turn spawn model ids or runtime enumeration.')
+  }
+  return {
+    discovered: composeEvidence(discovered, { spawnModelIds, humanNamedModelIds }),
+    notes,
+    completeness: spawnModelIds.length > 0 ? 'full' : 'partial',
+  }
+}
+
+// Claude Code discovery — HONEST-PARTIAL. settings.json carries no enumerable
+// model list; the runnable set is the harness's built-in Agent-tool model enum,
+// which is not a file a script can read. Emit the settings `model` field only if
+// one is somehow present, and always record the honest note.
+export function discoverClaudeCode({ settings }) {
+  const discovered = []
+  const notes = []
+
+  if (settings && typeof settings.model === 'string' && settings.model.length) {
+    discovered.push({ id: settings.model, source: 'settings.json:model' })
+  }
+
+  notes.push(
+    'Claude Code\'s runnable model set is the harness\'s built-in Agent-tool model enum, which is ' +
+      'not a file this script can read — discovery for this harness stays manual: enumerate the Agent ' +
+      'tool\'s model options in-session. ~/.claude/model-catalog.json is the banked catalog (a cache of ' +
+      'confirmations), never an availability source, so it is deliberately not read here.',
+  )
+  return { discovered, notes }
+}
+
+// ---- filesystem layer (reads real config under home; calls the pure cores) ----
+
+export function discover(harness, env, options = {}) {
+  const home = homeDir(env)
+  if (harness === 'pi') {
+    const settings = readJson(path.join(home, '.pi', 'agent', 'settings.json'))
+    const models = readJson(path.join(home, '.pi', 'agent', 'models.json'))
+    const auth = readJson(path.join(home, '.pi', 'agent', 'auth.json'))
+    const configured = discoverPi({ settings, models, auth, env })
+    return {
+      harness,
+      completeness: 'full',
+      discovered: composeEvidence(configured.discovered, options),
+      notes: configured.notes,
+    }
+  }
+  if (harness === 'codex') {
+    const configText = readText(path.join(home, '.codex', 'config.toml'))
+    const result = discoverCodex({ configText, ...options })
+    return { harness, ...result }
+  }
+  if (harness === 'claude-code') {
+    const settings = readJson(path.join(home, '.claude', 'settings.json'))
+    const configured = discoverClaudeCode({ settings })
+    const spawnModelIds = validIds(options.spawnModelIds)
+    return {
+      harness,
+      completeness: spawnModelIds.length > 0 ? 'full' : 'partial',
+      discovered: composeEvidence(configured.discovered, options),
+      notes: configured.notes,
+    }
+  }
+  throw new Error(`unknown harness "${harness}"`)
+}
+
+// Auto-detect the harness from which config files exist under home. Returns the
+// sole present harness, or null when zero or more than one are present (the caller
+// then requires an explicit --harness rather than guessing).
+export function detectHarness(env) {
+  const home = homeDir(env)
+  const present = []
+  if (fs.existsSync(path.join(home, '.pi', 'agent', 'settings.json')) ||
+      fs.existsSync(path.join(home, '.pi', 'agent', 'models.json'))) present.push('pi')
+  if (fs.existsSync(path.join(home, '.codex', 'config.toml'))) present.push('codex')
+  if (fs.existsSync(path.join(home, '.claude', 'settings.json'))) present.push('claude-code')
+  return present.length === 1 ? present[0] : null
+}
+
+// ---- CLI ----
+
+const USAGE = `discover-models — enumerate runnable model evidence without overstating completeness.
+Usage:
+  discover-models --harness <pi|codex|claude-code> [--available <ids>] [--human-named <ids>]
+  discover-models                       (auto-detect when exactly one harness config is present)
+Options:
+  --harness <h>       which harness to read: pi | codex | claude-code
+  --available <ids>   comma-separated ids advertised by the current-turn spawn primitive
+                      (alias: --spawn-model-ids)
+  --human-named <ids> comma-separated ids a human explicitly identified as runnable
+  --help, -h          show this help and exit 0
+Output: JSON on stdout — { harness, completeness, discovered: [{id, source}], notes }.
+  Feed the ids to \`model-catalog.mjs tiers --seen <id,id,...> --harness <h>\`.
+Exit codes: 0 = discovery ran (JSON emitted, possibly an empty set with a note),
+            2 = usage error (unknown flag, unknown --harness, or ambiguous auto-detect).
+Mechanization: Pi's credential-aware config is full. Codex and Claude Code config is partial;
+current-turn advertised ids make their discovery full. Human-named ids remain valid evidence
+without claiming completeness. No registry / \`pi --models\` / models-API dump is read.`
+
+function parseArgs(argv) {
+  const f = {}
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i]
+    if (a === '--help' || a === '-h' || a === 'help') f.help = true
+    else if (a === '--harness') f.harness = argv[++i]
+    else if (a.startsWith('--harness=')) f.harness = a.slice('--harness='.length)
+    else if (a === '--available' || a === '--spawn-model-ids') f.available = argv[++i]
+    else if (a.startsWith('--available=')) f.available = a.slice('--available='.length)
+    else if (a.startsWith('--spawn-model-ids=')) f.available = a.slice('--spawn-model-ids='.length)
+    else if (a === '--human-named') f.humanNamed = argv[++i]
+    else if (a.startsWith('--human-named=')) f.humanNamed = a.slice('--human-named='.length)
+    else f.unknown = a
+  }
+  return f
+}
+
+export function runCli(argv, env = process.env) {
+  const f = parseArgs(argv)
+  if (f.help) {
+    console.log(USAGE)
+    return 0
+  }
+  if (f.unknown) {
+    console.error(`discover-models: unknown argument "${f.unknown}"\n${USAGE}`)
+    return 2
+  }
+
+  let harness = f.harness
+  if (harness === undefined) {
+    harness = detectHarness(env)
+    if (!harness) {
+      console.error(
+        'discover-models: no --harness given and could not auto-detect a single harness ' +
+          `(present configs are ambiguous or absent). Pass --harness <${HARNESSES.join('|')}>.\n${USAGE}`,
+      )
+      return 2
+    }
+  }
+  if (!HARNESSES.includes(harness)) {
+    console.error(`discover-models: unknown --harness "${harness}" (expected ${HARNESSES.join(' | ')})\n${USAGE}`)
+    return 2
+  }
+
+  const parseIds = (flag, raw) => {
+    if (raw === undefined) return []
+    const ids = raw.split(',').map((id) => id.trim()).filter(Boolean)
+    if (ids.length === 0) throw new Error(`${flag} requires at least one model id`)
+    return [...new Set(ids)]
+  }
+
+  let spawnModelIds
+  let humanNamedModelIds
+  try {
+    spawnModelIds = parseIds('--available', f.available)
+    humanNamedModelIds = parseIds('--human-named', f.humanNamed)
+  } catch (error) {
+    console.error(`discover-models: ${error.message}\n${USAGE}`)
+    return 2
+  }
+
+  const result = discover(harness, env, { spawnModelIds, humanNamedModelIds })
+  console.log(JSON.stringify(result, null, 2))
+  // A one-line human cue on stderr for someone running it by hand.
+  console.error(
+    `discover-models: ${harness} — ${result.discovered.length} model(s) discovered; ` +
+      'feed the ids to `model-catalog.mjs tiers --seen`.',
+  )
+  return 0
+}

package/skills/engineering/harness-runtime/fixtures/native-question-schemas.json

@@ -0,0 +1,33 @@
+{
+  "pi": {
+    "toolName": "ask_user_question",
+    "source": "@juicesharp/rpiv-ask-user-question@2.4.0 tool/types.ts and docs/tool-schema.md",
+    "questions": { "minimum": 1, "maximum": 4 },
+    "headerMaximum": 16,
+    "options": { "minimum": 2, "maximum": 4 },
+    "optionLabelMaximum": 60,
+    "multiSelect": true,
+    "secret": false,
+    "reservedLabels": ["Other", "Type something.", "Next"]
+  },
+  "codex": {
+    "toolName": "request_user_input",
+    "source": "openai/codex@3d4d253f8f4a812c595cd59e2c114c2c3696c293 codex-rs/core/src/tools/handlers/request_user_input_spec.rs",
+    "questions": { "minimum": 1, "maximum": 3 },
+    "headerMaximum": 12,
+    "options": { "minimum": 2, "maximum": 3 },
+    "questionIdFormat": "snake_case",
+    "recommendationLabelSuffix": " (Recommended)",
+    "multiSelect": false,
+    "secret": false
+  },
+  "claude-code": {
+    "toolName": "AskUserQuestion",
+    "source": "anthropics/claude-code@54cc51a08a5d3900e5abd02ad75a2ce46f3f008c plugins/plugin-dev/skills/command-development/references/interactive-commands.md",
+    "questions": { "minimum": 1, "maximum": 4 },
+    "headerMaximum": 12,
+    "options": { "minimum": 2, "maximum": 4 },
+    "multiSelect": true,
+    "secret": false
+  }
+}

package/skills/engineering/harness-runtime/fixtures/question-responses.json

@@ -0,0 +1,54 @@
+{
+  "pi": {
+    "success": {
+      "content": [{ "type": "text", "text": "User has answered your questions." }],
+      "details": {
+        "answers": [
+          { "questionIndex": 0, "question": "How should delivery proceed?", "kind": "option", "answer": "Queue" },
+          { "questionIndex": 1, "question": "How should context proceed?", "kind": "custom", "answer": "After lunch" }
+        ],
+        "cancelled": false
+      }
+    },
+    "multi": {
+      "content": [{ "type": "text", "text": "User has answered your questions." }],
+      "details": {
+        "answers": [
+          { "questionIndex": 0, "question": "How should features proceed?", "kind": "multi", "answer": null, "selected": ["Queue", "Direct"] }
+        ],
+        "cancelled": false
+      }
+    },
+    "cancelled": {
+      "content": [{ "type": "text", "text": "User declined to answer questions" }],
+      "details": { "answers": [], "cancelled": true }
+    }
+  },
+  "codex": {
+    "success": {
+      "answers": {
+        "delivery": { "answers": ["Queue (Recommended)"] },
+        "context": { "answers": ["After lunch"] }
+      }
+    },
+    "cancelled": { "cancelled": true }
+  },
+  "claude-code": {
+    "success": {
+      "questions": [],
+      "answers": {
+        "How should delivery proceed?": "Queue",
+        "How should context proceed?": "After lunch"
+      },
+      "annotations": {}
+    },
+    "multi": {
+      "questions": [],
+      "answers": {
+        "How should features proceed?": "Queue, Direct"
+      },
+      "annotations": {}
+    },
+    "cancelled": { "cancelled": true }
+  }
+}