@pellux/goodvibes-agent 0.1.53 → 0.1.55

package/src/runtime/services.ts

@@ -136,6 +136,114 @@ function ensureConfiguredModelIsRoutable(providerRegistry: ProviderRegistry, con
   });
 }
 
+const PROVIDER_STARTUP_PLACEHOLDER_API_KEY = 'goodvibes-agent-startup-placeholder';
+
+type ProviderRegistryConstructionOptions = ConstructorParameters<typeof ProviderRegistry>[0];
+
+type ProviderStartupEnv = {
+  readonly providerId: string;
+  readonly envVars: readonly string[];
+};
+
+type MutableApiKeyProvider = {
+  apiKey: string;
+};
+
+type MutableConfiguredProvider = {
+  configured: boolean;
+};
+
+const PROVIDER_STARTUP_PLACEHOLDER_ENVS: readonly ProviderStartupEnv[] = [
+  { providerId: 'openai', envVars: ['OPENAI_API_KEY', 'OPENAI_KEY'] },
+  { providerId: 'inceptionlabs', envVars: ['INCEPTION_API_KEY'] },
+  { providerId: 'openrouter', envVars: ['OPENROUTER_API_KEY'] },
+  { providerId: 'aihubmix', envVars: ['AIHUBMIX_API_KEY'] },
+  { providerId: 'groq', envVars: ['GROQ_API_KEY'] },
+  { providerId: 'cerebras', envVars: ['CEREBRAS_API_KEY'] },
+  { providerId: 'mistral', envVars: ['MISTRAL_API_KEY'] },
+  { providerId: 'ollama-cloud', envVars: ['OLLAMA_CLOUD_API_KEY', 'OLLAMA_API_KEY'] },
+  { providerId: 'huggingface', envVars: ['HF_API_KEY', 'HUGGINGFACE_API_KEY', 'HF_TOKEN'] },
+  { providerId: 'nvidia', envVars: ['NVIDIA_API_KEY'] },
+  { providerId: 'llm7', envVars: ['LLM7_API_KEY'] },
+  { providerId: 'deepseek', envVars: ['DEEPSEEK_API_KEY'] },
+  { providerId: 'fireworks', envVars: ['FIREWORKS_API_KEY'] },
+  { providerId: 'microsoft-foundry', envVars: ['AZURE_OPENAI_API_KEY'] },
+  { providerId: 'moonshot', envVars: ['MOONSHOT_API_KEY'] },
+  { providerId: 'qianfan', envVars: ['QIANFAN_API_KEY'] },
+  { providerId: 'qwen', envVars: ['QWEN_API_KEY', 'DASHSCOPE_API_KEY', 'MODELSTUDIO_API_KEY'] },
+  { providerId: 'sglang', envVars: ['SGLANG_API_KEY'] },
+  { providerId: 'stepfun', envVars: ['STEPFUN_API_KEY'] },
+  { providerId: 'together', envVars: ['TOGETHER_API_KEY'] },
+  { providerId: 'venice', envVars: ['VENICE_API_KEY'] },
+  { providerId: 'volcengine', envVars: ['VOLCANO_ENGINE_API_KEY'] },
+  { providerId: 'xai', envVars: ['XAI_API_KEY'] },
+  { providerId: 'xiaomi', envVars: ['XIAOMI_API_KEY'] },
+  { providerId: 'zai', envVars: ['ZAI_API_KEY', 'Z_AI_API_KEY'] },
+  { providerId: 'cloudflare-ai-gateway', envVars: ['CLOUDFLARE_AI_GATEWAY_API_KEY'] },
+  { providerId: 'vercel-ai-gateway', envVars: ['AI_GATEWAY_API_KEY'] },
+  { providerId: 'litellm', envVars: ['LITELLM_API_KEY'] },
+  { providerId: 'copilot-proxy', envVars: ['COPILOT_PROXY_API_KEY'] },
+];
+
+function hasMutableApiKeyProvider(value: unknown): value is MutableApiKeyProvider {
+  if (typeof value !== 'object' || value === null) return false;
+  const candidate = value as { readonly apiKey?: unknown };
+  return typeof candidate.apiKey === 'string';
+}
+
+function hasMutableConfiguredProvider(value: unknown): value is MutableConfiguredProvider {
+  if (typeof value !== 'object' || value === null) return false;
+  const candidate = value as { readonly configured?: unknown };
+  return typeof candidate.configured === 'boolean';
+}
+
+function hasAnyConfiguredEnv(envVars: readonly string[]): boolean {
+  return envVars.some((envVar) => {
+    const value = process.env[envVar];
+    return typeof value === 'string' && value.trim().length > 0;
+  });
+}
+
+export function createLaunchTolerantProviderRegistry(options: ProviderRegistryConstructionOptions): ProviderRegistry {
+  const placeholders = PROVIDER_STARTUP_PLACEHOLDER_ENVS
+    .filter((entry) => !hasAnyConfiguredEnv(entry.envVars))
+    .map((entry) => ({ providerId: entry.providerId, envVar: entry.envVars[0] }))
+    .filter((entry): entry is { readonly providerId: string; readonly envVar: string } => typeof entry.envVar === 'string');
+
+  if (placeholders.length === 0) {
+    return new ProviderRegistry(options);
+  }
+
+  const previousValues = new Map<string, string | undefined>();
+  for (const placeholder of placeholders) {
+    previousValues.set(placeholder.envVar, process.env[placeholder.envVar]);
+    process.env[placeholder.envVar] = PROVIDER_STARTUP_PLACEHOLDER_API_KEY;
+  }
+  let providerRegistry: ProviderRegistry;
+  try {
+    providerRegistry = new ProviderRegistry(options);
+  } finally {
+    for (const [envVar, previousValue] of previousValues) {
+      if (previousValue === undefined) {
+        delete process.env[envVar];
+      } else {
+        process.env[envVar] = previousValue;
+      }
+    }
+  }
+
+  for (const placeholder of placeholders) {
+    const provider = providerRegistry.get(placeholder.providerId);
+    if (hasMutableApiKeyProvider(provider) && provider.apiKey === PROVIDER_STARTUP_PLACEHOLDER_API_KEY) {
+      provider.apiKey = '';
+    }
+    if (hasMutableConfiguredProvider(provider)) {
+      provider.configured = false;
+    }
+  }
+  return providerRegistry;
+}
+
 export interface RuntimeServicesOptions {
   readonly runtimeBus: RuntimeEventBus;
   readonly runtimeStore: RuntimeStore;
@@ -285,7 +393,7 @@ export function createRuntimeServices(options: RuntimeServicesOptions): RuntimeS
   const modelLimitsService = new ModelLimitsService({
     cachePath: shellPaths.resolveUserPath(GOODVIBES_AGENT_SURFACE_ROOT, 'model-limits.json'),
   });
-  const providerRegistry = new ProviderRegistry({
+  const providerRegistry = createLaunchTolerantProviderRegistry({
     configManager,
     subscriptionManager,
     secretsManager,

package/src/version.ts

@@ -6,7 +6,7 @@ import { join } from 'node:path';
 // The prebuild script updates the fallback value before compilation.
 // Uses import.meta.dir (Bun) to locate package.json relative to this file,
 // which is correct regardless of the process working directory.
-let _version = '0.1.53';
+let _version = '0.1.55';
 let _sdkVersion = '0.33.35';
 try {
   const pkg = JSON.parse(readFileSync(join(import.meta.dir, '..', 'package.json'), 'utf-8')) as {

package/src/input/commands/eval.ts

@@ -1,217 +0,0 @@
-/**
- * /eval command handler.
- *
- * Implements the Evaluation Harness commands:
- *
- *   /eval list                    — List all available eval suites
- *   /eval run <suite> --yes       — Run a named suite (or 'all')
- *   /eval compare <baseline-file> — Compare last run against a baseline file
- *   /eval gate <suite> --yes      — Run suite and apply CI gate (exits 1 on regression)
- */
-
-import type { SlashCommand, CommandContext } from '../command-registry.ts';
-import { EvalRunner } from '@/runtime/index.ts';
-import { BUILTIN_SUITES } from '@/runtime/index.ts';
-import { formatScorecard } from '@/runtime/index.ts';
-import { loadBaseline, captureBaseline, formatBaselineComparison, writeBaseline } from '@/runtime/index.ts';
-import type { EvalRegistry } from '../../panels/eval-panel.ts';
-import { formatSuiteResult, formatGateResult } from '@/runtime/index.ts';
-import { requireShellPaths } from './runtime-services.ts';
-import { summarizeError } from '@pellux/goodvibes-sdk/platform/utils';
-import { requireYesFlag, stripYesFlag } from './confirmation.ts';
-
-// ── Subcommand helpers ────────────────────────────────────────────────────────
-
-function printSuiteList(context: CommandContext): void {
-  context.print('[eval] Available suites:');
-  for (const [name, scenarios] of Object.entries(BUILTIN_SUITES)) {
-    context.print(`  ${name}  (${scenarios.length} scenarios)`);
-    for (const s of scenarios) {
-      context.print(`    - ${s.id}: ${s.name}`);
-    }
-  }
-  context.print('[eval] Usage: /eval run <suite> --yes  or  /eval run all --yes');
-}
-
-function getRegistry(context: CommandContext): EvalRegistry | undefined {
-  return context.extensions.evalRegistry;
-}
-
-// ── /eval list ────────────────────────────────────────────────────────────────
-
-function handleList(_args: string[], context: CommandContext): void {
-  printSuiteList(context);
-}
-
-// ── /eval run ────────────────────────────────────────────────────────────────
-
-async function handleRun(args: string[], context: CommandContext): Promise<void> {
-  const { rest, yes } = stripYesFlag(args);
-  const suiteName = rest[0] ?? 'all';
-  const registry = getRegistry(context);
-
-  const suitesToRun =
-    suiteName === 'all'
-      ? Object.keys(BUILTIN_SUITES)
-      : BUILTIN_SUITES[suiteName]
-        ? [suiteName]
-        : null;
-
-  if (!suitesToRun) {
-    context.print(`[eval] Unknown suite: "${suiteName}". Run /eval list to see available suites.`);
-    return;
-  }
-  if (!yes) {
-    requireYesFlag(context, `run eval suite ${suiteName}`, '/eval run <suite|all> --yes');
-    return;
-  }
-
-  const runner = new EvalRunner();
-  registry?.setRunning(true);
-
-  for (const name of suitesToRun) {
-    const scenarios = BUILTIN_SUITES[name];
-    if (!scenarios) continue;
-
-    context.print(`[eval] Running suite: ${name} (${scenarios.length} scenarios)...`);
-    const result = await runner.runSuite(name, scenarios);
-    registry?.push(result);
-
-    context.print(formatSuiteResult(result));
-
-    for (const r of result.results) {
-      context.print(formatScorecard(r.scorecard));
-    }
-  }
-
-  registry?.setRunning(false);
-}
-
-// ── /eval compare ─────────────────────────────────────────────────────────────
-
-async function handleCompare(args: string[], context: CommandContext): Promise<void> {
-  const baselineFile = args[0] ?? '.goodvibes/eval/baseline.json';
-  const registry = getRegistry(context);
-  const projectRoot = requireShellPaths(context).workingDirectory;
-  const suiteResults = registry?.getSuiteResults() ?? [];
-
-  if (suiteResults.length === 0) {
-    context.print('[eval] No suite results to compare. Run /eval run <suite> --yes first.');
-    return;
-  }
-
-  const baseline = await loadBaseline(baselineFile, projectRoot);
-  if (!baseline) {
-    context.print(`[eval] Baseline file not found: ${baselineFile}`);
-    context.print('[eval] Tip: run /eval gate <suite> [baseline-file] --save-baseline --yes to create a baseline.');
-    return;
-  }
-
-  for (const result of suiteResults) {
-    context.print(formatBaselineComparison(baseline, result));
-  }
-}
-
-// ── /eval gate ────────────────────────────────────────────────────────────────
-
-async function handleGate(args: string[], context: CommandContext): Promise<void> {
-  const { rest, yes } = stripYesFlag(args);
-  const positional = rest.filter((arg) => arg !== '--save-baseline');
-  const suiteName = positional[0];
-  const baselineFile = positional[1] ?? '.goodvibes/eval/baseline.json';
-  const saveFlag = rest.includes('--save-baseline');
-  const projectRoot = requireShellPaths(context).workingDirectory;
-
-  if (!suiteName) {
-    context.print('[eval] Usage: /eval gate <suite> [baseline-file] [--save-baseline] --yes');
-    return;
-  }
-
-  const scenarios = BUILTIN_SUITES[suiteName];
-  if (!scenarios) {
-    context.print(`[eval] Unknown suite: "${suiteName}". Run /eval list to see available suites.`);
-    return;
-  }
-  if (!yes) {
-    requireYesFlag(context, `run eval gate ${suiteName}`, '/eval gate <suite> [baseline-file] [--save-baseline] --yes');
-    return;
-  }
-
-  const registry = getRegistry(context);
-  const runner = new EvalRunner();
-
-  context.print(`[eval] Gate: running suite "${suiteName}"...`);
-  registry?.setRunning(true);
-  const fresh = await runner.runSuite(suiteName, scenarios);
-  registry?.push(fresh);
-  registry?.setRunning(false);
-
-  const baseline = await loadBaseline(baselineFile, projectRoot);
-  const gate = runner.evaluateGate(fresh, baseline);
-  registry?.pushGate(gate);
-
-  context.print(formatGateResult(gate));
-
-  if (saveFlag || !baseline) {
-    const label = suiteName ?? 'latest';
-    const newBaseline = captureBaseline(label, [fresh]);
-    try {
-      await writeBaseline(baselineFile, newBaseline, projectRoot);
-      context.print(`[eval] Baseline saved to ${baselineFile}`);
-    } catch (err) {
-      context.print(`[eval] Warning: could not save baseline: ${summarizeError(err)}`);
-    }
-  }
-
-  if (!gate.passed) {
-    context.print(`[eval] Gate FAILED: ${gate.regressions.length} regression(s) detected.`);
-  } else {
-    context.print('[eval] Gate PASSED.');
-  }
-}
-
-// ── Top-level command ─────────────────────────────────────────────────────────
-
-export const evalCommand: SlashCommand = {
-  name: 'eval',
-  description: 'Evaluation harness: run benchmark suites, compare baselines, and gate regressions.',
-  usage: '<subcommand> [args]',
-  argsHint: 'list|run <suite> --yes|compare <baseline>|gate <suite> --yes',
-  handler: async (args: string[], context: CommandContext): Promise<void> => {
-    const [sub, ...rest] = args;
-
-    switch (sub) {
-      case 'list':
-      case 'ls':
-        handleList(rest, context);
-        break;
-
-      case 'run':
-        await handleRun(rest, context);
-        break;
-
-      case 'compare':
-      case 'cmp':
-        await handleCompare(rest, context);
-        break;
-
-      case 'gate':
-        await handleGate(rest, context);
-        break;
-
-      default: {
-        const usage = [
-          'Usage: /eval <subcommand>',
-          '  list                           — List all available eval suites',
-          '  run <suite|all> --yes          — Run a named suite (or all suites)',
-          '  compare [baseline-file]        — Compare last results against baseline',
-          '  gate <suite> [baseline-file] --yes',
-          '                                 — Run suite and apply regression gate',
-          '    --save-baseline              — Save fresh run as new baseline',
-        ].join('\n');
-        context.print(usage);
-        break;
-      }
-    }
-  },
-};