@pellux/goodvibes-agent 1.1.6 → 1.2.0

package/src/tools/agent-harness-local-model-benchmarks.ts

@@ -0,0 +1,199 @@
+import type { ArtifactDescriptor } from '@pellux/goodvibes-sdk/platform/artifacts';
+import type { CommandContext } from '../input/command-registry.ts';
+import { previewHarnessText } from './agent-harness-text.ts';
+import { readArtifactStore } from './agent-harness-model-catalog.ts';
+import { localStackFor } from './agent-harness-local-model-endpoints.ts';
+import type { LocalModelBenchmarkEvidence, LocalModelBenchmarkPlan, LocalModelBenchmarkWinner, LocalModelRecipe } from './agent-harness-model-routing-types.ts';
+import { readRecord, readString } from './agent-harness-model-routing-utils.ts';
+
+export function isLocalModelBenchmarkArtifact(artifact: ArtifactDescriptor): boolean {
+  const purpose = readString(artifact.metadata.purpose);
+  if (purpose !== 'agent-model-compare') return false;
+  if (readString(artifact.metadata.benchmarkKind) === 'local-model-route') return true;
+  const promptPreview = readString(artifact.metadata.promptPreview).toLowerCase();
+  return promptPreview.includes('local model benchmark') || promptPreview.includes('benchmark this local route');
+}
+
+export function isModelCompareJudgmentArtifact(artifact: ArtifactDescriptor): boolean {
+  return readString(artifact.metadata.purpose) === 'agent-model-compare-judgment';
+}
+
+export function benchmarkCreatedAt(artifact: ArtifactDescriptor): string | null {
+  const timestamp = typeof artifact.createdAt === 'number' ? artifact.createdAt : null;
+  return timestamp == null ? null : new Date(timestamp).toISOString();
+}
+
+export function describeLocalBenchmarkArtifact(artifact: ArtifactDescriptor): Record<string, unknown> {
+  return {
+    artifactId: artifact.id,
+    ...(artifact.filename ? { filename: artifact.filename } : {}),
+    createdAt: benchmarkCreatedAt(artifact),
+    comparisonId: readString(artifact.metadata.comparisonId) || null,
+    promptPreview: previewHarnessText(readString(artifact.metadata.promptPreview) || 'local model benchmark', 120),
+    candidateCount: artifact.metadata.candidateCount ?? null,
+    completedCandidates: artifact.metadata.completedCandidates ?? null,
+    benchmarkKind: readString(artifact.metadata.benchmarkKind) || 'local-model-route',
+    reviewRoute: `agent_model_compare review artifactId:"${artifact.id}"`,
+    revealRoute: `agent_model_compare reveal artifactId:"${artifact.id}"`,
+  };
+}
+
+export function describeLocalBenchmarkJudgment(artifact: ArtifactDescriptor): Record<string, unknown> {
+  const winnerModel = readString(artifact.metadata.winnerModel);
+  const sourceArtifactId = readString(artifact.metadata.sourceArtifactId);
+  return {
+    artifactId: artifact.id,
+    ...(artifact.filename ? { filename: artifact.filename } : {}),
+    createdAt: benchmarkCreatedAt(artifact),
+    judgmentId: readString(artifact.metadata.judgmentId) || null,
+    comparisonId: readString(artifact.metadata.comparisonId) || null,
+    sourceArtifactId: sourceArtifactId || null,
+    winnerBlindId: readString(artifact.metadata.winnerBlindId) || null,
+    revealIncludedInJudgment: artifact.metadata.revealIncludedInJudgment === true,
+    winnerModel: winnerModel || null,
+    winnerStack: winnerModel ? localStackFor(winnerModel) : null,
+    promptPreview: previewHarnessText(readString(artifact.metadata.promptPreview) || 'local model benchmark judgment', 120),
+    analyticsRoute: 'agent_model_compare analytics benchmarkKind:"local-model-route" includeReasons:true',
+    exportRoute: `agent_model_compare export artifactId:"${artifact.id}" confirm:true explicitUserRequest:"Export this local benchmark judgment."`,
+    applyRoute: winnerModel
+      ? `agent_model_compare apply artifactId:"${artifact.id}" confirm:true explicitUserRequest:"Apply this revealed local benchmark winner."`
+      : null,
+  };
+}
+
+export function localBenchmarkEvidence(
+  comparisons: readonly ArtifactDescriptor[],
+  judgments: readonly ArtifactDescriptor[],
+  storeAvailable: boolean,
+): LocalModelBenchmarkEvidence {
+  if (!storeAvailable) {
+    return {
+      status: 'unavailable',
+      comparisonCount: 0,
+      completedCandidateCount: 0,
+      revealedJudgmentCount: 0,
+      hiddenJudgmentCount: 0,
+      winnerStacks: [],
+      winnerModels: [],
+      summary: 'Artifact history is unavailable in this runtime.',
+      confidence: 'estimated',
+    };
+  }
+  const winnerModels: LocalModelBenchmarkWinner[] = [];
+  let hiddenJudgmentCount = 0;
+  for (const judgment of judgments) {
+    const winnerModel = readString(judgment.metadata.winnerModel);
+    if (!winnerModel) {
+      hiddenJudgmentCount += 1;
+      continue;
+    }
+    const sourceArtifactId = readString(judgment.metadata.sourceArtifactId);
+    winnerModels.push({
+      judgmentArtifactId: judgment.id,
+      sourceArtifactId: sourceArtifactId || null,
+      registryKey: winnerModel,
+      stack: localStackFor(winnerModel),
+      promptPreview: previewHarnessText(readString(judgment.metadata.promptPreview) || 'local model benchmark judgment', 120),
+      reviewRoute: sourceArtifactId
+        ? `agent_model_compare review artifactId:"${sourceArtifactId}"`
+        : 'agent_model_compare review',
+      exportRoute: `agent_model_compare export artifactId:"${judgment.id}" confirm:true explicitUserRequest:"Export this local benchmark judgment."`,
+      applyRoute: `agent_model_compare apply artifactId:"${judgment.id}" confirm:true explicitUserRequest:"Apply this revealed local benchmark winner."`,
+    });
+  }
+  const winnerStacks = [...new Set(winnerModels.map((winner) => winner.stack).filter((stack): stack is string => Boolean(stack)))].sort((a, b) => a.localeCompare(b));
+  const completedCandidateCount = comparisons.reduce((total, artifact) => {
+    const value = artifact.metadata.completedCandidates;
+    return total + (typeof value === 'number' && Number.isFinite(value) ? value : 0);
+  }, 0);
+  const status: LocalModelBenchmarkEvidence['status'] = winnerModels.length > 0
+    ? 'reviewed-winner'
+    : comparisons.length > 0
+      ? 'comparison-saved'
+      : 'unmeasured';
+  return {
+    status,
+    comparisonCount: comparisons.length,
+    completedCandidateCount,
+    revealedJudgmentCount: winnerModels.length,
+    hiddenJudgmentCount,
+    winnerStacks,
+    winnerModels,
+    summary: winnerModels.length > 0
+      ? `Reviewed benchmark winner(s): ${winnerModels.map((winner) => winner.registryKey).join(', ')}.`
+      : comparisons.length > 0
+        ? 'Saved local benchmark comparison exists; save a revealed judgment before route recommendations.'
+        : 'No saved local benchmark comparison has been recorded yet.',
+    confidence: winnerModels.length > 0 ? 'measured' : 'estimated',
+  };
+}
+
+export function localModelBenchmarkHistory(context: CommandContext, includeParameters: boolean): Record<string, unknown> {
+  const store = readArtifactStore(context);
+  if (!store?.list) {
+    const evidence = localBenchmarkEvidence([], [], false);
+    return {
+      status: 'unavailable',
+      count: 0,
+      reason: 'Artifact history is unavailable in this runtime.',
+      saveRoute: 'agent_model_compare run benchmarkKind:"local-model-route" taskType:"local-model-route" confirm:true explicitUserRequest:"..."',
+      evidence,
+    };
+  }
+  const allArtifacts = store.list(100);
+  const artifacts = allArtifacts.filter(isLocalModelBenchmarkArtifact)
+    .slice(0, includeParameters ? 10 : 3);
+  const localComparisonIds = new Set(artifacts.map((artifact) => artifact.id));
+  const judgments = allArtifacts
+    .filter((artifact) => {
+      if (!isModelCompareJudgmentArtifact(artifact)) return false;
+      const sourceArtifactId = readString(artifact.metadata.sourceArtifactId);
+      if (sourceArtifactId && localComparisonIds.has(sourceArtifactId)) return true;
+      return readString(artifact.metadata.promptPreview).toLowerCase().includes('local model benchmark');
+    })
+    .slice(0, includeParameters ? 10 : 3);
+  const evidence = localBenchmarkEvidence(artifacts, judgments, true);
+  return {
+    status: artifacts.length > 0 ? 'history-found' : 'no-history',
+    count: artifacts.length,
+    artifacts: artifacts.map(describeLocalBenchmarkArtifact),
+    judgments: judgments.map(describeLocalBenchmarkJudgment),
+    evidence,
+    nextAction: artifacts.length > 0
+      ? evidence.status === 'reviewed-winner'
+        ? 'Use the revealed saved judgment as evidence only; apply/update still needs a separate confirmed user request.'
+        : `Review saved local benchmark ${artifacts[0]!.id}, then save a revealed judgment before recommending any default-model change.`
+      : 'Run the setupPlan benchmark prompt and save the comparison artifact before recommending any default-model change.',
+    analyticsRoute: 'agent_model_compare analytics benchmarkKind:"local-model-route" includeReasons:true',
+    saveRoute: 'agent_model_compare run benchmarkKind:"local-model-route" taskType:"local-model-route" confirm:true explicitUserRequest:"..."',
+    policy: 'Benchmark history is read-only evidence. Route changes still require a separate revealed judgment and confirmed apply/update action.',
+  };
+}
+
+export function localModelBenchmarkPlan(recipe: LocalModelRecipe): LocalModelBenchmarkPlan {
+  return {
+    status: 'plan-ready',
+    prompt: [
+      'Benchmark this local route on one practical task:',
+      '1. summarize the current project goal in five bullets,',
+      '2. identify one likely setup risk,',
+      '3. propose one next action with a command or route.',
+    ].join(' '),
+    measurements: [
+      'time to first useful token',
+      'total response latency',
+      'whether the answer followed the requested structure',
+      'whether the model handled project-specific nouns without hallucinating',
+      'whether the route supports the needed context window and tool workflow',
+    ],
+    workspaceActionRoute: 'agent_harness mode:"run_workspace_action" actionId:"account-run-local-model-benchmark" confirm:true fields.confirm:"yes" fields.modelRefs:"<local-route>,<baseline-route>" explicitUserRequest:"Compare this local model route before making it default."',
+    compareRoute: `agent_model_compare run prompt:"local model benchmark: ${recipe.label}" benchmarkKind:"local-model-route" taskType:"local-model-route" confirm:true explicitUserRequest:"Compare this local model route before making it default."`,
+    refreshRoute: 'agent_harness mode:"run_command" command:"/refresh-models" confirm:true explicitUserRequest:"Refresh model catalog, benchmarks, and token limits after local model setup."',
+    notes: [
+      'Use the workspace action when the user wants a form with benchmark defaults; use compareRoute when the model already has exact modelRefs.',
+      'Run the same prompt against the local route and a baseline route before selecting a winner.',
+      'Keep benchmark notes in the saved comparison artifact before changing the default model.',
+      'Do not treat cached public benchmark scores as a substitute for this local latency and fit check.',
+    ],
+  };
+}

package/src/tools/agent-harness-local-model-cookbook.ts

@@ -0,0 +1,238 @@
+import type { CommandContext } from '../input/command-registry.ts';
+import { previewHarnessText } from './agent-harness-text.ts';
+import { localModelDetection, localModelServerHealthMap } from './agent-harness-local-model-endpoints.ts';
+import { localHardwareProfile, localRecipeReadinessScore, localRecipeStackId, scoreLocalModelRecipe } from './agent-harness-model-readiness.ts';
+import { localModelBenchmarkHistory, localModelBenchmarkPlan } from './agent-harness-local-model-benchmarks.ts';
+import type { LocalModelDetection, LocalModelHardwareProfile, LocalModelRecipe, LocalModelRecipeFit, LocalModelSetupPlan, LocalModelBenchmarkEvidence } from './agent-harness-model-routing-types.ts';
+import { readRecord, readString } from './agent-harness-model-routing-utils.ts';
+
+export function localModelDownloadGuidance(recipe: LocalModelRecipe, hardware: LocalModelHardwareProfile): readonly string[] {
+  if (recipe.id === 'ollama') {
+    const model = hardware.ramGb >= 32 ? 'qwen2.5-coder:14b' : 'qwen2.5-coder:7b';
+    return [
+      'Install Ollama from the vendor package or package manager.',
+      'Start the Ollama service before refreshing Agent models.',
+      `Suggested first pull: ollama pull ${model}`,
+      `Smoke test: ollama run ${model} "Say ready in one sentence."`,
+    ];
+  }
+  if (recipe.id === 'llama-cpp') {
+    return [
+      'Choose a GGUF model that fits available RAM; prefer Q4/Q5 quantization on constrained systems.',
+      'Download the GGUF from the model publisher or a trusted mirror.',
+      'Start an OpenAI-compatible server: llama-server -m /path/to/model.gguf --host 127.0.0.1 --port 8080',
+      'Smoke test the /v1/models endpoint before adding the provider route.',
+    ];
+  }
+  if (recipe.id === 'vllm') {
+    return [
+      'Verify CUDA driver, GPU memory, and Python environment before installing vLLM.',
+      'Install vLLM in an isolated environment.',
+      'Serve a small first model: python -m vllm.entrypoints.openai.api_server --model Qwen/Qwen2.5-Coder-7B-Instruct --host 127.0.0.1 --port 8000',
+      'Smoke test /v1/models before adding the provider route.',
+    ];
+  }
+  return [
+    'Start the local OpenAI-compatible server in its own app or service.',
+    'Confirm the server exposes /v1/models and a private localhost or trusted LAN base URL.',
+    'Use the server app to download or load the model before adding it to Agent.',
+    'Keep LAN endpoints private unless the user explicitly intends shared access.',
+  ];
+}
+
+export function localModelProviderRoutes(recipe: LocalModelRecipe): readonly string[] {
+  if (recipe.id === 'ollama') {
+    return [
+      'agent_harness mode:"open_ui_surface" surfaceId:"provider-picker" confirm:true explicitUserRequest:"Select the discovered Ollama route."',
+      'agent_harness mode:"run_command" command:"/refresh-models" confirm:true explicitUserRequest:"Refresh models after starting Ollama."',
+    ];
+  }
+  if (recipe.id === 'llama-cpp') {
+    return [
+      'agent_harness mode:"run_command" command:"/provider add llama-cpp-local http://127.0.0.1:8080/v1 local --yes" confirm:true explicitUserRequest:"Add a local llama.cpp OpenAI-compatible provider."',
+      'agent_harness mode:"run_command" command:"/refresh-models" confirm:true explicitUserRequest:"Refresh models after starting llama.cpp."',
+    ];
+  }
+  if (recipe.id === 'vllm') {
+    return [
+      'agent_harness mode:"run_command" command:"/provider add vllm-local http://127.0.0.1:8000/v1 local --yes" confirm:true explicitUserRequest:"Add a local vLLM OpenAI-compatible provider."',
+      'agent_harness mode:"run_command" command:"/refresh-models" confirm:true explicitUserRequest:"Refresh models after starting vLLM."',
+    ];
+  }
+  return [
+    'agent_harness mode:"run_command" command:"/provider add local-openai http://127.0.0.1:1234/v1 local --yes" confirm:true explicitUserRequest:"Add a local OpenAI-compatible provider."',
+    'agent_harness mode:"run_command" command:"/refresh-models" confirm:true explicitUserRequest:"Refresh models after starting the local server."',
+  ];
+}
+
+export function localModelSetupPlan(
+  recipe: LocalModelRecipe,
+  hardware: LocalModelHardwareProfile,
+  detection: LocalModelDetection,
+  fit: LocalModelRecipeFit,
+): LocalModelSetupPlan {
+  const stackId = localRecipeStackId(recipe);
+  const detected = detection.stacks.includes(stackId);
+  return {
+    status: detected ? 'detected' : fit.level === 'weak' ? 'needs-hardware-review' : 'ready-to-try',
+    priority: fit.score,
+    downloadGuidance: localModelDownloadGuidance(recipe, hardware),
+    providerRoutes: localModelProviderRoutes(recipe),
+    benchmarkPlan: localModelBenchmarkPlan(recipe),
+    confirmationBoundary: 'The plan is read-only guidance. Installs, downloads, server starts, provider edits, refreshes, comparisons, and route changes require explicit user action or confirmation.',
+  };
+}
+
+export function localModelRecipes(): readonly LocalModelRecipe[] {
+  return [
+    {
+      id: 'ollama',
+      label: 'Ollama',
+      fit: 'Best first local route for most users.',
+      bestFor: 'Fast setup, chat, coding help, private everyday assistant work.',
+      hardware: 'Usable on modern CPU or Apple Silicon; 16 GB RAM is comfortable for 7B/8B quantized models, 32 GB+ for 14B/32B.',
+      setup: [
+        'Install Ollama and start the local service.',
+        'Pull one practical model, such as qwen2.5-coder:7b or llama3.1:8b.',
+        'Refresh models in GoodVibes Agent, then choose the discovered ollama:<model> route.',
+      ],
+      modelExamples: ['qwen2.5-coder:7b', 'llama3.1:8b', 'mistral:7b'],
+      cautions: ['Large models may run slowly without enough memory.', 'Use vLLM instead when the goal is multi-user throughput.'],
+    },
+    {
+      id: 'llama-cpp',
+      label: 'llama.cpp',
+      fit: 'Best low-dependency offline route.',
+      bestFor: 'CPU/Metal inference, GGUF files, portable offline use, constrained machines.',
+      hardware: 'Works without a GPU; use Q4/Q5 GGUF files for small systems and larger quantization only when memory allows.',
+      setup: [
+        'Download a GGUF model that fits memory.',
+        'Run llama-server with an OpenAI-compatible endpoint.',
+        'Add or refresh the local OpenAI-compatible provider, then select it in the model picker.',
+      ],
+      modelExamples: ['Qwen2.5-Coder 7B Instruct GGUF', 'Llama 3.1 8B Instruct GGUF', 'Phi-3 Mini GGUF'],
+      cautions: ['Manual model-file choice matters more than with Ollama.', 'Throughput is lower than GPU serving.'],
+    },
+    {
+      id: 'vllm',
+      label: 'vLLM',
+      fit: 'Best high-throughput local or LAN server route.',
+      bestFor: 'NVIDIA GPU serving, batching, OpenAI-compatible APIs, team/shared local models.',
+      hardware: 'Prefer CUDA GPUs; 16 GB VRAM can serve small quantized models, 24-48 GB+ is better for larger coder models.',
+      setup: [
+        'Install vLLM in a CUDA-ready Python environment.',
+        'Serve a model with the OpenAI-compatible API server.',
+        'Add the endpoint as a custom provider and select that route.',
+      ],
+      modelExamples: ['Qwen2.5-Coder 7B Instruct', 'Llama 3.1 8B Instruct', 'DeepSeek Coder V2 Lite'],
+      cautions: ['Not the easiest first setup.', 'GPU drivers and model memory limits are the common failure points.'],
+    },
+    {
+      id: 'openai-compatible-local',
+      label: 'Local OpenAI-compatible server',
+      fit: 'Best when the user already has LM Studio, LocalAI, TGI, or another local endpoint.',
+      bestFor: 'Reusing an existing localhost or LAN model server through one familiar API.',
+      hardware: 'Depends on the server backend; verify context window and memory in the serving app first.',
+      setup: [
+        'Start the local server and confirm its /v1/models endpoint works.',
+        'Add a custom provider with the local base URL.',
+        'Refresh models and select the discovered route.',
+      ],
+      modelExamples: ['LM Studio loaded model', 'LocalAI model', 'TGI-served model'],
+      cautions: ['Some servers omit context-window metadata.', 'Keep LAN endpoints private unless explicitly intended.'],
+    },
+  ];
+}
+
+export function describeLocalModelRecipe(
+  recipe: LocalModelRecipe,
+  detection: LocalModelDetection,
+  hardware: LocalModelHardwareProfile,
+  benchmarkEvidence: LocalModelBenchmarkEvidence,
+  includeParameters: boolean,
+): Record<string, unknown> {
+  const stackId = localRecipeStackId(recipe);
+  const detected = detection.stacks.includes(stackId);
+  const fit = scoreLocalModelRecipe(recipe, hardware, detection);
+  const readiness = localRecipeReadinessScore(recipe, fit, detected, benchmarkEvidence);
+  return {
+    id: recipe.id,
+    label: recipe.label,
+    fit: recipe.fit,
+    fitScore: fit.score,
+    fitLevel: fit.level,
+    readinessScore: readiness.score,
+    readinessLevel: readiness.level,
+    readiness: includeParameters
+      ? readiness
+      : {
+        score: readiness.score,
+        level: readiness.level,
+        confidence: readiness.confidence,
+        nextStep: readiness.nextStep,
+      },
+    bestFor: recipe.bestFor,
+    hardware: previewHarnessText(recipe.hardware, includeParameters ? 180 : 96),
+    hardwareMatched: fit.reasons.slice(0, includeParameters ? 6 : 3),
+    detected,
+      modelRoute: 'models action:"status" or agent_harness mode:"open_ui_surface"',
+    ...(includeParameters ? {
+      setup: recipe.setup,
+      modelExamples: recipe.modelExamples,
+      cautions: recipe.cautions,
+      setupPlan: localModelSetupPlan(recipe, hardware, detection, fit),
+    } : {}),
+  };
+}
+
+export function localModelCookbook(context: CommandContext, includeParameters: boolean): Record<string, unknown> {
+  const detection = localModelDetection(context);
+  const hardwareProfile = localHardwareProfile();
+  const benchmarkHistory = localModelBenchmarkHistory(context, includeParameters);
+  const benchmarkEvidence = readRecord(benchmarkHistory.evidence) as unknown as LocalModelBenchmarkEvidence;
+  const localServerHealth = localModelServerHealthMap(context, includeParameters);
+  const recipes = localModelRecipes()
+    .map((recipe) => describeLocalModelRecipe(recipe, detection, hardwareProfile, benchmarkEvidence, includeParameters))
+    .sort((left, right) => Number(readRecord(right).fitScore ?? 0) - Number(readRecord(left).fitScore ?? 0));
+  const topRecipe = readRecord(recipes[0]);
+  const topLabel = readString(topRecipe.label) || 'Ollama';
+  const nextActions = [
+    localServerHealth.endpointCount > 0
+      ? `Smoke test detected local endpoint(s): ${localServerHealth.endpoints[0]?.modelsUrl ?? 'see localServerHealth.endpoints'}.`
+      : detection.stacks.length > 0
+      ? `Inspect detected local route(s): ${detection.modelRoutes.join(', ') || detection.providerIds.join(', ')}.`
+      : `Start with ${topLabel}: inspect its setupPlan, then install/start the server outside Agent.`,
+    'Refresh the model catalog after the local server is running.',
+    'Run the local benchmark workspace action or saved model comparison before changing the default route.',
+  ];
+  return {
+    status: detection.stacks.length > 0
+      ? 'detected-local-route'
+      : localServerHealth.endpointCount > 0
+        ? 'detected-local-server'
+        : 'recommendations-only',
+    recommendation: detection.stacks.includes('ollama')
+      ? 'Use the discovered Ollama route first unless throughput requirements point to vLLM.'
+      : `Best current fit: ${topLabel}. Ollama remains the easiest first local route; use llama.cpp for offline GGUF files or vLLM for GPU throughput.`,
+    hardwareProfile,
+    detected: detection,
+    localServerHealth,
+    recipes,
+    benchmarkHistory,
+    readinessRubric: {
+      score: '0-100 estimated readiness for autonomous Agent work.',
+      confidence: 'estimated until a live route benchmark records latency and task fit on this machine',
+      dimensions: [
+        { id: 'latency', weight: 20 },
+        { id: 'context-window', weight: 20 },
+        { id: 'tool-support', weight: 20 },
+        { id: 'vision', weight: 10 },
+        { id: 'cost', weight: 15 },
+        { id: 'privacy', weight: 15 },
+      ],
+    },
+    nextActions,
+    modelRoute: 'models action:"local"',
+    policy: 'Read-only hardware-aware cookbook. Readiness scores are estimated until a live benchmark is recorded. Setup plans include download/start guidance and a confirmed benchmark action route, but installs, downloads, live benchmarks, provider edits, and route changes stay separate visible user actions.',
+  };
+}