ethagent 2.2.0 → 2.4.0

package/src/models/llamacpp.ts

@@ -50,6 +50,7 @@ export type LlamaCppStartFailureCode =
   | 'spawn-failed'
   | 'runner-exited'
   | 'readiness-timeout'
+  | 'untracked-server'
 
 export type LlamaCppStartResult =
   | { ok: true; alreadyRunning: boolean }
@@ -362,25 +363,45 @@ export async function startLlamaCppServer(args: {
   modelAlias: string
   host?: string
   ctxSize?: number
+  mmprojPath?: string
   readinessTimeoutMs?: number
   pollMs?: number
   deps?: LlamaCppStartDeps
 }): Promise<LlamaCppStartResult> {
   const host = args.host ?? DEFAULT_LLAMA_HOST
   const initialStatus = await servedModelStatus(host, args.modelAlias)
-  if (initialStatus.state === 'ready') return { ok: true, alreadyRunning: true }
+  if (initialStatus.state === 'ready') {
+    if (args.mmprojPath) {
+      const pid = await readPidFile()
+      if (!pid) {
+        return startFailure('untracked-server', {
+          detail: 'A llama-server is already serving this alias but ethagent did not launch it, so we cannot apply the vision projector. Stop the external process and reopen ethagent.',
+        })
+      }
+    }
+    return { ok: true, alreadyRunning: true }
+  }
   if (initialStatus.state === 'different') {
     return startFailure('different-model-running', {
       servedModels: initialStatus.models,
     })
   }
 
+  const accessFn = args.deps?.access ?? fs.access
   try {
-    await (args.deps?.access ?? fs.access)(args.modelPath)
+    await accessFn(args.modelPath)
   } catch {
     return startFailure('model-file-missing', { detail: args.modelPath })
   }
 
+  if (args.mmprojPath) {
+    try {
+      await accessFn(args.mmprojPath)
+    } catch {
+      return startFailure('model-file-missing', { detail: args.mmprojPath })
+    }
+  }
+
   const binaryPath = args.deps?.binaryPath ?? (await findAndPersistLlamaCppServer()).path
   if (!binaryPath) {
     return startFailure('runner-not-installed')
@@ -390,21 +411,23 @@ export async function startLlamaCppServer(args: {
   const listenHost = url.hostname || '127.0.0.1'
   const port = url.port || (url.protocol === 'https:' ? '443' : '8080')
   const spawnImpl = args.deps?.spawnImpl ?? spawn
+  const spawnArgs: string[] = [
+    '-m',
+    args.modelPath,
+    '--host',
+    listenHost,
+    '--port',
+    port,
+    '--alias',
+    args.modelAlias,
+    '--ctx-size',
+    String(args.ctxSize ?? 32768),
+    '--jinja',
+  ]
+  if (args.mmprojPath) spawnArgs.push('--mmproj', args.mmprojPath)
   let child: ReturnType<typeof spawn>
   try {
-    child = spawnImpl(binaryPath, [
-      '-m',
-      args.modelPath,
-      '--host',
-      listenHost,
-      '--port',
-      port,
-      '--alias',
-      args.modelAlias,
-      '--ctx-size',
-      String(args.ctxSize ?? 32768),
-      '--jinja',
-    ], {
+    child = spawnImpl(binaryPath, spawnArgs, {
       detached: true,
       stdio: ['ignore', 'pipe', 'pipe'],
       windowsHide: true,
@@ -424,6 +447,9 @@ export async function startLlamaCppServer(args: {
     })
   })
   child.unref()
+  if (typeof child.pid === 'number') {
+    await writePidFile(child.pid).catch(() => {})
+  }
 
   const ready = await waitForServedModel({
     host,
@@ -468,6 +494,73 @@ async function waitForServedModel(args: {
   return startFailure('readiness-timeout')
 }
 
+function pidFilePath(): string {
+  return path.join(getConfigDir(), 'llamacpp.pid')
+}
+
+async function writePidFile(pid: number): Promise<void> {
+  await ensureConfigDir()
+  await atomicWriteText(pidFilePath(), String(pid))
+}
+
+async function readPidFile(): Promise<number | null> {
+  try {
+    const raw = await fs.readFile(pidFilePath(), 'utf8')
+    const pid = Number.parseInt(raw.trim(), 10)
+    return Number.isInteger(pid) && pid > 0 ? pid : null
+  } catch {
+    return null
+  }
+}
+
+async function clearPidFile(): Promise<void> {
+  await fs.rm(pidFilePath(), { force: true }).catch(() => {})
+}
+
+export async function stopLlamaCppServer(args: {
+  host?: string
+  timeoutMs?: number
+  pollMs?: number
+  killImpl?: (pid: number, signal?: NodeJS.Signals | number) => void
+} = {}): Promise<
+  | { ok: true; stopped: boolean; reason?: 'untracked-server'; servedModels?: string[] }
+  | { ok: false; message: string }
+> {
+  const pid = await readPidFile()
+  if (!pid) {
+    const host = args.host ?? DEFAULT_LLAMA_HOST
+    const { up, models } = await fetchServedModels(host, 1500)
+    if (up && models.length > 0) {
+      return { ok: true, stopped: false, reason: 'untracked-server', servedModels: models }
+    }
+    return { ok: true, stopped: false }
+  }
+  const kill = args.killImpl ?? ((p, signal) => process.kill(p, signal))
+  try {
+    kill(pid, 'SIGTERM')
+  } catch (err: unknown) {
+    const code = (err as NodeJS.ErrnoException).code
+    if (code === 'ESRCH') {
+      await clearPidFile()
+      return { ok: true, stopped: false }
+    }
+    return { ok: false, message: (err as Error).message }
+  }
+  const host = args.host ?? DEFAULT_LLAMA_HOST
+  const deadline = Date.now() + (args.timeoutMs ?? 5000)
+  const pollMs = args.pollMs ?? 250
+  while (Date.now() < deadline) {
+    const status = await servedModelStatus(host, '__nothing__')
+    if (status.state === 'not-up' || status.models.length === 0) {
+      await clearPidFile()
+      return { ok: true, stopped: true }
+    }
+    await new Promise<void>(resolve => setTimeout(resolve, pollMs))
+  }
+  await clearPidFile()
+  return { ok: true, stopped: true }
+}
+
 async function servedModelStatus(host: string, modelAlias: string): Promise<
   | { state: 'not-up'; models: string[] }
   | { state: 'ready'; models: string[] }
@@ -507,6 +600,8 @@ function startFailureMessage(code: LlamaCppStartFailureCode, servedModels: strin
       return 'local runner closed before becoming ready'
     case 'readiness-timeout':
       return 'local runner is still loading or did not answer in time'
+    case 'untracked-server':
+      return detail ?? 'a llama-server is already running and ethagent did not launch it; cannot apply the vision encoder until that process is stopped'
   }
 }
 

package/src/models/llamacppPreflight.ts

@@ -1,5 +1,6 @@
 import {
   startLlamaCppServer,
+  stopLlamaCppServer,
   type LlamaCppStartFailureCode,
   type LlamaCppStartResult,
 } from './llamacpp.js'
@@ -21,9 +22,13 @@ export type LlamaCppPreflightDeps = {
   fetchImpl?: typeof fetch
   findLocalModel?: typeof findLocalHfModel
   startServer?: typeof startLlamaCppServer
+  stopServer?: typeof stopLlamaCppServer
   timeoutMs?: number
 }
 
+const UNTRACKED_VISION_DETAIL =
+  'A llama-server is already serving this alias but ethagent did not launch it, so we cannot apply the vision projector. Stop the external process and reopen ethagent.'
+
 type ModelsProbe =
   | { up: true; models: string[] }
   | { up: false; models: [] }
@@ -50,18 +55,31 @@ export async function ensureLlamaCppRunnerReady(
 
   const probe = await probeLlamaCppModels(baseUrl, deps)
   if (probe.up) {
-    if (probe.models.length === 0 || probe.models.includes(config.model)) {
-      return { ok: true, alreadyRunning: true }
+    if (probe.models.length > 0 && !probe.models.includes(config.model)) {
+      return {
+        ok: false,
+        code: 'different-model-running',
+        message: formatPreflightFailure(
+          'local runner is serving a different model',
+          config.model,
+          `a different local model is already running (${probe.models.join(', ')}); stop it before switching models`,
+        ),
+        servedModels: probe.models,
+      }
     }
-    return {
-      ok: false,
-      code: 'different-model-running',
-      message: formatPreflightFailure(
-        'local runner is serving a different model',
-        config.model,
-        `a different local model is already running (${probe.models.join(', ')}); stop it before switching models`,
-      ),
-      servedModels: probe.models,
+    if (!local.mmprojPath) return { ok: true, alreadyRunning: true }
+    const stopped = await (deps.stopServer ?? stopLlamaCppServer)().catch(() => null)
+    if (stopped && stopped.ok && stopped.reason === 'untracked-server') {
+      return withPreflightMessage(
+        {
+          ok: false,
+          code: 'untracked-server',
+          message: UNTRACKED_VISION_DETAIL,
+          detail: UNTRACKED_VISION_DETAIL,
+          servedModels: stopped.servedModels,
+        },
+        local,
+      )
     }
   }
 
@@ -69,6 +87,7 @@ export async function ensureLlamaCppRunnerReady(
     modelPath: local.localPath,
     modelAlias: local.id,
     host: llamaCppServerHostFromBaseUrl(baseUrl),
+    mmprojPath: local.mmprojPath,
   })
   if (result.ok) return { ok: true, alreadyRunning: result.alreadyRunning }
   return withPreflightMessage(result, local)

package/src/models/modelPickerOptions.ts

@@ -7,26 +7,14 @@ import { type SelectOption } from '../ui/Select.js'
 import { formatLocalHfModelDisplayName, formatModelDisplayName } from './modelDisplay.js'
 import { localModelId, quantizationFromFilename } from './huggingface.js'
 import type { UncensoredCatalogEntry } from './uncensoredCatalog.js'
+import { cloudProviderDisplayName, providerDisplayName, type CloudProviderId } from './providerDisplay.js'
 
-export type CloudProviderId = Exclude<ProviderId, 'llamacpp'>
+export { cloudProviderDisplayName, providerDisplayName, type CloudProviderId }
 
 export const MODEL_PICKER_CLOUD_PROVIDERS: CloudProviderId[] = ['openai', 'anthropic', 'gemini']
 export const LOCAL_MODEL_LINK_HINT = 'Paste a GGUF link'
 export const LOCAL_MODEL_LINK_EXAMPLE = 'e.g. https://huggingface.co/Qwen/Qwen3-8B-GGUF'
 
-export function cloudProviderDisplayName(provider: CloudProviderId): string {
-  switch (provider) {
-    case 'openai': return 'OpenAI'
-    case 'anthropic': return 'Anthropic'
-    case 'gemini': return 'Gemini'
-  }
-}
-
-export function providerDisplayName(provider: ProviderId): string {
-  if (provider === 'llamacpp') return 'llama.cpp'
-  return cloudProviderDisplayName(provider)
-}
-
 export type LocalHfPickerModel = {
   id: string
   displayName: string
@@ -35,6 +23,9 @@ export type LocalHfPickerModel = {
   risk: HfRisk
   task: HfTask
   status: 'ready' | 'incomplete'
+  mmprojPath?: string
+  mmprojAvailable?: boolean
+  mmprojSizeBytes?: number
 }
 
 export type CloudCredentialKind = 'apikey' | 'oauth'
@@ -197,12 +188,22 @@ function appendHfModelOptions(
       displayName: model.displayName,
       maxLength,
     })
+    const tags = ['Installed']
+    if (model.mmprojPath) tags.push('Vision encoder loaded')
     options.push(rowOption(
       `hf:${id}`,
       contextFitLabel('llamacpp', id, `${active ? '* ' : '  '}${displayName}`, context.contextFit),
       undefined,
-      modelMetadataSubtext(size, ['Installed']),
+      modelMetadataSubtext(size, tags),
     ))
+    if (model.mmprojAvailable && !model.mmprojPath) {
+      const projectorSize = model.mmprojSizeBytes ? ` (+${formatSize(model.mmprojSizeBytes)})` : ''
+      options.push(rowOption(
+        `hfmmproj:${id}`,
+        `    + Add Vision Encoder${projectorSize}`,
+        'Enable image input on this local model',
+      ))
+    }
   }
 }
 

package/src/models/providerDisplay.ts

@@ -0,0 +1,16 @@
+import type { ProviderId } from '../storage/config.js'
+
+export type CloudProviderId = Exclude<ProviderId, 'llamacpp'>
+
+export function cloudProviderDisplayName(provider: CloudProviderId): string {
+  switch (provider) {
+    case 'openai': return 'OpenAI'
+    case 'anthropic': return 'Anthropic'
+    case 'gemini': return 'Gemini'
+  }
+}
+
+export function providerDisplayName(provider: ProviderId): string {
+  if (provider === 'llamacpp') return 'llama.cpp'
+  return cloudProviderDisplayName(provider)
+}

package/src/providers/anthropic.ts

@@ -4,6 +4,7 @@ import { ProviderError } from './contracts.js'
 import { providerErrorFromResponse } from './errors.js'
 import { fetchWithRetryStreamEvents } from './retry.js'
 import { iterSseEvents } from './sse.js'
+import { hasImageBlocks, ImageLoadError, loadImageBlock } from '../utils/images.js'
 
 export type AnthropicToolDefinition = {
   name: string
@@ -75,7 +76,22 @@ export class AnthropicProvider implements Provider {
       return
     }
 
-    const { system, conversation } = splitMessages(messages)
+    if (hasImageBlocks(messages) && !supportsAnthropicImages(this.model)) {
+      yield { type: 'error', message: `image input is not enabled for ${this.model}` }
+      return
+    }
+
+    let split: { system?: string; conversation: Awaited<ReturnType<typeof splitMessages>>['conversation'] }
+    try {
+      split = await splitMessages(messages)
+    } catch (err: unknown) {
+      if (err instanceof ImageLoadError) {
+        yield { type: 'error', message: err.message }
+        return
+      }
+      throw err
+    }
+    const { system, conversation } = split
 
     let response: Response
     try {
@@ -195,22 +211,24 @@ export class AnthropicProvider implements Provider {
   }
 }
 
-function splitMessages(messages: Message[]): {
+async function splitMessages(messages: Message[]): Promise<{
   system?: string
   conversation: Array<{
     role: 'user' | 'assistant'
     content: Array<
       | { type: 'text'; text: string }
+      | { type: 'image'; source: { type: 'base64'; media_type: string; data: string } }
       | { type: 'tool_use'; id: string; name: string; input: Record<string, unknown> }
       | { type: 'tool_result'; tool_use_id: string; content: string; is_error?: boolean }
     >
   }>
-} {
+}> {
   const systemParts: string[] = []
   const conversation: Array<{
     role: 'user' | 'assistant'
     content: Array<
       | { type: 'text'; text: string }
+      | { type: 'image'; source: { type: 'base64'; media_type: string; data: string } }
       | { type: 'tool_use'; id: string; name: string; input: Record<string, unknown> }
       | { type: 'tool_result'; tool_use_id: string; content: string; is_error?: boolean }
     >
@@ -226,11 +244,16 @@ function splitMessages(messages: Message[]): {
     }
     conversation.push({
       role: message.role,
-      content: blocks.map(block => {
+      content: await Promise.all(blocks.map(async block => {
         if (block.type === 'text') return { type: 'text', text: block.text }
+        if (block.type === 'image') {
+          const loaded = await loadImageBlock(block)
+          if (!loaded.dataBase64 || !loaded.mimeType) throw new Error(`could not load image: ${block.path}`)
+          return { type: 'image', source: { type: 'base64', media_type: loaded.mimeType, data: loaded.dataBase64 } }
+        }
         if (block.type === 'tool_use') return { type: 'tool_use', id: block.id, name: block.name, input: block.input }
         return { type: 'tool_result', tool_use_id: block.toolUseId, content: block.content, is_error: block.isError }
-      }),
+      })),
     })
   }
 
@@ -251,6 +274,14 @@ function normalizeBlocks(content: Message['content']): MessageContentBlock[] {
   })
 }
 
+export function supportsAnthropicImages(model: string): boolean {
+  const normalized = model.toLowerCase()
+  return normalized.includes('claude-3')
+    || normalized.includes('claude-sonnet-4')
+    || normalized.includes('claude-opus-4')
+    || normalized.includes('claude-haiku-4')
+}
+
 function normalizeStopReason(value?: string): 'end_turn' | 'tool_use' | 'max_tokens' | 'stop_sequence' | 'unknown' {
   if (value === 'end_turn' || value === 'tool_use' || value === 'max_tokens' || value === 'stop_sequence') {
     return value

package/src/providers/contracts.ts

@@ -8,6 +8,14 @@ export type TextBlock = {
   text: string
 }
 
+export type ImageBlock = {
+  type: 'image'
+  path: string
+  mimeType?: string
+  url?: string
+  dataBase64?: string
+}
+
 export type ToolUseBlock = {
   type: 'tool_use'
   id: string
@@ -22,7 +30,7 @@ export type ToolResultBlock = {
   isError?: boolean
 }
 
-export type MessageContentBlock = TextBlock | ToolUseBlock | ToolResultBlock
+export type MessageContentBlock = TextBlock | ImageBlock | ToolUseBlock | ToolResultBlock
 
 export type Message = {
   role: Role

package/src/providers/errors.ts

@@ -1,6 +1,7 @@
 import type { ProviderId } from '../storage/config.js'
 import { ProviderError } from './contracts.js'
 import { formatGeminiRateLimitMessage } from './gemini.js'
+import { providerDisplayName } from '../models/providerDisplay.js'
 
 type ErrorBody =
   | string
@@ -30,19 +31,20 @@ export async function providerErrorFromResponse(
     && /API[_ ]?key( not valid| not found|_invalid)|invalid api key/i.test(detail)
   ) {
     return new ProviderError(
-      'gemini: API key rejected — verify your key at https://aistudio.google.com/app/apikey, then run /key gemini to set it again',
+      'Gemini: API key rejected — verify your key at https://aistudio.google.com/app/apikey, then run /key gemini to set it again',
     )
   }
 
   if (provider !== 'llamacpp') {
+    const name = providerDisplayName(provider)
     if (response.status === 401 || response.status === 403) {
-      return new ProviderError(`auth failed: check your ${provider} key (/doctor to verify)`)
+      return new ProviderError(`auth failed: check your ${name} key (/doctor to verify)`)
     }
     if (response.status === 429) {
-      return new ProviderError(detail || `${provider} rate limit exceeded`, { transient: true })
+      return new ProviderError(detail || `${name} rate limit exceeded`, { transient: true })
     }
     if (response.status >= 500) {
-      return new ProviderError(detail || `${provider} server error (${response.status})`, { transient: true })
+      return new ProviderError(detail || `${name} server error (${response.status})`, { transient: true })
     }
   }
 

package/src/providers/gemini.ts

@@ -4,6 +4,7 @@ import { ProviderError } from './contracts.js'
 import { providerErrorFromResponse } from './errors.js'
 import { fetchWithRetryStreamEvents } from './retry.js'
 import { iterSseFrames } from './sse.js'
+import { hasImageBlocks, ImageLoadError, loadImageBlock } from '../utils/images.js'
 
 export type GeminiToolDefinition = {
   name: string
@@ -41,6 +42,7 @@ type GeminiChunk = {
 
 type GeminiContentPart =
   | { text: string }
+  | { inlineData: { mimeType: string; data: string } }
   | { functionCall: { name: string; args: Record<string, unknown> } }
   | { functionResponse: { name: string; response: Record<string, unknown> } }
 
@@ -92,8 +94,21 @@ export class GeminiProvider implements Provider {
       yield { type: 'error', message: error.message }
       return
     }
+    if (hasImageBlocks(messages) && !supportsGeminiImages(this.model)) {
+      yield { type: 'error', message: `image input is not enabled for ${this.model}` }
+      return
+    }
 
-    const payload = buildGeminiPayload(messages, this.tools, options)
+    let payload: GeminiPayload
+    try {
+      payload = await buildGeminiPayload(messages, this.tools, options)
+    } catch (err: unknown) {
+      if (err instanceof ImageLoadError) {
+        yield { type: 'error', message: err.message }
+        return
+      }
+      throw err
+    }
     const modelName = this.model.replace(/^models\//, '')
     const url = `https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(modelName)}:streamGenerateContent?alt=sse`
 
@@ -181,11 +196,11 @@ export class GeminiProvider implements Provider {
   }
 }
 
-export function buildGeminiPayload(
+export async function buildGeminiPayload(
   messages: Message[],
   tools: GeminiToolDefinition[] = [],
   options: ProviderCompleteOptions = {},
-): GeminiPayload {
+): Promise<GeminiPayload> {
   const systemParts: string[] = []
   const contents: GeminiContent[] = []
   const toolUseNamesById = new Map<string, string>()
@@ -222,6 +237,10 @@ export function buildGeminiPayload(
     for (const block of blocks) {
       if (block.type === 'text') {
         parts.push({ text: block.text })
+      } else if (block.type === 'image') {
+        const loaded = await loadImageBlock(block)
+        if (!loaded.dataBase64 || !loaded.mimeType) throw new Error(`could not load image: ${block.path}`)
+        parts.push({ inlineData: { mimeType: loaded.mimeType, data: loaded.dataBase64 } })
       } else if (block.type === 'tool_result') {
         const name = toolUseNamesById.get(block.toolUseId) ?? 'unknown'
         const response: Record<string, unknown> = block.isError
@@ -258,6 +277,13 @@ function normalizeBlocks(content: Message['content']): MessageContentBlock[] {
   })
 }
 
+export function supportsGeminiImages(model: string): boolean {
+  const normalized = model.toLowerCase()
+  return normalized.includes('gemini-1.5')
+    || normalized.includes('gemini-2.0')
+    || normalized.includes('gemini-2.5')
+}
+
 function normalizeFinishReason(reason: string, sawToolCall: boolean): DoneStopReason {
   if (sawToolCall) return 'tool_use'
   switch (reason) {