ethagent 2.3.0 → 2.4.0

package/src/models/huggingface.ts

@@ -54,6 +54,12 @@ export type HfSafetyReview = {
   reasons: string[]
 }
 
+export type HfMmprojCandidate = {
+  filename: string
+  sizeBytes: number
+  localPath: string
+}
+
 export type HfDownloadPlan = {
   repo: HuggingFaceRepoInfo
   repoId: string
@@ -64,6 +70,8 @@ export type HfDownloadPlan = {
   localPath: string
   displayName: string
   review: HfSafetyReview
+  mmprojCandidate?: HfMmprojCandidate
+  includeMmproj?: boolean
 }
 
 export type LocalHfModel = {
@@ -90,6 +98,9 @@ export type LocalHfModel = {
   installedAt: string
   status: LocalHfStatus
   sha256?: string
+  mmprojPath?: string
+  mmprojAvailable?: boolean
+  mmprojSizeBytes?: number
 }
 
 export type HfDownloadProgress = {
@@ -291,6 +302,14 @@ export function ggufFiles(repo: HuggingFaceRepoInfo): HuggingFaceSibling[] {
     .sort((a, b) => a.filename.localeCompare(b.filename))
 }
 
+export function isMmprojFilename(filename: string): boolean {
+  return filename.toLowerCase().startsWith('mmproj-') && filename.toLowerCase().endsWith('.gguf')
+}
+
+export function findMmprojSibling(repo: HuggingFaceRepoInfo): HuggingFaceSibling | undefined {
+  return repo.siblings.find(file => isMmprojFilename(file.filename))
+}
+
 export async function createHfDownloadPlan(
   input: string,
   filename?: string,
@@ -320,6 +339,14 @@ export async function createHfDownloadPlan(
     requestedRevision,
     resolvedRevision,
   })
+  const mmprojSibling = findMmprojSibling(repo)
+  const mmprojCandidate: HfMmprojCandidate | undefined = mmprojSibling
+    ? {
+      filename: mmprojSibling.filename,
+      sizeBytes: mmprojSibling.sizeBytes ?? 0,
+      localPath: localPathFor(repo.repoId, resolvedRevision, mmprojSibling.filename),
+    }
+    : undefined
   return {
     repo,
     repoId: repo.repoId,
@@ -330,6 +357,7 @@ export async function createHfDownloadPlan(
     localPath: localPathFor(repo.repoId, resolvedRevision, selected.filename),
     displayName: displayNameFor(repo.repoId, selected.filename),
     review,
+    mmprojCandidate,
   }
 }
 
@@ -432,10 +460,151 @@ export async function* downloadHfModel(
   }
 
   await fs.rename(partialPath, plan.localPath)
-  await upsertLocalHfModel(modelFromPlan(plan, hash.digest('hex'), 'ready'))
+
+  let mmprojPath: string | undefined
+  if (plan.includeMmproj && plan.mmprojCandidate) {
+    yield* downloadMmprojFile(plan.repoId, plan.resolvedRevision, plan.mmprojCandidate, signal, fetchImpl)
+    mmprojPath = plan.mmprojCandidate.localPath
+  }
+
+  await upsertLocalHfModel(modelFromPlan(plan, hash.digest('hex'), 'ready', mmprojPath))
   yield { status: 'success', completed, total: Number.isFinite(total) ? total : completed }
 }
 
+async function* downloadMmprojFile(
+  repoId: string,
+  resolvedRevision: string,
+  candidate: HfMmprojCandidate,
+  signal: AbortSignal | undefined,
+  fetchImpl: FetchImpl,
+): AsyncIterable<HfDownloadProgress> {
+  await fs.mkdir(path.dirname(candidate.localPath), { recursive: true })
+  const partialPath = `${candidate.localPath}.partial`
+  const response = await fetchImpl(resolveUrl(repoId, resolvedRevision, candidate.filename), { signal })
+  if (!response.ok || !response.body) {
+    throw new Error(response.ok ? 'empty projector download body' : `projector download HTTP ${response.status}`)
+  }
+
+  const total = Number.parseInt(response.headers.get('content-length') ?? '', 10)
+  const handle = await fs.open(partialPath, 'w')
+  let completed = 0
+  let complete = false
+  let lastProgressAt = Date.now()
+  let lastProgressBytes = 0
+  yield { status: 'downloading-mmproj', completed, total: Number.isFinite(total) ? total : undefined }
+  try {
+    const reader = response.body.getReader()
+    while (true) {
+      const { done, value } = await reader.read()
+      if (done) break
+      if (signal?.aborted) throw new Error('Cancelled')
+      const buffer = Buffer.from(value)
+      await handle.write(buffer)
+      completed += buffer.byteLength
+      const now = Date.now()
+      if (shouldReportDownloadProgress(completed, lastProgressBytes, now, lastProgressAt)) {
+        lastProgressAt = now
+        lastProgressBytes = completed
+        yield { status: 'downloading-mmproj', completed, total: Number.isFinite(total) ? total : undefined }
+      }
+    }
+    complete = true
+  } finally {
+    await handle.close()
+    if (!complete) {
+      await fs.unlink(partialPath).catch(() => {})
+    }
+  }
+
+  await fs.rename(partialPath, candidate.localPath)
+}
+
+export async function backfillMmprojAvailability(
+  model: LocalHfModel,
+  fetchImpl: FetchImpl = fetch,
+): Promise<LocalHfModel> {
+  if (model.mmprojAvailable !== undefined) return model
+  try {
+    const repo = await fetchHuggingFaceRepoInfo({ repoId: model.repoId }, fetchImpl)
+    const sibling = findMmprojSibling(repo)
+    const next: LocalHfModel = {
+      ...model,
+      mmprojAvailable: Boolean(sibling),
+      mmprojSizeBytes: sibling?.sizeBytes,
+    }
+    await upsertLocalHfModel(next)
+    return next
+  } catch {
+    return model
+  }
+}
+
+export async function backfillMmprojForModels(
+  models: LocalHfModel[],
+  fetchImpl: FetchImpl = fetch,
+): Promise<LocalHfModel[]> {
+  const repoIdToProbe = new Map<string, Promise<HuggingFaceRepoInfo | null>>()
+  for (const model of models) {
+    if (model.mmprojAvailable !== undefined) continue
+    if (repoIdToProbe.has(model.repoId)) continue
+    repoIdToProbe.set(
+      model.repoId,
+      fetchHuggingFaceRepoInfo({ repoId: model.repoId }, fetchImpl).catch(() => null),
+    )
+  }
+  if (repoIdToProbe.size === 0) return models
+  const resolved = new Map<string, HuggingFaceRepoInfo | null>()
+  for (const [repoId, promise] of repoIdToProbe) {
+    resolved.set(repoId, await promise)
+  }
+  const out: LocalHfModel[] = []
+  for (const model of models) {
+    if (model.mmprojAvailable !== undefined) {
+      out.push(model)
+      continue
+    }
+    const repo = resolved.get(model.repoId)
+    if (!repo) {
+      out.push(model)
+      continue
+    }
+    const sibling = findMmprojSibling(repo)
+    const next: LocalHfModel = {
+      ...model,
+      mmprojAvailable: Boolean(sibling),
+      mmprojSizeBytes: sibling?.sizeBytes,
+    }
+    await upsertLocalHfModel(next)
+    out.push(next)
+  }
+  return out
+}
+
+export async function* addMmprojToInstalledModel(
+  modelId: string,
+  signal?: AbortSignal,
+  deps: { fetchImpl?: FetchImpl } = {},
+): AsyncIterable<HfDownloadProgress> {
+  const fetchImpl = deps.fetchImpl ?? fetch
+  const existing = await findLocalHfModel(modelId)
+  if (!existing) throw new Error(`model not installed: ${modelId}`)
+  if (existing.mmprojPath) {
+    yield { status: 'success', completed: 0 }
+    return
+  }
+  const repo = await fetchHuggingFaceRepoInfo({ repoId: existing.repoId }, fetchImpl)
+  const sibling = findMmprojSibling(repo)
+  if (!sibling) throw new Error(`no vision encoder available for ${existing.repoId}`)
+  const candidate: HfMmprojCandidate = {
+    filename: sibling.filename,
+    sizeBytes: sibling.sizeBytes ?? 0,
+    localPath: localPathFor(existing.repoId, existing.resolvedRevision, sibling.filename),
+  }
+  yield* downloadMmprojFile(existing.repoId, existing.resolvedRevision, candidate, signal, fetchImpl)
+  await upsertLocalHfModel({ ...existing, mmprojPath: candidate.localPath })
+  yield { status: 'success', completed: candidate.sizeBytes }
+}
+
 export function shouldReportDownloadProgress(
   completed: number,
   lastCompleted: number,
@@ -446,7 +615,13 @@ export function shouldReportDownloadProgress(
     || completed - lastCompleted >= DOWNLOAD_PROGRESS_MIN_BYTES
 }
 
-export function modelFromPlan(plan: HfDownloadPlan, sha256: string | undefined, status: LocalHfStatus): LocalHfModel {
+export function modelFromPlan(
+  plan: HfDownloadPlan,
+  sha256: string | undefined,
+  status: LocalHfStatus,
+  mmprojPath?: string,
+): LocalHfModel {
+  const mmprojAvailable = Boolean(plan.mmprojCandidate)
   const now = new Date().toISOString()
   return {
     id: localModelId(plan.repoId, plan.filename),
@@ -472,6 +647,9 @@ export function modelFromPlan(plan: HfDownloadPlan, sha256: string | undefined,
     installedAt: now,
     status,
     sha256,
+    mmprojPath,
+    mmprojAvailable,
+    mmprojSizeBytes: plan.mmprojCandidate?.sizeBytes,
   }
 }
 

package/src/models/llamacpp.ts

@@ -50,6 +50,7 @@ export type LlamaCppStartFailureCode =
   | 'spawn-failed'
   | 'runner-exited'
   | 'readiness-timeout'
+  | 'untracked-server'
 
 export type LlamaCppStartResult =
   | { ok: true; alreadyRunning: boolean }
@@ -362,25 +363,45 @@ export async function startLlamaCppServer(args: {
   modelAlias: string
   host?: string
   ctxSize?: number
+  mmprojPath?: string
   readinessTimeoutMs?: number
   pollMs?: number
   deps?: LlamaCppStartDeps
 }): Promise<LlamaCppStartResult> {
   const host = args.host ?? DEFAULT_LLAMA_HOST
   const initialStatus = await servedModelStatus(host, args.modelAlias)
-  if (initialStatus.state === 'ready') return { ok: true, alreadyRunning: true }
+  if (initialStatus.state === 'ready') {
+    if (args.mmprojPath) {
+      const pid = await readPidFile()
+      if (!pid) {
+        return startFailure('untracked-server', {
+          detail: 'A llama-server is already serving this alias but ethagent did not launch it, so we cannot apply the vision projector. Stop the external process and reopen ethagent.',
+        })
+      }
+    }
+    return { ok: true, alreadyRunning: true }
+  }
   if (initialStatus.state === 'different') {
     return startFailure('different-model-running', {
       servedModels: initialStatus.models,
     })
   }
 
+  const accessFn = args.deps?.access ?? fs.access
   try {
-    await (args.deps?.access ?? fs.access)(args.modelPath)
+    await accessFn(args.modelPath)
   } catch {
     return startFailure('model-file-missing', { detail: args.modelPath })
   }
 
+  if (args.mmprojPath) {
+    try {
+      await accessFn(args.mmprojPath)
+    } catch {
+      return startFailure('model-file-missing', { detail: args.mmprojPath })
+    }
+  }
+
   const binaryPath = args.deps?.binaryPath ?? (await findAndPersistLlamaCppServer()).path
   if (!binaryPath) {
     return startFailure('runner-not-installed')
@@ -390,21 +411,23 @@ export async function startLlamaCppServer(args: {
   const listenHost = url.hostname || '127.0.0.1'
   const port = url.port || (url.protocol === 'https:' ? '443' : '8080')
   const spawnImpl = args.deps?.spawnImpl ?? spawn
+  const spawnArgs: string[] = [
+    '-m',
+    args.modelPath,
+    '--host',
+    listenHost,
+    '--port',
+    port,
+    '--alias',
+    args.modelAlias,
+    '--ctx-size',
+    String(args.ctxSize ?? 32768),
+    '--jinja',
+  ]
+  if (args.mmprojPath) spawnArgs.push('--mmproj', args.mmprojPath)
   let child: ReturnType<typeof spawn>
   try {
-    child = spawnImpl(binaryPath, [
-      '-m',
-      args.modelPath,
-      '--host',
-      listenHost,
-      '--port',
-      port,
-      '--alias',
-      args.modelAlias,
-      '--ctx-size',
-      String(args.ctxSize ?? 32768),
-      '--jinja',
-    ], {
+    child = spawnImpl(binaryPath, spawnArgs, {
       detached: true,
       stdio: ['ignore', 'pipe', 'pipe'],
       windowsHide: true,
@@ -424,6 +447,9 @@ export async function startLlamaCppServer(args: {
     })
   })
   child.unref()
+  if (typeof child.pid === 'number') {
+    await writePidFile(child.pid).catch(() => {})
+  }
 
   const ready = await waitForServedModel({
     host,
@@ -468,6 +494,73 @@ async function waitForServedModel(args: {
   return startFailure('readiness-timeout')
 }
 
+function pidFilePath(): string {
+  return path.join(getConfigDir(), 'llamacpp.pid')
+}
+
+async function writePidFile(pid: number): Promise<void> {
+  await ensureConfigDir()
+  await atomicWriteText(pidFilePath(), String(pid))
+}
+
+async function readPidFile(): Promise<number | null> {
+  try {
+    const raw = await fs.readFile(pidFilePath(), 'utf8')
+    const pid = Number.parseInt(raw.trim(), 10)
+    return Number.isInteger(pid) && pid > 0 ? pid : null
+  } catch {
+    return null
+  }
+}
+
+async function clearPidFile(): Promise<void> {
+  await fs.rm(pidFilePath(), { force: true }).catch(() => {})
+}
+
+export async function stopLlamaCppServer(args: {
+  host?: string
+  timeoutMs?: number
+  pollMs?: number
+  killImpl?: (pid: number, signal?: NodeJS.Signals | number) => void
+} = {}): Promise<
+  | { ok: true; stopped: boolean; reason?: 'untracked-server'; servedModels?: string[] }
+  | { ok: false; message: string }
+> {
+  const pid = await readPidFile()
+  if (!pid) {
+    const host = args.host ?? DEFAULT_LLAMA_HOST
+    const { up, models } = await fetchServedModels(host, 1500)
+    if (up && models.length > 0) {
+      return { ok: true, stopped: false, reason: 'untracked-server', servedModels: models }
+    }
+    return { ok: true, stopped: false }
+  }
+  const kill = args.killImpl ?? ((p, signal) => process.kill(p, signal))
+  try {
+    kill(pid, 'SIGTERM')
+  } catch (err: unknown) {
+    const code = (err as NodeJS.ErrnoException).code
+    if (code === 'ESRCH') {
+      await clearPidFile()
+      return { ok: true, stopped: false }
+    }
+    return { ok: false, message: (err as Error).message }
+  }
+  const host = args.host ?? DEFAULT_LLAMA_HOST
+  const deadline = Date.now() + (args.timeoutMs ?? 5000)
+  const pollMs = args.pollMs ?? 250
+  while (Date.now() < deadline) {
+    const status = await servedModelStatus(host, '__nothing__')
+    if (status.state === 'not-up' || status.models.length === 0) {
+      await clearPidFile()
+      return { ok: true, stopped: true }
+    }
+    await new Promise<void>(resolve => setTimeout(resolve, pollMs))
+  }
+  await clearPidFile()
+  return { ok: true, stopped: true }
+}
+
 async function servedModelStatus(host: string, modelAlias: string): Promise<
   | { state: 'not-up'; models: string[] }
   | { state: 'ready'; models: string[] }
@@ -507,6 +600,8 @@ function startFailureMessage(code: LlamaCppStartFailureCode, servedModels: strin
       return 'local runner closed before becoming ready'
     case 'readiness-timeout':
       return 'local runner is still loading or did not answer in time'
+    case 'untracked-server':
+      return detail ?? 'a llama-server is already running and ethagent did not launch it; cannot apply the vision encoder until that process is stopped'
   }
 }
 

package/src/models/llamacppPreflight.ts

@@ -1,5 +1,6 @@
 import {
   startLlamaCppServer,
+  stopLlamaCppServer,
   type LlamaCppStartFailureCode,
   type LlamaCppStartResult,
 } from './llamacpp.js'
@@ -21,9 +22,13 @@ export type LlamaCppPreflightDeps = {
   fetchImpl?: typeof fetch
   findLocalModel?: typeof findLocalHfModel
   startServer?: typeof startLlamaCppServer
+  stopServer?: typeof stopLlamaCppServer
   timeoutMs?: number
 }
 
+const UNTRACKED_VISION_DETAIL =
+  'A llama-server is already serving this alias but ethagent did not launch it, so we cannot apply the vision projector. Stop the external process and reopen ethagent.'
+
 type ModelsProbe =
   | { up: true; models: string[] }
   | { up: false; models: [] }
@@ -50,18 +55,31 @@ export async function ensureLlamaCppRunnerReady(
 
   const probe = await probeLlamaCppModels(baseUrl, deps)
   if (probe.up) {
-    if (probe.models.length === 0 || probe.models.includes(config.model)) {
-      return { ok: true, alreadyRunning: true }
+    if (probe.models.length > 0 && !probe.models.includes(config.model)) {
+      return {
+        ok: false,
+        code: 'different-model-running',
+        message: formatPreflightFailure(
+          'local runner is serving a different model',
+          config.model,
+          `a different local model is already running (${probe.models.join(', ')}); stop it before switching models`,
+        ),
+        servedModels: probe.models,
+      }
     }
-    return {
-      ok: false,
-      code: 'different-model-running',
-      message: formatPreflightFailure(
-        'local runner is serving a different model',
-        config.model,
-        `a different local model is already running (${probe.models.join(', ')}); stop it before switching models`,
-      ),
-      servedModels: probe.models,
+    if (!local.mmprojPath) return { ok: true, alreadyRunning: true }
+    const stopped = await (deps.stopServer ?? stopLlamaCppServer)().catch(() => null)
+    if (stopped && stopped.ok && stopped.reason === 'untracked-server') {
+      return withPreflightMessage(
+        {
+          ok: false,
+          code: 'untracked-server',
+          message: UNTRACKED_VISION_DETAIL,
+          detail: UNTRACKED_VISION_DETAIL,
+          servedModels: stopped.servedModels,
+        },
+        local,
+      )
     }
   }
 
@@ -69,6 +87,7 @@ export async function ensureLlamaCppRunnerReady(
     modelPath: local.localPath,
     modelAlias: local.id,
     host: llamaCppServerHostFromBaseUrl(baseUrl),
+    mmprojPath: local.mmprojPath,
   })
   if (result.ok) return { ok: true, alreadyRunning: result.alreadyRunning }
   return withPreflightMessage(result, local)

package/src/models/modelPickerOptions.ts

@@ -23,6 +23,9 @@ export type LocalHfPickerModel = {
   risk: HfRisk
   task: HfTask
   status: 'ready' | 'incomplete'
+  mmprojPath?: string
+  mmprojAvailable?: boolean
+  mmprojSizeBytes?: number
 }
 
 export type CloudCredentialKind = 'apikey' | 'oauth'
@@ -185,12 +188,22 @@ function appendHfModelOptions(
       displayName: model.displayName,
       maxLength,
     })
+    const tags = ['Installed']
+    if (model.mmprojPath) tags.push('Vision encoder loaded')
     options.push(rowOption(
       `hf:${id}`,
       contextFitLabel('llamacpp', id, `${active ? '* ' : '  '}${displayName}`, context.contextFit),
       undefined,
-      modelMetadataSubtext(size, ['Installed']),
+      modelMetadataSubtext(size, tags),
     ))
+    if (model.mmprojAvailable && !model.mmprojPath) {
+      const projectorSize = model.mmprojSizeBytes ? ` (+${formatSize(model.mmprojSizeBytes)})` : ''
+      options.push(rowOption(
+        `hfmmproj:${id}`,
+        `    + Add Vision Encoder${projectorSize}`,
+        'Enable image input on this local model',
+      ))
+    }
   }
 }
 

package/src/providers/anthropic.ts

@@ -4,6 +4,7 @@ import { ProviderError } from './contracts.js'
 import { providerErrorFromResponse } from './errors.js'
 import { fetchWithRetryStreamEvents } from './retry.js'
 import { iterSseEvents } from './sse.js'
+import { hasImageBlocks, ImageLoadError, loadImageBlock } from '../utils/images.js'
 
 export type AnthropicToolDefinition = {
   name: string
@@ -75,7 +76,22 @@ export class AnthropicProvider implements Provider {
       return
     }
 
-    const { system, conversation } = splitMessages(messages)
+    if (hasImageBlocks(messages) && !supportsAnthropicImages(this.model)) {
+      yield { type: 'error', message: `image input is not enabled for ${this.model}` }
+      return
+    }
+
+    let split: { system?: string; conversation: Awaited<ReturnType<typeof splitMessages>>['conversation'] }
+    try {
+      split = await splitMessages(messages)
+    } catch (err: unknown) {
+      if (err instanceof ImageLoadError) {
+        yield { type: 'error', message: err.message }
+        return
+      }
+      throw err
+    }
+    const { system, conversation } = split
 
     let response: Response
     try {
@@ -195,22 +211,24 @@ export class AnthropicProvider implements Provider {
   }
 }
 
-function splitMessages(messages: Message[]): {
+async function splitMessages(messages: Message[]): Promise<{
   system?: string
   conversation: Array<{
     role: 'user' | 'assistant'
     content: Array<
       | { type: 'text'; text: string }
+      | { type: 'image'; source: { type: 'base64'; media_type: string; data: string } }
       | { type: 'tool_use'; id: string; name: string; input: Record<string, unknown> }
       | { type: 'tool_result'; tool_use_id: string; content: string; is_error?: boolean }
     >
   }>
-} {
+}> {
   const systemParts: string[] = []
   const conversation: Array<{
     role: 'user' | 'assistant'
     content: Array<
       | { type: 'text'; text: string }
+      | { type: 'image'; source: { type: 'base64'; media_type: string; data: string } }
       | { type: 'tool_use'; id: string; name: string; input: Record<string, unknown> }
       | { type: 'tool_result'; tool_use_id: string; content: string; is_error?: boolean }
     >
@@ -226,11 +244,16 @@ function splitMessages(messages: Message[]): {
     }
     conversation.push({
       role: message.role,
-      content: blocks.map(block => {
+      content: await Promise.all(blocks.map(async block => {
         if (block.type === 'text') return { type: 'text', text: block.text }
+        if (block.type === 'image') {
+          const loaded = await loadImageBlock(block)
+          if (!loaded.dataBase64 || !loaded.mimeType) throw new Error(`could not load image: ${block.path}`)
+          return { type: 'image', source: { type: 'base64', media_type: loaded.mimeType, data: loaded.dataBase64 } }
+        }
         if (block.type === 'tool_use') return { type: 'tool_use', id: block.id, name: block.name, input: block.input }
         return { type: 'tool_result', tool_use_id: block.toolUseId, content: block.content, is_error: block.isError }
-      }),
+      })),
     })
   }
 
@@ -251,6 +274,14 @@ function normalizeBlocks(content: Message['content']): MessageContentBlock[] {
   })
 }
 
+export function supportsAnthropicImages(model: string): boolean {
+  const normalized = model.toLowerCase()
+  return normalized.includes('claude-3')
+    || normalized.includes('claude-sonnet-4')
+    || normalized.includes('claude-opus-4')
+    || normalized.includes('claude-haiku-4')
+}
+
 function normalizeStopReason(value?: string): 'end_turn' | 'tool_use' | 'max_tokens' | 'stop_sequence' | 'unknown' {
   if (value === 'end_turn' || value === 'tool_use' || value === 'max_tokens' || value === 'stop_sequence') {
     return value

package/src/providers/contracts.ts

@@ -8,6 +8,14 @@ export type TextBlock = {
   text: string
 }
 
+export type ImageBlock = {
+  type: 'image'
+  path: string
+  mimeType?: string
+  url?: string
+  dataBase64?: string
+}
+
 export type ToolUseBlock = {
   type: 'tool_use'
   id: string
@@ -22,7 +30,7 @@ export type ToolResultBlock = {
   isError?: boolean
 }
 
-export type MessageContentBlock = TextBlock | ToolUseBlock | ToolResultBlock
+export type MessageContentBlock = TextBlock | ImageBlock | ToolUseBlock | ToolResultBlock
 
 export type Message = {
   role: Role