ethagent 2.2.0 → 2.4.0

package/src/models/ModelPicker.tsx

@@ -13,6 +13,7 @@ import {
   installLlamaCppRunner,
   setLlamaCppServerPath,
   startLlamaCppServer,
+  stopLlamaCppServer,
   type LlamaCppInstallProgress,
   type LlamaCppInstallResult,
   type LlamaCppStartResult,
@@ -32,6 +33,8 @@ import { defaultModelFor, type EthagentConfig, type ProviderId } from '../storag
 import { clearModelCatalogCache, discoverProviderModels, isOpenAIOAuthAllowedModel, OPENAI_OAUTH_DEFAULT_MODEL, type ModelCatalogResult } from './catalog.js'
 import { contextWindowInfo } from '../runtime/compaction.js'
 import {
+  addMmprojToInstalledModel,
+  backfillMmprojForModels,
   createHfDownloadPlan,
   downloadHfModel,
   fetchHuggingFaceRepoInfo,
@@ -68,7 +71,7 @@ import { formatLocalHfModelDisplayName, formatModelDisplayName } from './modelDi
 import { fetchUncensoredGgufCatalog, type UncensoredCatalogEntry } from './uncensoredCatalog.js'
 
 export type ModelPickerSelection =
-  | { kind: 'llamacpp'; model: string }
+  | { kind: 'llamacpp'; model: string; mmprojPath?: string }
   | { kind: 'cloud'; provider: CloudProviderId; model: string; keyJustSet: boolean }
 
 type ModelPickerProps = {
@@ -113,6 +116,9 @@ type State =
   | { kind: 'localRunnerPathEntry'; data: LoadedData; model: LocalHfModel; submitting: boolean; error?: string }
   | { kind: 'localRunnerStarting'; data: LoadedData; model: LocalHfModel; startedAt: number }
   | { kind: 'localRunnerStartFail'; data: LoadedData; model: LocalHfModel; result: Extract<LlamaCppStartResult, { ok: false }> }
+  | { kind: 'mmprojOffer'; data: LoadedData; model: LocalHfModel }
+  | { kind: 'mmprojDownloading'; data: LoadedData; model: LocalHfModel; progress: HfDownloadProgress }
+  | { kind: 'mmprojError'; data: LoadedData; model: LocalHfModel; message: string }
 
 export const ModelPicker: React.FC<ModelPickerProps> = ({
   currentConfig,
@@ -244,6 +250,7 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
     const canDownload = plan.review.risk !== 'high' && plan.review.runtime === 'llama.cpp runnable'
     const fit = state.data.machineSpec ? estimateGgufMachineFit(plan.sizeBytes, state.data.machineSpec) : null
     const recommended = state.data.machineSpec ? recommendGgufFile(plan.repo, ggufFiles(plan.repo), state.data.machineSpec) : null
+    const mmproj = plan.mmprojCandidate
     return (
       <Surface
         title="Review Model Link"
@@ -260,15 +267,22 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
           <Text color={riskColor(plan.review.risk)}>safety: {safetyLabel(plan.review.risk)} · source: {credibilityLabel(plan.review.credibility)}</Text>
           <Text color={theme.dim}>signals: {formatSignals(plan.repo.downloads, plan.repo.likes)}</Text>
           <Text color={theme.dim}>notes: {friendlyReasons(plan.review.reasons).join('; ')}</Text>
+          {mmproj ? (
+            <Text color={theme.dim}>vision encoder available: {friendlyFileName(mmproj.filename)} (+{formatBytes(mmproj.sizeBytes)})</Text>
+          ) : null}
         </Box>
-        <Select<'download' | 'pick' | 'cancel'>
+        <Select<'download' | 'downloadWithMmproj' | 'pick' | 'cancel'>
           options={[
-            { value: 'download', label: 'Download This Model', disabled: !canDownload },
+            ...(mmproj ? [{ value: 'downloadWithMmproj' as const, label: `Download Model + Vision Encoder (+${formatBytes(mmproj.sizeBytes)}) · recommended`, disabled: !canDownload }] : []),
+            { value: 'download', label: mmproj ? 'Download Without Vision Encoder' : 'Download This Model', disabled: !canDownload },
             { value: 'pick', label: 'Pick Another File' },
             { value: 'cancel', label: 'Cancel' },
           ]}
           onSubmit={choice => {
             if (choice === 'download') void startHfDownload(state, setState, hfAbortRef, onPick)
+            else if (choice === 'downloadWithMmproj') {
+              void startHfDownload({ ...state, plan: { ...plan, includeMmproj: true } }, setState, hfAbortRef, onPick)
+            }
             else if (choice === 'pick') void inspectHfInput({ kind: 'hfInput', data: state.data }, plan.repoId, setState)
             else setState({ kind: 'list', data: state.data })
           }}
@@ -291,6 +305,68 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
     )
   }
 
+  if (state.kind === 'mmprojOffer') {
+    const sizeLabel = state.model.mmprojSizeBytes ? `+${formatBytes(state.model.mmprojSizeBytes)}` : 'additional download'
+    return (
+      <Surface
+        title="Add Image Support?"
+        subtitle={`${state.model.displayName} has a vision encoder available in its Hugging Face repo.`}
+        footer="enter select · esc back"
+      >
+        <Box flexDirection="column" marginBottom={1}>
+          <Text color={theme.dim}>Loading the vision encoder lets this model accept pasted images.</Text>
+          <Text color={theme.dim}>Without it, image paste is declined at submit time.</Text>
+        </Box>
+        <Select<'add' | 'skip' | 'cancel'>
+          options={[
+            { value: 'add', label: `Add Vision Encoder (${sizeLabel}) And Use` },
+            { value: 'skip', label: 'Use Without Image Support' },
+            { value: 'cancel', label: 'Cancel' },
+          ]}
+          onSubmit={choice => {
+            if (choice === 'add') void downloadMmprojAndContinue(state, setState, onPick)
+            else if (choice === 'skip') void startAndPickHfModel({ ...state.model, mmprojAvailable: false }, state, setState, onPick)
+            else setState({ kind: 'list', data: state.data })
+          }}
+          onCancel={() => setState({ kind: 'list', data: state.data })}
+        />
+      </Surface>
+    )
+  }
+
+  if (state.kind === 'mmprojDownloading') {
+    const total = state.progress.total ?? state.model.mmprojSizeBytes ?? 0
+    const completed = state.progress.completed ?? 0
+    const progress = total > 0 ? completed / total : 0
+    const suffix = total > 0 ? `${formatBytes(completed)} / ${formatBytes(total)}` : formatBytes(completed)
+    return (
+      <Surface title="Downloading Vision Encoder" subtitle={state.model.displayName}>
+        <Text color={theme.dim}>{state.progress.status}</Text>
+        <ProgressBar progress={progress} suffix={suffix} />
+      </Surface>
+    )
+  }
+
+  if (state.kind === 'mmprojError') {
+    return (
+      <Surface title="Vision Encoder Download Failed" subtitle={state.message} tone="error" footer="enter select · esc back">
+        <Select<'retry' | 'skip' | 'back'>
+          options={[
+            { value: 'retry', label: 'Retry Download' },
+            { value: 'skip', label: 'Use Without Image Support' },
+            { value: 'back', label: 'Back To Picker' },
+          ]}
+          onSubmit={choice => {
+            if (choice === 'retry') setState({ kind: 'mmprojOffer', data: state.data, model: state.model })
+            else if (choice === 'skip') void startAndPickHfModel({ ...state.model, mmprojAvailable: false }, state, setState, onPick)
+            else setState({ kind: 'list', data: state.data })
+          }}
+          onCancel={() => setState({ kind: 'list', data: state.data })}
+        />
+      </Surface>
+    )
+  }
+
   if (state.kind === 'hfDone') {
     return (
       <Surface
@@ -612,9 +688,11 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
         ) : (
           <Select
             options={[
-              { value: 'signin', label: 'Sign in Again' },
-              { value: 'signout', label: 'Sign Out' },
-              { value: 'cancel', label: 'Back' },
+              { value: 'hdr:account', label: 'Account', disabled: true, role: 'section', bold: true },
+              { value: 'signin', label: 'Sign in Again', indent: 2 },
+              { value: 'signout', label: 'Sign Out', indent: 2 },
+              { value: 'hdr:nav', label: 'Navigation', disabled: true, role: 'section', bold: true },
+              { value: 'cancel', label: 'Back', indent: 2 },
             ]}
             onSubmit={(value) => {
               if (value === 'signin') {
@@ -832,6 +910,18 @@ function handleSubmit(
     })()
     return
   }
+  if (value.startsWith('hfmmproj:') && state.kind === 'list') {
+    const id = value.slice('hfmmproj:'.length)
+    void (async () => {
+      const local = await findLocalHfModel(id)
+      if (!local) {
+        setState({ kind: 'hfError', data: state.data, message: 'local model metadata was not found' })
+        return
+      }
+      setState({ kind: 'mmprojOffer', data: state.data, model: local })
+    })()
+    return
+  }
   if (value.startsWith('uc:') && state.kind === 'localCatalog') {
     const entry = state.catalog.find(item => catalogOptionValue(item.repo.repoId, item.file.filename) === value)
     if (entry) void reviewCatalogModel(state, entry, setState)
@@ -1248,6 +1338,8 @@ function localRunnerStartFailureSubtitle(result: Extract<LlamaCppStartResult, {
       return result.message
     case 'runner-not-installed':
       return 'this machine still needs a local runner'
+    case 'untracked-server':
+      return result.message
   }
 }
 
@@ -1443,6 +1535,39 @@ async function uninstallLocalModel(
   }
 }
 
+async function downloadMmprojAndContinue(
+  state: Extract<State, { kind: 'mmprojOffer' }>,
+  setState: (s: State) => void,
+  onPick: (sel: ModelPickerSelection) => void,
+): Promise<void> {
+  setState({ kind: 'mmprojDownloading', data: state.data, model: state.model, progress: { status: 'starting' } })
+  try {
+    for await (const progress of addMmprojToInstalledModel(state.model.id)) {
+      setState({ kind: 'mmprojDownloading', data: state.data, model: state.model, progress })
+    }
+  } catch (err: unknown) {
+    setState({ kind: 'mmprojError', data: state.data, model: state.model, message: (err as Error).message })
+    return
+  }
+  const updated = await findLocalHfModel(state.model.id)
+  if (!updated || !updated.mmprojPath) {
+    setState({ kind: 'mmprojError', data: state.data, model: state.model, message: 'projector downloaded but path was not persisted' })
+    return
+  }
+  const stopResult = await stopLlamaCppServer().catch(() => null)
+  if (stopResult && stopResult.ok && stopResult.reason === 'untracked-server') {
+    setState({
+      kind: 'mmprojError',
+      data: state.data,
+      model: updated,
+      message: 'Vision encoder downloaded, but a llama-server is already running and ethagent did not launch it. Quit ethagent, stop the external llama-server (taskkill /F /IM llama-server.exe on Windows, pkill llama-server on macOS or Linux), then reopen ethagent to load the projector.',
+    })
+    return
+  }
+  const data = { ...state.data, hfModels: await loadHfPickerModels() }
+  await startAndPickHfModel(updated, { kind: 'mmprojOffer', data, model: updated }, setState, onPick)
+}
+
 async function refreshLocalModelData(data: LoadedData): Promise<LoadedData> {
   const hfModels = await loadHfPickerModels()
   return {
@@ -1453,7 +1578,7 @@ async function refreshLocalModelData(data: LoadedData): Promise<LoadedData> {
 
 async function startAndPickHfModel(
   model: LocalHfModel,
-  state: Extract<State, { kind: 'list' | 'localCatalog' | 'hfDone' }>,
+  state: Extract<State, { kind: 'list' | 'localCatalog' | 'hfDone' | 'mmprojOffer' | 'mmprojError' }>,
   setState: (s: State) => void,
   onPick: (sel: ModelPickerSelection) => void,
 ): Promise<void> {
@@ -1461,10 +1586,15 @@ async function startAndPickHfModel(
     setState({ kind: 'hfError', data: state.data, message: 'blocked high-risk model; choose a model from a more credible source' })
     return
   }
+  if (model.mmprojAvailable && !model.mmprojPath && state.kind !== 'mmprojOffer' && state.kind !== 'mmprojError') {
+    setState({ kind: 'mmprojOffer', data: state.data, model })
+    return
+  }
   setState({ kind: 'localRunnerStarting', data: state.data, model, startedAt: Date.now() })
   const result = await startLlamaCppServer({
     modelPath: model.localPath,
     modelAlias: model.id,
+    mmprojPath: model.mmprojPath,
   })
   const llamaCpp = await probeLlamaCpp()
   const data = { ...state.data, llamaCpp }
@@ -1476,7 +1606,7 @@ async function startAndPickHfModel(
     setState({ kind: 'localRunnerStartFail', data, model, result })
     return
   }
-  onPick({ kind: 'llamacpp', model: model.id })
+  onPick({ kind: 'llamacpp', model: model.id, mmprojPath: model.mmprojPath })
 }
 
 async function installRunnerAndStart(
@@ -1574,7 +1704,8 @@ function formatContextWindow(tokens: number): string {
 
 async function loadHfPickerModels(): Promise<ModelPickerOptionsData['hfModels']> {
   const installed = await loadLocalHfModels()
-  return installed.map(model => ({
+  const backfilled = await backfillMmprojForModels(installed)
+  return backfilled.map(model => ({
     id: model.id,
     displayName: model.displayName,
     sizeBytes: model.sizeBytes,
@@ -1582,6 +1713,9 @@ async function loadHfPickerModels(): Promise<ModelPickerOptionsData['hfModels']>
     risk: model.risk,
     task: model.task,
     status: model.status,
+    mmprojPath: model.mmprojPath,
+    mmprojAvailable: model.mmprojAvailable,
+    mmprojSizeBytes: model.mmprojSizeBytes,
   }))
 }
 

package/src/models/catalog.ts

@@ -2,6 +2,7 @@ import { defaultModelFor, type EthagentConfig, type ProviderId } from '../storag
 import { getKey } from '../storage/secrets.js'
 import { loadLocalHfModels } from './huggingface.js'
 import { hasOpenAIOAuthCredentials } from '../auth/openaiOAuth/credentials.js'
+import { providerDisplayName } from './providerDisplay.js'
 
 const OPENAI_OAUTH_MODEL_IDS = ['gpt-5.5', 'gpt-5.4', 'gpt-5.4-mini', 'gpt-5.3-codex', 'gpt-5.2'] as const
 
@@ -85,7 +86,7 @@ export async function discoverProviderModels(
     if (provider === 'openai' && await hasOpenAIOAuthCredentials()) {
       return openAIOAuthCatalog()
     }
-    return fallbackResult(config, `missing ${provider} API key`)
+    return fallbackResult(config, `missing ${providerDisplayName(provider)} API key`)
   }
 
   const baseUrl = provider === 'openai' ? openAIBaseUrlFor(config) : ''

package/src/models/huggingface.ts

@@ -54,6 +54,12 @@ export type HfSafetyReview = {
   reasons: string[]
 }
 
+export type HfMmprojCandidate = {
+  filename: string
+  sizeBytes: number
+  localPath: string
+}
+
 export type HfDownloadPlan = {
   repo: HuggingFaceRepoInfo
   repoId: string
@@ -64,6 +70,8 @@ export type HfDownloadPlan = {
   localPath: string
   displayName: string
   review: HfSafetyReview
+  mmprojCandidate?: HfMmprojCandidate
+  includeMmproj?: boolean
 }
 
 export type LocalHfModel = {
@@ -90,6 +98,9 @@ export type LocalHfModel = {
   installedAt: string
   status: LocalHfStatus
   sha256?: string
+  mmprojPath?: string
+  mmprojAvailable?: boolean
+  mmprojSizeBytes?: number
 }
 
 export type HfDownloadProgress = {
@@ -291,6 +302,14 @@ export function ggufFiles(repo: HuggingFaceRepoInfo): HuggingFaceSibling[] {
     .sort((a, b) => a.filename.localeCompare(b.filename))
 }
 
+export function isMmprojFilename(filename: string): boolean {
+  return filename.toLowerCase().startsWith('mmproj-') && filename.toLowerCase().endsWith('.gguf')
+}
+
+export function findMmprojSibling(repo: HuggingFaceRepoInfo): HuggingFaceSibling | undefined {
+  return repo.siblings.find(file => isMmprojFilename(file.filename))
+}
+
 export async function createHfDownloadPlan(
   input: string,
   filename?: string,
@@ -320,6 +339,14 @@ export async function createHfDownloadPlan(
     requestedRevision,
     resolvedRevision,
   })
+  const mmprojSibling = findMmprojSibling(repo)
+  const mmprojCandidate: HfMmprojCandidate | undefined = mmprojSibling
+    ? {
+      filename: mmprojSibling.filename,
+      sizeBytes: mmprojSibling.sizeBytes ?? 0,
+      localPath: localPathFor(repo.repoId, resolvedRevision, mmprojSibling.filename),
+    }
+    : undefined
   return {
     repo,
     repoId: repo.repoId,
@@ -330,6 +357,7 @@ export async function createHfDownloadPlan(
     localPath: localPathFor(repo.repoId, resolvedRevision, selected.filename),
     displayName: displayNameFor(repo.repoId, selected.filename),
     review,
+    mmprojCandidate,
   }
 }
 
@@ -432,10 +460,151 @@ export async function* downloadHfModel(
   }
 
   await fs.rename(partialPath, plan.localPath)
-  await upsertLocalHfModel(modelFromPlan(plan, hash.digest('hex'), 'ready'))
+
+  let mmprojPath: string | undefined
+  if (plan.includeMmproj && plan.mmprojCandidate) {
+    yield* downloadMmprojFile(plan.repoId, plan.resolvedRevision, plan.mmprojCandidate, signal, fetchImpl)
+    mmprojPath = plan.mmprojCandidate.localPath
+  }
+
+  await upsertLocalHfModel(modelFromPlan(plan, hash.digest('hex'), 'ready', mmprojPath))
   yield { status: 'success', completed, total: Number.isFinite(total) ? total : completed }
 }
 
+async function* downloadMmprojFile(
+  repoId: string,
+  resolvedRevision: string,
+  candidate: HfMmprojCandidate,
+  signal: AbortSignal | undefined,
+  fetchImpl: FetchImpl,
+): AsyncIterable<HfDownloadProgress> {
+  await fs.mkdir(path.dirname(candidate.localPath), { recursive: true })
+  const partialPath = `${candidate.localPath}.partial`
+  const response = await fetchImpl(resolveUrl(repoId, resolvedRevision, candidate.filename), { signal })
+  if (!response.ok || !response.body) {
+    throw new Error(response.ok ? 'empty projector download body' : `projector download HTTP ${response.status}`)
+  }
+
+  const total = Number.parseInt(response.headers.get('content-length') ?? '', 10)
+  const handle = await fs.open(partialPath, 'w')
+  let completed = 0
+  let complete = false
+  let lastProgressAt = Date.now()
+  let lastProgressBytes = 0
+  yield { status: 'downloading-mmproj', completed, total: Number.isFinite(total) ? total : undefined }
+  try {
+    const reader = response.body.getReader()
+    while (true) {
+      const { done, value } = await reader.read()
+      if (done) break
+      if (signal?.aborted) throw new Error('Cancelled')
+      const buffer = Buffer.from(value)
+      await handle.write(buffer)
+      completed += buffer.byteLength
+      const now = Date.now()
+      if (shouldReportDownloadProgress(completed, lastProgressBytes, now, lastProgressAt)) {
+        lastProgressAt = now
+        lastProgressBytes = completed
+        yield { status: 'downloading-mmproj', completed, total: Number.isFinite(total) ? total : undefined }
+      }
+    }
+    complete = true
+  } finally {
+    await handle.close()
+    if (!complete) {
+      await fs.unlink(partialPath).catch(() => {})
+    }
+  }
+
+  await fs.rename(partialPath, candidate.localPath)
+}
+
+export async function backfillMmprojAvailability(
+  model: LocalHfModel,
+  fetchImpl: FetchImpl = fetch,
+): Promise<LocalHfModel> {
+  if (model.mmprojAvailable !== undefined) return model
+  try {
+    const repo = await fetchHuggingFaceRepoInfo({ repoId: model.repoId }, fetchImpl)
+    const sibling = findMmprojSibling(repo)
+    const next: LocalHfModel = {
+      ...model,
+      mmprojAvailable: Boolean(sibling),
+      mmprojSizeBytes: sibling?.sizeBytes,
+    }
+    await upsertLocalHfModel(next)
+    return next
+  } catch {
+    return model
+  }
+}
+
+export async function backfillMmprojForModels(
+  models: LocalHfModel[],
+  fetchImpl: FetchImpl = fetch,
+): Promise<LocalHfModel[]> {
+  const repoIdToProbe = new Map<string, Promise<HuggingFaceRepoInfo | null>>()
+  for (const model of models) {
+    if (model.mmprojAvailable !== undefined) continue
+    if (repoIdToProbe.has(model.repoId)) continue
+    repoIdToProbe.set(
+      model.repoId,
+      fetchHuggingFaceRepoInfo({ repoId: model.repoId }, fetchImpl).catch(() => null),
+    )
+  }
+  if (repoIdToProbe.size === 0) return models
+  const resolved = new Map<string, HuggingFaceRepoInfo | null>()
+  for (const [repoId, promise] of repoIdToProbe) {
+    resolved.set(repoId, await promise)
+  }
+  const out: LocalHfModel[] = []
+  for (const model of models) {
+    if (model.mmprojAvailable !== undefined) {
+      out.push(model)
+      continue
+    }
+    const repo = resolved.get(model.repoId)
+    if (!repo) {
+      out.push(model)
+      continue
+    }
+    const sibling = findMmprojSibling(repo)
+    const next: LocalHfModel = {
+      ...model,
+      mmprojAvailable: Boolean(sibling),
+      mmprojSizeBytes: sibling?.sizeBytes,
+    }
+    await upsertLocalHfModel(next)
+    out.push(next)
+  }
+  return out
+}
+
+export async function* addMmprojToInstalledModel(
+  modelId: string,
+  signal?: AbortSignal,
+  deps: { fetchImpl?: FetchImpl } = {},
+): AsyncIterable<HfDownloadProgress> {
+  const fetchImpl = deps.fetchImpl ?? fetch
+  const existing = await findLocalHfModel(modelId)
+  if (!existing) throw new Error(`model not installed: ${modelId}`)
+  if (existing.mmprojPath) {
+    yield { status: 'success', completed: 0 }
+    return
+  }
+  const repo = await fetchHuggingFaceRepoInfo({ repoId: existing.repoId }, fetchImpl)
+  const sibling = findMmprojSibling(repo)
+  if (!sibling) throw new Error(`no vision encoder available for ${existing.repoId}`)
+  const candidate: HfMmprojCandidate = {
+    filename: sibling.filename,
+    sizeBytes: sibling.sizeBytes ?? 0,
+    localPath: localPathFor(existing.repoId, existing.resolvedRevision, sibling.filename),
+  }
+  yield* downloadMmprojFile(existing.repoId, existing.resolvedRevision, candidate, signal, fetchImpl)
+  await upsertLocalHfModel({ ...existing, mmprojPath: candidate.localPath })
+  yield { status: 'success', completed: candidate.sizeBytes }
+}
+
 export function shouldReportDownloadProgress(
   completed: number,
   lastCompleted: number,
@@ -446,7 +615,13 @@ export function shouldReportDownloadProgress(
     || completed - lastCompleted >= DOWNLOAD_PROGRESS_MIN_BYTES
 }
 
-export function modelFromPlan(plan: HfDownloadPlan, sha256: string | undefined, status: LocalHfStatus): LocalHfModel {
+export function modelFromPlan(
+  plan: HfDownloadPlan,
+  sha256: string | undefined,
+  status: LocalHfStatus,
+  mmprojPath?: string,
+): LocalHfModel {
+  const mmprojAvailable = Boolean(plan.mmprojCandidate)
   const now = new Date().toISOString()
   return {
     id: localModelId(plan.repoId, plan.filename),
@@ -472,6 +647,9 @@ export function modelFromPlan(plan: HfDownloadPlan, sha256: string | undefined,
     installedAt: now,
     status,
     sha256,
+    mmprojPath,
+    mmprojAvailable,
+    mmprojSizeBytes: plan.mmprojCandidate?.sizeBytes,
   }
 }