ethagent 2.3.0 → 3.0.0

package/src/models/ModelPicker.tsx

@@ -7,12 +7,13 @@ import { Surface } from '../ui/Surface.js'
 import { ProgressBar } from '../ui/ProgressBar.js'
 import { theme } from '../ui/theme.js'
 import {
-  buildLlamaCppRunner,
   DEFAULT_LLAMA_HOST,
   detectLlamaCpp,
   installLlamaCppRunner,
+  killRogueLlamaProcesses,
   setLlamaCppServerPath,
   startLlamaCppServer,
+  stopLlamaCppServer,
   type LlamaCppInstallProgress,
   type LlamaCppInstallResult,
   type LlamaCppStartResult,
@@ -32,6 +33,8 @@ import { defaultModelFor, type EthagentConfig, type ProviderId } from '../storag
 import { clearModelCatalogCache, discoverProviderModels, isOpenAIOAuthAllowedModel, OPENAI_OAUTH_DEFAULT_MODEL, type ModelCatalogResult } from './catalog.js'
 import { contextWindowInfo } from '../runtime/compaction.js'
 import {
+  addMmprojToInstalledModel,
+  backfillMmprojForModels,
   createHfDownloadPlan,
   downloadHfModel,
   fetchHuggingFaceRepoInfo,
@@ -68,7 +71,7 @@ import { formatLocalHfModelDisplayName, formatModelDisplayName } from './modelDi
 import { fetchUncensoredGgufCatalog, type UncensoredCatalogEntry } from './uncensoredCatalog.js'
 
 export type ModelPickerSelection =
-  | { kind: 'llamacpp'; model: string }
+  | { kind: 'llamacpp'; model: string; mmprojPath?: string }
   | { kind: 'cloud'; provider: CloudProviderId; model: string; keyJustSet: boolean }
 
 type ModelPickerProps = {
@@ -77,6 +80,7 @@ type ModelPickerProps = {
   currentModel: string
   contextFit?: ModelPickerContextFit | null
   featuredHfRepo?: string
+  localOnly?: boolean
   onPick: (selection: ModelPickerSelection) => void
   onCancel: () => void
 }
@@ -113,6 +117,9 @@ type State =
   | { kind: 'localRunnerPathEntry'; data: LoadedData; model: LocalHfModel; submitting: boolean; error?: string }
   | { kind: 'localRunnerStarting'; data: LoadedData; model: LocalHfModel; startedAt: number }
   | { kind: 'localRunnerStartFail'; data: LoadedData; model: LocalHfModel; result: Extract<LlamaCppStartResult, { ok: false }> }
+  | { kind: 'mmprojOffer'; data: LoadedData; model: LocalHfModel }
+  | { kind: 'mmprojDownloading'; data: LoadedData; model: LocalHfModel; progress: HfDownloadProgress }
+  | { kind: 'mmprojError'; data: LoadedData; model: LocalHfModel; message: string }
 
 export const ModelPicker: React.FC<ModelPickerProps> = ({
   currentConfig,
@@ -120,12 +127,20 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
   currentModel,
   contextFit,
   featuredHfRepo,
+  localOnly = false,
   onPick,
   onCancel,
 }) => {
   const [state, setState] = useState<State>({ kind: 'loading' })
   const hfAbortRef = useRef<AbortController | null>(null)
   const oauthServiceRef = useRef<OpenAIOAuthService | null>(null)
+  const dismissToList = (data: LoadedData) => () => {
+    if (localOnly) {
+      onCancel()
+    } else {
+      setState({ kind: 'list', data })
+    }
+  }
 
   useEffect(() => {
     let cancelled = false
@@ -187,8 +202,20 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
 
   if (state.kind === 'loading') {
     return (
-      <Surface title={contextFit ? 'Switch to Larger-Context Model' : 'Switch Provider · Model'} subtitle="Loading providers and models.">
-        <Spinner label="loading providers..." />
+      <Surface
+        title={localOnly ? 'Local Model' : (contextFit ? 'Switch to Larger-Context Model' : 'Switch Provider · Model')}
+        subtitle="Loading providers and models."
+        footer="esc back"
+      >
+        <Spinner label={localOnly ? 'loading local models...' : 'loading providers...'} />
+        <Box marginTop={1}>
+          <Select<'cancel'>
+            options={[{ value: 'cancel', label: 'Back', hint: 'Return to the previous screen', role: 'utility' }]}
+            hintLayout="inline"
+            onSubmit={() => onCancel()}
+            onCancel={() => onCancel()}
+          />
+        </Box>
       </Surface>
     )
   }
@@ -204,7 +231,7 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
           label="Model Link"
           placeholder={LOCAL_MODEL_LINK_HINT}
           onSubmit={value => void inspectHfInput(state, value, setState)}
-          onCancel={() => setState({ kind: 'list', data: state.data })}
+          onCancel={dismissToList(state.data)}
         />
         {state.error ? <Text color={theme.accentError}>{state.error}</Text> : null}
       </Surface>
@@ -244,6 +271,7 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
     const canDownload = plan.review.risk !== 'high' && plan.review.runtime === 'llama.cpp runnable'
     const fit = state.data.machineSpec ? estimateGgufMachineFit(plan.sizeBytes, state.data.machineSpec) : null
     const recommended = state.data.machineSpec ? recommendGgufFile(plan.repo, ggufFiles(plan.repo), state.data.machineSpec) : null
+    const mmproj = plan.mmprojCandidate
     return (
       <Surface
         title="Review Model Link"
@@ -260,19 +288,28 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
           <Text color={riskColor(plan.review.risk)}>safety: {safetyLabel(plan.review.risk)} · source: {credibilityLabel(plan.review.credibility)}</Text>
           <Text color={theme.dim}>signals: {formatSignals(plan.repo.downloads, plan.repo.likes)}</Text>
           <Text color={theme.dim}>notes: {friendlyReasons(plan.review.reasons).join('; ')}</Text>
+          {mmproj ? (
+            <Text color={theme.dim}>vision encoder available: {friendlyFileName(mmproj.filename)} (+{formatBytes(mmproj.sizeBytes)})</Text>
+          ) : null}
         </Box>
-        <Select<'download' | 'pick' | 'cancel'>
+        <Select<'download' | 'downloadWithMmproj' | 'pick' | 'cancel'>
           options={[
-            { value: 'download', label: 'Download This Model', disabled: !canDownload },
+            { value: 'download', role: 'section', label: 'Download' },
+            ...(mmproj ? [{ value: 'downloadWithMmproj' as const, label: `Download Model + Vision Encoder (+${formatBytes(mmproj.sizeBytes)}) · recommended`, disabled: !canDownload }] : []),
+            { value: 'download', label: mmproj ? 'Download Without Vision Encoder' : 'Download This Model', disabled: !canDownload },
+            { value: 'pick', role: 'section', label: 'Navigation' },
             { value: 'pick', label: 'Pick Another File' },
-            { value: 'cancel', label: 'Cancel' },
+            { value: 'cancel', label: 'Cancel', role: 'utility' },
           ]}
           onSubmit={choice => {
             if (choice === 'download') void startHfDownload(state, setState, hfAbortRef, onPick)
+            else if (choice === 'downloadWithMmproj') {
+              void startHfDownload({ ...state, plan: { ...plan, includeMmproj: true } }, setState, hfAbortRef, onPick)
+            }
             else if (choice === 'pick') void inspectHfInput({ kind: 'hfInput', data: state.data }, plan.repoId, setState)
-            else setState({ kind: 'list', data: state.data })
+            else dismissToList(state.data)()
           }}
-          onCancel={() => setState({ kind: 'list', data: state.data })}
+          onCancel={dismissToList(state.data)}
         />
       </Surface>
     )
@@ -291,6 +328,68 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
     )
   }
 
+  if (state.kind === 'mmprojOffer') {
+    const sizeLabel = state.model.mmprojSizeBytes ? `+${formatBytes(state.model.mmprojSizeBytes)}` : 'additional download'
+    return (
+      <Surface
+        title="Add Image Support?"
+        subtitle={`${state.model.displayName} has a vision encoder available in its Hugging Face repo.`}
+        footer="enter select · esc back"
+      >
+        <Box flexDirection="column" marginBottom={1}>
+          <Text color={theme.dim}>Loading the vision encoder lets this model accept pasted images.</Text>
+          <Text color={theme.dim}>Without it, image paste is declined at submit time.</Text>
+        </Box>
+        <Select<'add' | 'skip' | 'cancel'>
+          options={[
+            { value: 'add', label: `Add Vision Encoder (${sizeLabel}) And Use` },
+            { value: 'skip', label: 'Use Without Image Support' },
+            { value: 'cancel', label: 'Cancel' },
+          ]}
+          onSubmit={choice => {
+            if (choice === 'add') void downloadMmprojAndContinue(state, setState, onPick)
+            else if (choice === 'skip') void startAndPickHfModel({ ...state.model, mmprojAvailable: false }, state, setState, onPick)
+            else dismissToList(state.data)()
+          }}
+          onCancel={dismissToList(state.data)}
+        />
+      </Surface>
+    )
+  }
+
+  if (state.kind === 'mmprojDownloading') {
+    const total = state.progress.total ?? state.model.mmprojSizeBytes ?? 0
+    const completed = state.progress.completed ?? 0
+    const progress = total > 0 ? completed / total : 0
+    const suffix = total > 0 ? `${formatBytes(completed)} / ${formatBytes(total)}` : formatBytes(completed)
+    return (
+      <Surface title="Downloading Vision Encoder" subtitle={state.model.displayName}>
+        <Text color={theme.dim}>{state.progress.status}</Text>
+        <ProgressBar progress={progress} suffix={suffix} />
+      </Surface>
+    )
+  }
+
+  if (state.kind === 'mmprojError') {
+    return (
+      <Surface title="Vision Encoder Download Failed" subtitle={state.message} tone="error" footer="enter select · esc back">
+        <Select<'retry' | 'skip' | 'back'>
+          options={[
+            { value: 'retry', label: 'Retry Download' },
+            { value: 'skip', label: 'Use Without Image Support' },
+            { value: 'back', label: 'Back To Picker' },
+          ]}
+          onSubmit={choice => {
+            if (choice === 'retry') setState({ kind: 'mmprojOffer', data: state.data, model: state.model })
+            else if (choice === 'skip') void startAndPickHfModel({ ...state.model, mmprojAvailable: false }, state, setState, onPick)
+            else dismissToList(state.data)()
+          }}
+          onCancel={dismissToList(state.data)}
+        />
+      </Surface>
+    )
+  }
+
   if (state.kind === 'hfDone') {
     return (
       <Surface
@@ -305,9 +404,9 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
           ]}
           onSubmit={choice => {
             if (choice === 'use') void startAndPickHfModel(state.model, state, setState, onPick)
-            else setState({ kind: 'list', data: state.data })
+            else dismissToList(state.data)()
           }}
-          onCancel={() => setState({ kind: 'list', data: state.data })}
+          onCancel={dismissToList(state.data)}
         />
       </Surface>
     )
@@ -323,9 +422,9 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
           ]}
           onSubmit={choice => {
             if (choice === 'retry') setState({ kind: 'hfInput', data: state.data, error: state.input ? undefined : state.message })
-            else setState({ kind: 'list', data: state.data })
+            else dismissToList(state.data)()
           }}
-          onCancel={() => setState({ kind: 'list', data: state.data })}
+          onCancel={dismissToList(state.data)}
         />
       </Surface>
     )
@@ -355,7 +454,7 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
               const target = targets.find(item => `${item.kind}:${item.id}` === value)
               if (target) setState({ kind: 'localUninstallConfirm', data: state.data, target })
             }}
-            onCancel={() => setState({ kind: 'list', data: state.data })}
+            onCancel={dismissToList(state.data)}
           />
         )}
       </Surface>
@@ -401,8 +500,8 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
       <Surface title="Local Model Uninstalled" subtitle={state.modelName} footer="enter back to picker · esc close">
         <Select<'back'>
           options={[{ value: 'back', label: 'Back To Picker' }]}
-          onSubmit={() => setState({ kind: 'list', data: state.data })}
-          onCancel={() => setState({ kind: 'list', data: state.data })}
+          onSubmit={dismissToList(state.data)}
+          onCancel={dismissToList(state.data)}
         />
       </Surface>
     )
@@ -418,9 +517,9 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
           ]}
           onSubmit={choice => {
             if (choice === 'retry') void uninstallLocalModel({ kind: 'localUninstallConfirm', data: state.data, target: state.target }, setState)
-            else setState({ kind: 'list', data: state.data })
+            else dismissToList(state.data)()
           }}
-          onCancel={() => setState({ kind: 'list', data: state.data })}
+          onCancel={dismissToList(state.data)}
         />
       </Surface>
     )
@@ -449,9 +548,9 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
             if (choice === 'download') setState({ kind: 'hfInput', data: state.data })
             else if (choice === 'install') void installRunnerAndStart(state, setState, onPick)
             else if (choice === 'path') setState({ kind: 'localRunnerPathEntry', data: state.data, model: state.model, submitting: false })
-            else setState({ kind: 'list', data: state.data })
+            else dismissToList(state.data)()
           }}
-          onCancel={() => setState({ kind: 'list', data: state.data })}
+          onCancel={dismissToList(state.data)}
         />
       </Surface>
     )
@@ -470,15 +569,20 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
     const options = buildRunnerRecoveryOptions(state.result)
     return (
       <Surface title="Runner Setup Needs Attention" subtitle={state.result.message} tone="error" footer="enter select · esc back">
-        <Select<'retry' | 'build' | 'path' | 'back'>
+        <Select<'stop-and-retry' | 'path' | 'back'>
           options={options}
+          hintLayout="inline"
           onSubmit={choice => {
-            if (choice === 'retry') void installRunnerAndStart({ kind: 'localRunnerSetup', data: state.data, model: state.model }, setState, onPick)
-            else if (choice === 'build') void buildRunnerAndStart({ kind: 'localRunnerSetup', data: state.data, model: state.model }, setState, onPick)
+            if (choice === 'stop-and-retry') {
+              void (async () => {
+                await killRogueLlamaProcesses()
+                await installRunnerAndStart({ kind: 'localRunnerSetup', data: state.data, model: state.model }, setState, onPick)
+              })()
+            }
             else if (choice === 'path') setState({ kind: 'localRunnerPathEntry', data: state.data, model: state.model, submitting: false })
-            else setState({ kind: 'list', data: state.data })
+            else dismissToList(state.data)()
           }}
-          onCancel={() => setState({ kind: 'list', data: state.data })}
+          onCancel={dismissToList(state.data)}
         />
       </Surface>
     )
@@ -528,9 +632,9 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
             if (choice === 'retry') void startAndPickHfModel(state.model, { kind: 'hfDone', data: state.data, model: state.model }, setState, onPick)
             else if (choice === 'path') setState({ kind: 'localRunnerPathEntry', data: state.data, model: state.model, submitting: false })
             else if (choice === 'install') void installRunnerAndStart({ kind: 'localRunnerSetup', data: state.data, model: state.model }, setState, onPick)
-            else setState({ kind: 'list', data: state.data })
+            else dismissToList(state.data)()
           }}
-          onCancel={() => setState({ kind: 'list', data: state.data })}
+          onCancel={dismissToList(state.data)}
         />
       </Surface>
     )
@@ -554,7 +658,7 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
             placeholder={providerKeyPlaceholder(provider)}
             isSecret
             onSubmit={(value) => void submitKey(state, value, currentConfig, setState)}
-            onCancel={() => setState({ kind: 'list', data: state.data })}
+            onCancel={dismissToList(state.data)}
           />
         )}
         {error ? <Text color={theme.accentError}>{error}</Text> : null}
@@ -576,9 +680,11 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
         ) : (
           <Select
             options={[
+              { value: 'edit', role: 'section', label: 'Credential' },
               { value: 'edit', label: 'Replace Stored API Key' },
               { value: 'delete', label: 'Remove Stored API Key' },
-              { value: 'cancel', label: 'Back' },
+              { value: 'cancel', role: 'section', label: 'Navigation' },
+              { value: 'cancel', label: 'Back', role: 'utility' },
             ]}
             onSubmit={(value) => {
               if (value === 'edit') {
@@ -591,7 +697,7 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
               }
               void deleteKey(state, currentConfig, setState, onPick, currentProvider)
             }}
-            onCancel={() => setState({ kind: 'list', data: state.data })}
+            onCancel={dismissToList(state.data)}
           />
         )}
         {error ? <Text color={theme.accentError}>{error}</Text> : null}
@@ -629,7 +735,7 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
               }
               void signOutOAuth(state, currentConfig, setState, onPick, currentProvider)
             }}
-            onCancel={() => setState({ kind: 'list', data: state.data })}
+            onCancel={dismissToList(state.data)}
           />
         )}
         {error ? <Text color={theme.accentError}>{error}</Text> : null}
@@ -655,9 +761,9 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
             onSubmit={choice => {
               if (choice === 'retry') void startOpenAIOAuthFlow(state.data, currentConfig, setState, oauthServiceRef, onPick)
               else if (choice === 'apikey') setState({ kind: 'keyEntry', provider: 'openai', action: 'set', data: state.data, submitting: false })
-              else setState({ kind: 'list', data: state.data })
+              else dismissToList(state.data)()
             }}
-            onCancel={() => setState({ kind: 'list', data: state.data })}
+            onCancel={dismissToList(state.data)}
           />
         </Surface>
       )
@@ -719,11 +825,12 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
           options={options}
           initialIndex={initialIndex === -1 ? 0 : initialIndex}
           maxVisible={12}
+          hintLayout="inline"
           onSubmit={(value) => {
             const parsed = parseFullCatalogValue(value)
             if (parsed) onPick({ kind: 'cloud', provider: parsed.provider, model: parsed.model, keyJustSet: false })
           }}
-          onCancel={() => setState({ kind: 'list', data: state.data })}
+          onCancel={dismissToList(state.data)}
         />
       </Surface>
     )
@@ -749,9 +856,9 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
           onSubmit={choice => {
             if (choice === 'retry') void openLocalCatalog(state.data, setState)
             else if (choice === 'paste') setState({ kind: 'hfInput', data: state.data })
-            else setState({ kind: 'list', data: state.data })
+            else dismissToList(state.data)()
           }}
-          onCancel={() => setState({ kind: 'list', data: state.data })}
+          onCancel={dismissToList(state.data)}
         />
       </Surface>
     )
@@ -770,28 +877,30 @@ export const ModelPicker: React.FC<ModelPickerProps> = ({
           options={options}
           initialIndex={initialIndex === -1 ? 0 : initialIndex}
           maxVisible={12}
-          onSubmit={(value) => handleSubmit(value, state, setState, onPick, onCancel, currentConfig, oauthServiceRef)}
-          onCancel={() => setState({ kind: 'list', data: state.data })}
+          hintLayout="inline"
+          onSubmit={(value) => handleSubmit(value, state, setState, onPick, onCancel, currentConfig, oauthServiceRef, localOnly)}
+          onCancel={dismissToList(state.data)}
         />
       </Surface>
     )
   }
 
   const { data } = state
-  const options = buildModelPickerOptions(data, { currentProvider, currentModel, contextFit })
+  const options = buildModelPickerOptions(data, { currentProvider, currentModel, contextFit }, { localOnly })
   const initialIndex = localOrCloudOptionIndex(options, currentProvider, currentModel)
 
   return (
     <Surface
-      title={contextFit ? 'Switch to Larger-Context Model' : 'Switch Provider · Model'}
-      subtitle={contextFit ? contextFitSubtitle(contextFit) : 'Downloaded GGUF files · cloud providers'}
-      footer="enter select · esc close · /models lists installed models"
+      title={localOnly ? 'Local Model' : (contextFit ? 'Switch to Larger-Context Model' : 'Switch Provider · Model')}
+      subtitle={localOnly ? 'Downloaded GGUF files and curated catalog' : (contextFit ? contextFitSubtitle(contextFit) : 'Downloaded GGUF files · cloud providers')}
+      footer={localOnly ? 'enter select · esc back' : 'enter select · esc close · /models lists installed models'}
     >
       <Select
         options={options}
         initialIndex={initialIndex === -1 ? 0 : initialIndex}
         maxVisible={10}
-        onSubmit={(value) => handleSubmit(value, state, setState, onPick, onCancel, currentConfig, oauthServiceRef)}
+        hintLayout="inline"
+        onSubmit={(value) => handleSubmit(value, state, setState, onPick, onCancel, currentConfig, oauthServiceRef, localOnly)}
         onCancel={onCancel}
       />
     </Surface>
@@ -806,6 +915,7 @@ function handleSubmit(
   onCancel: () => void,
   currentConfig: EthagentConfig,
   oauthServiceRef: React.MutableRefObject<OpenAIOAuthService | null>,
+  localOnly: boolean = false,
 ): void {
   if (value.startsWith('hdr:')) return
   if (value === 'cancel') {
@@ -813,7 +923,8 @@ function handleSubmit(
     return
   }
   if (value === 'back' && state.kind === 'localCatalog') {
-    setState({ kind: 'list', data: state.data })
+    if (localOnly) onCancel()
+    else setState({ kind: 'list', data: state.data })
     return
   }
   if (value.startsWith('hf:')) {
@@ -834,6 +945,18 @@ function handleSubmit(
     })()
     return
   }
+  if (value.startsWith('hfmmproj:') && state.kind === 'list') {
+    const id = value.slice('hfmmproj:'.length)
+    void (async () => {
+      const local = await findLocalHfModel(id)
+      if (!local) {
+        setState({ kind: 'hfError', data: state.data, message: 'local model metadata was not found' })
+        return
+      }
+      setState({ kind: 'mmprojOffer', data: state.data, model: local })
+    })()
+    return
+  }
   if (value.startsWith('uc:') && state.kind === 'localCatalog') {
     const entry = state.catalog.find(item => catalogOptionValue(item.repo.repoId, item.file.filename) === value)
     if (entry) void reviewCatalogModel(state, entry, setState)
@@ -1216,24 +1339,13 @@ export function buildHfFileOptions(
 }
 
 function buildRunnerRecoveryOptions(
-  result: Extract<LlamaCppInstallResult, { ok: false }>,
-): SelectOption<'retry' | 'build' | 'path' | 'back'>[] {
-  const options: SelectOption<'retry' | 'build' | 'path' | 'back'>[] = []
-  if (result.recovery.includes('source-build')) {
-    options.push({
-      value: 'build',
-      label: 'Build Local Runner',
-      hint: 'Uses git and CMake if installed',
-    })
-  }
-  if (result.recovery.includes('runner-path')) {
-    options.push({ value: 'path', label: 'Use Existing Runner Path' })
-  }
-  if (result.recovery.includes('retry-install')) {
-    options.push({ value: 'retry', label: 'Retry Automatic Install' })
-  }
-  options.push({ value: 'back', label: 'Back To Picker' })
-  return options
+  _result: Extract<LlamaCppInstallResult, { ok: false }>,
+): SelectOption<'stop-and-retry' | 'path' | 'back'>[] {
+  return [
+    { value: 'stop-and-retry', label: 'Stop and Retry', hint: 'Stop background runners and try the install again' },
+    { value: 'path', label: 'Use Existing Runner Path' },
+    { value: 'back', label: 'Back To Picker' },
+  ]
 }
 
 function localRunnerStartFailureSubtitle(result: Extract<LlamaCppStartResult, { ok: false }>): string {
@@ -1445,6 +1557,30 @@ async function uninstallLocalModel(
   }
 }
 
+async function downloadMmprojAndContinue(
+  state: Extract<State, { kind: 'mmprojOffer' }>,
+  setState: (s: State) => void,
+  onPick: (sel: ModelPickerSelection) => void,
+): Promise<void> {
+  setState({ kind: 'mmprojDownloading', data: state.data, model: state.model, progress: { status: 'starting' } })
+  try {
+    for await (const progress of addMmprojToInstalledModel(state.model.id)) {
+      setState({ kind: 'mmprojDownloading', data: state.data, model: state.model, progress })
+    }
+  } catch (err: unknown) {
+    setState({ kind: 'mmprojError', data: state.data, model: state.model, message: (err as Error).message })
+    return
+  }
+  const updated = await findLocalHfModel(state.model.id)
+  if (!updated || !updated.mmprojPath) {
+    setState({ kind: 'mmprojError', data: state.data, model: state.model, message: 'projector downloaded but path was not persisted' })
+    return
+  }
+  await stopLlamaCppServer().catch(() => null)
+  const data = { ...state.data, hfModels: await loadHfPickerModels() }
+  await startAndPickHfModel(updated, { kind: 'mmprojOffer', data, model: updated }, setState, onPick)
+}
+
 async function refreshLocalModelData(data: LoadedData): Promise<LoadedData> {
   const hfModels = await loadHfPickerModels()
   return {
@@ -1455,7 +1591,7 @@ async function refreshLocalModelData(data: LoadedData): Promise<LoadedData> {
 
 async function startAndPickHfModel(
   model: LocalHfModel,
-  state: Extract<State, { kind: 'list' | 'localCatalog' | 'hfDone' }>,
+  state: Extract<State, { kind: 'list' | 'localCatalog' | 'hfDone' | 'mmprojOffer' | 'mmprojError' }>,
   setState: (s: State) => void,
   onPick: (sel: ModelPickerSelection) => void,
 ): Promise<void> {
@@ -1463,10 +1599,15 @@ async function startAndPickHfModel(
     setState({ kind: 'hfError', data: state.data, message: 'blocked high-risk model; choose a model from a more credible source' })
     return
   }
+  if (model.mmprojAvailable && !model.mmprojPath && state.kind !== 'mmprojOffer' && state.kind !== 'mmprojError') {
+    setState({ kind: 'mmprojOffer', data: state.data, model })
+    return
+  }
   setState({ kind: 'localRunnerStarting', data: state.data, model, startedAt: Date.now() })
   const result = await startLlamaCppServer({
     modelPath: model.localPath,
     modelAlias: model.id,
+    mmprojPath: model.mmprojPath,
   })
   const llamaCpp = await probeLlamaCpp()
   const data = { ...state.data, llamaCpp }
@@ -1478,7 +1619,7 @@ async function startAndPickHfModel(
     setState({ kind: 'localRunnerStartFail', data, model, result })
     return
   }
-  onPick({ kind: 'llamacpp', model: model.id })
+  onPick({ kind: 'llamacpp', model: model.id, mmprojPath: model.mmprojPath })
 }
 
 async function installRunnerAndStart(
@@ -1489,14 +1630,6 @@ async function installRunnerAndStart(
   await runRunnerSetup(state, setState, onPick, installLlamaCppRunner)
 }
 
-async function buildRunnerAndStart(
-  state: Extract<State, { kind: 'localRunnerSetup' }>,
-  setState: (s: State) => void,
-  onPick: (sel: ModelPickerSelection) => void,
-): Promise<void> {
-  await runRunnerSetup(state, setState, onPick, buildLlamaCppRunner)
-}
-
 async function runRunnerSetup(
   state: Extract<State, { kind: 'localRunnerSetup' }>,
   setState: (s: State) => void,
@@ -1576,7 +1709,8 @@ function formatContextWindow(tokens: number): string {
 
 async function loadHfPickerModels(): Promise<ModelPickerOptionsData['hfModels']> {
   const installed = await loadLocalHfModels()
-  return installed.map(model => ({
+  const backfilled = await backfillMmprojForModels(installed)
+  return backfilled.map(model => ({
     id: model.id,
     displayName: model.displayName,
     sizeBytes: model.sizeBytes,
@@ -1584,6 +1718,9 @@ async function loadHfPickerModels(): Promise<ModelPickerOptionsData['hfModels']>
     risk: model.risk,
     task: model.task,
     status: model.status,
+    mmprojPath: model.mmprojPath,
+    mmprojAvailable: model.mmprojAvailable,
+    mmprojSizeBytes: model.mmprojSizeBytes,
   }))
 }