anymd 0.0.2 → 0.0.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "anymd",
-  "version": "0.0.2",
+  "version": "0.0.3",
   "description": "Convert any document (PDF, DOC, DOCX) to clean Markdown for RAG",
   "keywords": [
     "markdown",

package/src/bootstrap.ts

@@ -11,38 +11,76 @@ interface BootstrapCallbacks {
 
 const REQUIRED_PACKAGES = ['marker', 'markitdown', 'mlx_vlm', 'pypdfium2']
 const PIP_PACKAGES = ['marker-pdf', 'markitdown', 'mlx-vlm', 'pypdfium2']
+const CHANDRA_MODEL_ID = 'mlx-community/chandra-8bit'
 
-const checkPackageImportable = async (pythonPath: string, pkg: string): Promise<boolean> => {
+const checkImportable = async (py: string, pkg: string): Promise<boolean> => {
   try {
-    const proc = Bun.spawn([pythonPath, '-c', `import ${pkg}`], { stderr: 'pipe', stdout: 'pipe' })
-    const code = await proc.exited
-    return code === 0
+    const proc = Bun.spawn([py, '-c', `import ${pkg}`], { stderr: 'pipe', stdout: 'pipe' })
+    return (await proc.exited) === 0
   } catch {
     return false
   }
 }
 
-const allPackagesInstalled = async (pythonPath: string): Promise<boolean> => {
-  const results = await Promise.all(REQUIRED_PACKAGES.map(async pkg => checkPackageImportable(pythonPath, pkg)))
+const allPackagesInstalled = async (py: string): Promise<boolean> => {
+  const results = await Promise.all(REQUIRED_PACKAGES.map(async pkg => checkImportable(py, pkg)))
   for (const r of results) if (!r) return false
   return true
 }
 
-const runCommand = async (args: string[]): Promise<{ ok: boolean; stderr: string }> => {
-  const proc = Bun.spawn(args, { stderr: 'pipe', stdout: 'pipe' })
-  const code = await proc.exited
-  const stderrText = await new Response(proc.stderr).text()
-  return { ok: code === 0, stderr: stderrText }
+const runQuiet = async (args: string[]): Promise<{ ok: boolean; stderr: string; stdout: string }> => {
+  try {
+    const proc = Bun.spawn(args, { stderr: 'pipe', stdout: 'pipe' })
+    const code = await proc.exited
+    const [stdout, stderr] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text()])
+    return { ok: code === 0, stderr, stdout }
+  } catch {
+    return { ok: false, stderr: 'command not found', stdout: '' }
+  }
+}
+
+const emitLines = (chunk: string, onLine: (l: string) => void): string => {
+  const lines = chunk.split('\n')
+  const remainder = lines.pop() ?? ''
+  for (const line of lines) {
+    const t = line.replaceAll('\r', '').trim()
+    if (t.length > 0) onLine(t)
+  }
+  return remainder
+}
+
+// eslint-disable-next-line max-statements
+const streamLines = async (stream: ReadableStream<Uint8Array>, onLine: (l: string) => void): Promise<void> => {
+  const reader = stream.getReader()
+  const decoder = new TextDecoder()
+  let buf = ''
+  for (;;) {
+    /** biome-ignore lint/performance/noAwaitInLoops: sequential stream reads */
+    const { done, value } = await reader.read() // eslint-disable-line no-await-in-loop
+    if (done) break
+    buf = emitLines(buf + decoder.decode(value, { stream: true }), onLine)
+  }
+  emitLines(`${buf}\n`, onLine)
+}
+
+const runStreaming = async (args: string[], onLine: (l: string) => void): Promise<boolean> => {
+  try {
+    const proc = Bun.spawn(args, { stderr: 'pipe', stdout: 'pipe' })
+    await Promise.all([
+      streamLines(proc.stdout as ReadableStream<Uint8Array>, onLine),
+      streamLines(proc.stderr as ReadableStream<Uint8Array>, onLine)
+    ])
+    return (await proc.exited) === 0
+  } catch {
+    return false
+  }
 }
 
 const UV_CANDIDATES = ['uv', '/opt/homebrew/bin/uv', join(homedir(), '.local', 'bin', 'uv')]
 
 const findUv = async (): Promise<string | undefined> => {
-  const checks = await Promise.all(
-    UV_CANDIDATES.map(async bin => ({ bin, ok: (await runCommand([bin, '--version'])).ok }))
-  )
-  const found = checks.find(c => c.ok)
-  return found?.bin
+  const checks = await Promise.all(UV_CANDIDATES.map(async bin => ({ bin, ok: (await runQuiet([bin, '--version'])).ok })))
+  return checks.find(c => c.ok)?.bin
 }
 
 interface VenvOpts {
@@ -53,39 +91,56 @@ interface VenvOpts {
 }
 
 const createVenv = async (opts: VenvOpts): Promise<boolean> => {
-  opts.cbs.onStep('Creating Python virtual environment via uv...')
-  const mkdirResult = await runCommand(['mkdir', '-p', opts.cacheDir])
-  if (!mkdirResult.ok) return false
-  const venvResult = await runCommand([opts.uv, 'venv', '--python', '3.13', opts.venvDir])
-  if (!venvResult.ok) {
-    opts.cbs.onStep(`Failed to create venv: ${venvResult.stderr}`)
+  opts.cbs.onStep('Creating Python 3.13 virtual environment...')
+  if (!(await runQuiet(['mkdir', '-p', opts.cacheDir])).ok) return false
+  const ok = await runStreaming([opts.uv, 'venv', '--python', '3.13', opts.venvDir], opts.cbs.onStep)
+  if (!ok) {
+    opts.cbs.onStep('Failed to create venv.')
     return false
   }
-  opts.cbs.onStep('Virtual environment created (Python 3.13).')
+  opts.cbs.onStep('Virtual environment created.')
   return true
 }
 
-const installPackages = async (venvDir: string, uv: string, cbs: BootstrapCallbacks): Promise<boolean> => {
+const installPkgs = async (venvDir: string, uv: string, cbs: BootstrapCallbacks): Promise<boolean> => {
   cbs.onStep(`Installing ${PIP_PACKAGES.join(', ')}...`)
-  const result = await runCommand([uv, 'pip', 'install', '--python', `${venvDir}/bin/python`, ...PIP_PACKAGES])
-  if (!result.ok) {
-    cbs.onStep(`uv pip install failed: ${result.stderr}`)
+  const ok = await runStreaming([uv, 'pip', 'install', '--python', `${venvDir}/bin/python`, ...PIP_PACKAGES], cbs.onStep)
+  if (!ok) {
+    cbs.onStep('Package installation failed.')
     return false
   }
-  cbs.onStep('Packages installed successfully.')
+  cbs.onStep('All packages installed.')
   return true
 }
 
-const ensureVenvExists = async (
-  uv: string,
-  cbs: BootstrapCallbacks
-): Promise<null | { skip: boolean; venvDir: string }> => {
+const downloadMarkerModels = async (py: string, cbs: BootstrapCallbacks): Promise<void> => {
+  cbs.onStep('Downloading marker PDF models (first run only)...')
+  const ok = await runStreaming([py, '-c', 'from marker.models import create_model_dict; create_model_dict()'], cbs.onStep)
+  cbs.onStep(ok ? 'Marker models ready.' : 'Marker model download failed (will retry on first convert).')
+}
+
+const downloadChandraModel = async (py: string, cbs: BootstrapCallbacks): Promise<void> => {
+  cbs.onStep(`Downloading OCR model ${CHANDRA_MODEL_ID} (first run only)...`)
+  const ok = await runStreaming([py, '-c', `from mlx_vlm import load; load("${CHANDRA_MODEL_ID}")`], cbs.onStep)
+  cbs.onStep(ok ? 'OCR model ready.' : 'OCR model download failed (will retry on first OCR).')
+}
+
+const chandraModelCached = async (py: string): Promise<boolean> => {
+  const r = await runQuiet([
+    py,
+    '-c',
+    `from huggingface_hub import scan_cache_dir; print(any(r.repo_id == "${CHANDRA_MODEL_ID}" for r in scan_cache_dir().repos))`
+  ])
+  return r.ok && r.stdout.trim() === 'True'
+}
+
+const ensureVenv = async (uv: string, cbs: BootstrapCallbacks): Promise<null | { skip: boolean; venvDir: string }> => {
   const { cacheDir, venvPython } = getPaths()
   const venvDir = `${cacheDir}/.venv`
   if (existsSync(venvPython)) {
     const installed = await allPackagesInstalled(venvPython)
     if (installed) return { skip: true, venvDir }
-    cbs.onStep('Some packages missing, installing...')
+    cbs.onStep('Some packages missing, reinstalling...')
     return { skip: false, venvDir }
   }
   const created = await createVenv({ cacheDir, cbs, uv, venvDir })
@@ -98,19 +153,29 @@ const requireUv = async (cbs: BootstrapCallbacks): Promise<string | undefined> =
   return uv
 }
 
-const setupAndInstall = async (uv: string, cbs: BootstrapCallbacks): Promise<boolean> => {
-  const result = await ensureVenvExists(uv, cbs)
+const ensurePackages = async (uv: string, cbs: BootstrapCallbacks): Promise<boolean> => {
+  const result = await ensureVenv(uv, cbs)
   if (!result) return false
   if (result.skip) return true
-  return installPackages(result.venvDir, uv, cbs)
+  return installPkgs(result.venvDir, uv, cbs)
+}
+
+const ensureModels = async (cbs: BootstrapCallbacks): Promise<void> => {
+  const py = getPaths().venvPython
+  const cached = await chandraModelCached(py)
+  if (cached) return
+  await downloadMarkerModels(py, cbs)
+  await downloadChandraModel(py, cbs)
 }
 
 const bootstrapPython = async (cbs: BootstrapCallbacks): Promise<boolean> => {
   const uv = await requireUv(cbs)
   if (!uv) return false
-  const ok = await setupAndInstall(uv, cbs)
-  if (ok) cbs.onDone()
-  return ok
+  const ok = await ensurePackages(uv, cbs)
+  if (!ok) return false
+  await ensureModels(cbs)
+  cbs.onDone()
+  return true
 }
 
 export { bootstrapPython }

package/tui.tsx

@@ -1003,25 +1003,22 @@ const App = () => {
       await clearErrorLog()
     }
 
-    const { existsSync } = await import('node:fs')
-    if (!existsSync(getPaths().venvPython)) {
-      dispatch({ line: 'Setting up Python environment...', type: 'APPEND_OUTPUT' })
-      const ok = await bootstrapPython({
-        onDone: () => {
-          dispatch({ line: 'Python environment ready.', type: 'APPEND_OUTPUT' })
-        },
-        onStep: (msg: string) => {
-          dispatch({ line: msg, type: 'APPEND_OUTPUT' })
-        }
-      })
-      if (!ok) {
-        dispatch({
-          errors: ['Python bootstrap failed. Install python3 and try again.'],
-          type: 'SET_PREFLIGHT',
-          warnings: []
-        })
-        return
+    dispatch({ line: 'Checking Python environment...', type: 'APPEND_OUTPUT' })
+    const ok = await bootstrapPython({
+      onDone: () => {
+        dispatch({ line: 'Python environment ready.', type: 'APPEND_OUTPUT' })
+      },
+      onStep: (msg: string) => {
+        dispatch({ line: msg, type: 'APPEND_OUTPUT' })
       }
+    })
+    if (!ok) {
+      dispatch({
+        errors: ['Python bootstrap failed. Install uv and try again.'],
+        type: 'SET_PREFLIGHT',
+        warnings: []
+      })
+      return
     }
 
     const preflight = await runPreflight()
@@ -1175,7 +1172,9 @@ const App = () => {
 
       <PreflightBanner errors={state.preflightErrors} warnings={state.preflightWarnings} />
 
-      {state.runningCommand ? <OutputBox lines={state.runningLines} status={state.runningStatus} /> : null}
+      {state.runningCommand || state.runningLines.length > 0 ? (
+        <OutputBox lines={state.runningLines} status={state.runningStatus} />
+      ) : null}
       {state.runningCommand && !state.failed ? <RunningFooter /> : null}
 
       {state.showLog ? <LogOverlay lines={state.logLines} /> : null}