anymd 0.0.0 → 0.0.2

package/README.md

@@ -50,16 +50,17 @@ Steps 2 and 3 run in parallel when both are needed.
 
 - macOS with Apple Silicon (64GB recommended for OCR)
 - [Bun](https://bun.sh) runtime
+- [uv](https://docs.astral.sh/uv/) — `curl -LsSf https://astral.sh/uv/install.sh | sh`
 - [poppler](https://poppler.freedesktop.org/) (`pdftotext`) — `brew install poppler`
 - [LibreOffice](https://www.libreoffice.org/) (`soffice`) — optional, only for `.doc` files
-- Python 3.13+ — venv and packages are auto-installed on first run
 
 The TUI preflight check tells you exactly what's missing.
 
 ## Auto-Bootstrap
 
-On first run, `anymd` creates `~/.cache/anymd/.venv` and installs:
+On first run, `anymd` uses `uv` to create `~/.cache/anymd/.venv` (Python 3.13) and installs:
 - `marker-pdf` — PDF to markdown conversion
+- `markitdown` — DOCX to markdown conversion
 - `mlx-vlm` — Apple Silicon MLX inference for OCR
 - `pypdfium2` — PDF page rendering
 

package/main.ts

@@ -6,30 +6,30 @@ import { getPaths, initPaths } from '~/paths'
 import { runPipeline } from '~/pipeline'
 import { logger } from '~/utils'
 
-const fileListIdx = process.argv.indexOf('--file-list'),
-  baseDirIdx = process.argv.indexOf('--base-dir')
-let inputDir = process.argv[2] ?? './data'
+const getArg = (flag: string): string | undefined => {
+  const idx = process.argv.indexOf(flag)
+  return idx === -1 ? undefined : process.argv[idx + 1]
+}
+
+const fileListPath = getArg('--file-list')
+const baseDir = getArg('--base-dir') ?? process.argv[2] ?? resolve('data')
+const outputDir = getArg('--output-dir') ?? resolve('./output')
+let inputDir = baseDir
 let inputFiles: string[] | undefined
-let baseDir = baseDirIdx === -1 ? '' : (process.argv[baseDirIdx + 1] ?? resolve('data'))
 
-if (fileListIdx !== -1) {
-  const listPath = process.argv[fileListIdx + 1]
-  if (listPath) {
-    const content = await readFile(listPath, 'utf8')
-    inputFiles = content
-      .trim()
-      .split('\n')
-      .filter(l => l.length > 0)
-    inputDir = '.'
-  }
+if (fileListPath) {
+  const content = await readFile(fileListPath, 'utf8')
+  inputFiles = content
+    .trim()
+    .split('\n')
+    .filter(l => l.length > 0)
+  inputDir = '.'
 }
 
-if (!baseDir) baseDir = inputDir
-
 try {
   getPaths()
 } catch {
-  initPaths(resolve(baseDir), resolve('./output'))
+  initPaths(resolve(baseDir), resolve(outputDir))
 }
 
 const config = {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "anymd",
-  "version": "0.0.0",
+  "version": "0.0.2",
   "description": "Convert any document (PDF, DOC, DOCX) to clean Markdown for RAG",
   "keywords": [
     "markdown",

package/scripts/batch-ocr.py

@@ -1,5 +1,6 @@
 import gc
 import json
+import sys
 import time
 from dataclasses import dataclass, field
 from pathlib import Path
@@ -8,11 +9,20 @@ import pypdfium2 as pdfium
 import pypdfium2.raw as pdfium_c
 from PIL import Image
 
-DATA_DIR = Path('data').resolve()
-DATA_FILE = Path('data/classification.json')
-OUTPUT_BASE = Path('output/ocr-raw')
-STATUS_FILE = Path('output/ocr-progress.json')
-LOG_FILE = Path('output/ocr-log.txt')
+
+def _get_arg(flag: str, default: str) -> str:
+  try:
+    idx = sys.argv.index(flag)
+    return sys.argv[idx + 1]
+  except (ValueError, IndexError):
+    return default
+
+
+DATA_DIR = Path(_get_arg('--data-dir', 'data')).resolve()
+DATA_FILE = Path(_get_arg('--classification', 'data/classification.json'))
+OUTPUT_BASE = Path(_get_arg('--output-base', 'output/ocr-raw'))
+STATUS_FILE = Path(_get_arg('--status-file', 'output/ocr-progress.json'))
+LOG_FILE = Path(_get_arg('--log-file', 'output/ocr-log.txt'))
 
 MODEL_ID = 'mlx-community/chandra-8bit'
 IMAGE_DPI = 150

package/src/bootstrap.ts

@@ -1,4 +1,6 @@
 import { existsSync } from 'node:fs'
+import { homedir } from 'node:os'
+import { join } from 'node:path'
 
 import { getPaths } from '~/paths'
 
@@ -7,7 +9,8 @@ interface BootstrapCallbacks {
   onStep: (message: string) => void
 }
 
-const REQUIRED_PACKAGES = ['marker', 'mlx_vlm', 'pypdfium2']
+const REQUIRED_PACKAGES = ['marker', 'markitdown', 'mlx_vlm', 'pypdfium2']
+const PIP_PACKAGES = ['marker-pdf', 'markitdown', 'mlx-vlm', 'pypdfium2']
 
 const checkPackageImportable = async (pythonPath: string, pkg: string): Promise<boolean> => {
   try {
@@ -32,32 +35,51 @@ const runCommand = async (args: string[]): Promise<{ ok: boolean; stderr: string
   return { ok: code === 0, stderr: stderrText }
 }
 
-const createVenv = async (cacheDir: string, venvDir: string, cbs: BootstrapCallbacks): Promise<boolean> => {
-  cbs.onStep('Creating Python virtual environment...')
-  const mkdirResult = await runCommand(['mkdir', '-p', cacheDir])
+const UV_CANDIDATES = ['uv', '/opt/homebrew/bin/uv', join(homedir(), '.local', 'bin', 'uv')]
+
+const findUv = async (): Promise<string | undefined> => {
+  const checks = await Promise.all(
+    UV_CANDIDATES.map(async bin => ({ bin, ok: (await runCommand([bin, '--version'])).ok }))
+  )
+  const found = checks.find(c => c.ok)
+  return found?.bin
+}
+
+interface VenvOpts {
+  cacheDir: string
+  cbs: BootstrapCallbacks
+  uv: string
+  venvDir: string
+}
+
+const createVenv = async (opts: VenvOpts): Promise<boolean> => {
+  opts.cbs.onStep('Creating Python virtual environment via uv...')
+  const mkdirResult = await runCommand(['mkdir', '-p', opts.cacheDir])
   if (!mkdirResult.ok) return false
-  const venvResult = await runCommand(['python3', '-m', 'venv', venvDir])
+  const venvResult = await runCommand([opts.uv, 'venv', '--python', '3.13', opts.venvDir])
   if (!venvResult.ok) {
-    cbs.onStep(`Failed to create venv: ${venvResult.stderr}`)
+    opts.cbs.onStep(`Failed to create venv: ${venvResult.stderr}`)
     return false
   }
-  cbs.onStep('Virtual environment created.')
+  opts.cbs.onStep('Virtual environment created (Python 3.13).')
   return true
 }
 
-const installPackages = async (venvDir: string, cbs: BootstrapCallbacks): Promise<boolean> => {
-  cbs.onStep('Installing marker-pdf, mlx-vlm, pypdfium2...')
-  const pipPath = `${venvDir}/bin/pip`
-  const result = await runCommand([pipPath, 'install', 'marker-pdf', 'mlx-vlm', 'pypdfium2'])
+const installPackages = async (venvDir: string, uv: string, cbs: BootstrapCallbacks): Promise<boolean> => {
+  cbs.onStep(`Installing ${PIP_PACKAGES.join(', ')}...`)
+  const result = await runCommand([uv, 'pip', 'install', '--python', `${venvDir}/bin/python`, ...PIP_PACKAGES])
   if (!result.ok) {
-    cbs.onStep(`pip install failed: ${result.stderr}`)
+    cbs.onStep(`uv pip install failed: ${result.stderr}`)
     return false
   }
   cbs.onStep('Packages installed successfully.')
   return true
 }
 
-const ensureVenvExists = async (cbs: BootstrapCallbacks): Promise<null | { skip: boolean; venvDir: string }> => {
+const ensureVenvExists = async (
+  uv: string,
+  cbs: BootstrapCallbacks
+): Promise<null | { skip: boolean; venvDir: string }> => {
   const { cacheDir, venvPython } = getPaths()
   const venvDir = `${cacheDir}/.venv`
   if (existsSync(venvPython)) {
@@ -66,21 +88,29 @@ const ensureVenvExists = async (cbs: BootstrapCallbacks): Promise<null | { skip:
     cbs.onStep('Some packages missing, installing...')
     return { skip: false, venvDir }
   }
-  const created = await createVenv(cacheDir, venvDir, cbs)
+  const created = await createVenv({ cacheDir, cbs, uv, venvDir })
   return created ? { skip: false, venvDir } : null
 }
 
-const bootstrapPython = async (cbs: BootstrapCallbacks): Promise<boolean> => {
-  const result = await ensureVenvExists(cbs)
+const requireUv = async (cbs: BootstrapCallbacks): Promise<string | undefined> => {
+  const uv = await findUv()
+  if (!uv) cbs.onStep('uv not found. Install it: curl -LsSf https://astral.sh/uv/install.sh | sh')
+  return uv
+}
+
+const setupAndInstall = async (uv: string, cbs: BootstrapCallbacks): Promise<boolean> => {
+  const result = await ensureVenvExists(uv, cbs)
   if (!result) return false
-  if (result.skip) {
-    cbs.onDone()
-    return true
-  }
-  const ok = await installPackages(result.venvDir, cbs)
-  if (!ok) return false
-  cbs.onDone()
-  return true
+  if (result.skip) return true
+  return installPackages(result.venvDir, uv, cbs)
+}
+
+const bootstrapPython = async (cbs: BootstrapCallbacks): Promise<boolean> => {
+  const uv = await requireUv(cbs)
+  if (!uv) return false
+  const ok = await setupAndInstall(uv, cbs)
+  if (ok) cbs.onDone()
+  return ok
 }
 
 export { bootstrapPython }

package/src/preflight.ts

@@ -37,7 +37,7 @@ const checkPythonPackage = async (pythonPath: string, pkg: string): Promise<bool
 }
 
 const checkVenvPackages = async (venvPython: string): Promise<string[]> => {
-  const requiredPackages = ['marker', 'mlx_vlm', 'pypdfium2']
+  const requiredPackages = ['marker', 'markitdown', 'mlx_vlm', 'pypdfium2']
   const results = await Promise.all(
     requiredPackages.map(async pkg => ({ missing: !(await checkPythonPackage(venvPython, pkg)), pkg }))
   )
@@ -67,7 +67,7 @@ const collectToolIssues = (
   if (!tools.hasSoffice) warnings.push('soffice not found \u2014 install LibreOffice for .doc/.docx support')
   if (!tools.hasVenv)
     errors.push(
-      `Python venv not found at ${venvPython} \u2014 run: python3 -m venv .venv && .venv/bin/pip install marker-pdf mlx-vlm pypdfium2`
+      `Python venv not found at ${venvPython} \u2014 install uv (curl -LsSf https://astral.sh/uv/install.sh | sh) and re-run`
     )
   if (!checkFileExists(getPaths().dataDir)) warnings.push('No data directory found \u2014 create it and add documents')
   return { errors, warnings }
@@ -81,9 +81,7 @@ const runPreflight = async (): Promise<PreflightResult> => {
   if (tools.hasVenv) {
     const missing = await checkVenvPackages(venvPython)
     if (missing.length > 0)
-      warnings.push(
-        `Python packages missing: ${missing.join(', ')} \u2014 run: .venv/bin/pip install ${missing.join(' ')}`
-      )
+      warnings.push(`Python packages missing: ${missing.join(', ')} \u2014 re-run anymd to auto-install`)
   }
 
   return { errors, warnings }

package/src/tui-data.ts

@@ -387,9 +387,35 @@ const spawnCommand = (key: CommandKey): null | { args: string[]; label: string;
     classify: null,
     dataset: null,
     enhance: null,
-    ocr: { args: [p.venvPython, join(p.scriptsDir, 'batch-ocr.py')], label: 'Chandra OCR' },
+    ocr: {
+      args: [
+        p.venvPython,
+        join(p.scriptsDir, 'batch-ocr.py'),
+        '--data-dir',
+        p.dataDir,
+        '--classification',
+        p.classification,
+        '--output-base',
+        p.ocrRaw,
+        '--status-file',
+        p.ocrProgress,
+        '--log-file',
+        p.ocrLog
+      ],
+      label: 'Chandra OCR'
+    },
     pipeline: {
-      args: ['bun', 'run', join(packageRoot, 'main.ts'), '--file-list', p.nativeFileList, '--base-dir', p.dataDir],
+      args: [
+        'bun',
+        'run',
+        join(packageRoot, 'main.ts'),
+        '--file-list',
+        p.nativeFileList,
+        '--base-dir',
+        p.dataDir,
+        '--output-dir',
+        p.outputDir
+      ],
       label: 'Doc/PDF \u2192 Markdown pipeline'
     }
   }