anymd 0.0.0

package/README.md

@@ -0,0 +1,139 @@
+# anymd
+
+Convert any document (PDF, DOC, DOCX) to clean Markdown for RAG. macOS Apple Silicon only.
+
+## Install
+
+```bash
+bunx anymd --input-dir ./my-documents
+```
+
+On first run, `anymd` automatically sets up a Python virtual environment in `~/.cache/anymd/` and installs all required Python packages (marker-pdf, mlx-vlm, pypdfium2). Subsequent runs start instantly.
+
+## Usage
+
+```bash
+bunx anymd --input-dir <path> [--output-dir <path>] [--config <path>]
+```
+
+| Flag | Required | Default | Description |
+|------|----------|---------|-------------|
+| `--input-dir` | Yes | — | Directory containing documents (any nested structure) |
+| `--output-dir` | No | `./output` | Where to write all output files |
+| `--config` | No | `./config.json` | Path to configuration file |
+
+Launches a TUI that auto-runs all 5 pipeline steps with live per-file progress. On completion: rings terminal bell, sends macOS notification, and updates the terminal title. Safe to Ctrl+C — progress is saved, re-run to resume.
+
+## Pipeline Steps
+
+| Step | What | Tool | Output |
+|------|------|------|--------|
+| 1. Classify | Detect native/scanned/mixed PDFs | pdftotext (TypeScript) | `<input-dir>/classification.json` |
+| 2. Convert | Doc/docx/native PDF → raw markdown | soffice + markitdown + marker | `<output-dir>/raw-md/` |
+| 3. OCR | Scanned/mixed PDF → markdown | mlx-vlm chandra-8bit | `<output-dir>/ocr-raw/` |
+| 4. Enhance | Heading detection + cleanup | TypeScript | `<output-dir>/markdown/` |
+| 5. Dataset | Collect all markdown → JSONL | TypeScript | `<output-dir>/dataset/dataset.jsonl` |
+
+Steps 2 and 3 run in parallel when both are needed.
+
+## TUI Hotkeys
+
+| Key | Action |
+|-----|--------|
+| R | Retry failed step |
+| S | Skip failed step |
+| L | Toggle log overlay (pipeline + OCR, scrollable) |
+| ↑↓ | Scroll log overlay |
+| Q / ESC | Quit |
+
+## System Requirements
+
+- macOS with Apple Silicon (64GB recommended for OCR)
+- [Bun](https://bun.sh) runtime
+- [poppler](https://poppler.freedesktop.org/) (`pdftotext`) — `brew install poppler`
+- [LibreOffice](https://www.libreoffice.org/) (`soffice`) — optional, only for `.doc` files
+- Python 3.13+ — venv and packages are auto-installed on first run
+
+The TUI preflight check tells you exactly what's missing.
+
+## Auto-Bootstrap
+
+On first run, `anymd` creates `~/.cache/anymd/.venv` and installs:
+- `marker-pdf` — PDF to markdown conversion
+- `mlx-vlm` — Apple Silicon MLX inference for OCR
+- `pypdfium2` — PDF page rendering
+
+This takes ~2 minutes. Progress is shown in the TUI. Subsequent runs detect the existing venv and skip setup.
+
+## Configuration
+
+Create a `config.json` in your working directory (or pass `--config <path>`):
+
+| Key | Default | Description |
+|-----|---------|-------------|
+| `classifyBatchSize` | 20 | PDFs classified per batch |
+| `datasetConcurrency` | 50 | Parallel file reads during dataset build |
+| `enhanceConcurrency` | 10 | Parallel markdown enhancement workers |
+| `markerWorkers` | 3 | Parallel marker-pdf processes for PDF → markdown |
+| `minTextLength` | 50 | Minimum characters for a document to be included in dataset |
+| `nativeThreshold` | 200 | Alpha chars above which a PDF is classified as native |
+| `scannedThreshold` | 50 | Alpha chars below which a PDF is classified as scanned |
+
+All fields are optional — omitted fields use defaults. No config file = all defaults.
+
+## Notifications
+
+- **Terminal title** updates with current step, progress percentage, and parallel status
+- **Terminal bell** rings on pipeline completion
+- **macOS notification** fires on completion via `osascript`
+- **Error log** written to `<output-dir>/errors.log` with timestamps — persists across all steps
+
+## Output Structure
+
+```
+<output-dir>/
+├── raw-md/                      Raw markdown from conversion (step 2)
+├── ocr-raw/                     OCR markdown from scanned PDFs (step 3)
+├── markdown/                    Final enhanced markdown (step 4)
+├── dataset/dataset.jsonl        JSONL dataset for RAG (step 5)
+├── pipeline-log.txt             Pipeline conversion log
+├── ocr-log.txt                  OCR processing log
+└── errors.log                   Timestamped error log (all steps)
+```
+
+## Input Requirements
+
+- Put documents anywhere inside `--input-dir` in any folder structure
+- Supports `.doc`, `.docx`, `.pdf` (native, scanned, mixed)
+- Output files use flat naming with `--` separator: `docs/foo/bar/doc.pdf` → `foo--bar--doc.md`
+
+## Dataset Deduplication
+
+Step 5 deduplicates entries by content hash. If two source files produce identical markdown, only one entry appears in the JSONL. The TUI completion summary shows the dedup count.
+
+## Resume Support
+
+All steps support resume:
+
+- Classify: re-runs if `classification.json` missing
+- Convert: skips files already in `raw-md/`
+- OCR: skips files already in `ocr-raw/`
+- Enhance: skips files already in `markdown/`
+- Dataset: always regenerates from `markdown/`
+
+## OCR Details
+
+Native PDFs use marker-pdf for structured markdown extraction (headings, bold, lists). Scanned/mixed PDFs use `mlx-community/chandra-8bit` via mlx-vlm at 150 DPI (~32s per page on Apple Silicon). Chandra was chosen for superior Vietnamese diacritical accuracy over marker's surya OCR. Mixed PDFs use native text for text-heavy pages and OCR for scanned pages.
+
+## Development
+
+```bash
+bun run doc                    # Run locally (uses ./data as input)
+bun test                       # 73 unit tests
+bun fix                        # TypeScript linting (oxlint + eslint + biome + tsc)
+ruff format && ruff check --fix  # Python linting
+```
+
+## License
+
+MIT

package/cli.tsx

@@ -0,0 +1,42 @@
+#!/usr/bin/env bun
+import { resolve } from 'node:path'
+
+import { loadConfig } from '~/config'
+import { initPaths } from '~/paths'
+
+const printUsage = (): void => {
+  const text = `Usage: anymd --input-dir <path> [--output-dir <path>] [--config <path>]
+
+Options:
+  --input-dir   Input directory containing documents (required)
+  --output-dir  Output directory (default: ./output)
+  --config      Path to config.json (default: ./config.json)`
+  process.stdout.write(`${text}\n`)
+}
+
+const getArgValue = (argv: string[], flag: string): string | undefined => {
+  const idx = argv.indexOf(flag)
+  return idx === -1 ? undefined : argv[idx + 1]
+}
+
+const parseArgs = (argv: string[]): { configPath?: string; inputDir?: string; outputDir: string } => ({
+  configPath: getArgValue(argv, '--config'),
+  inputDir: getArgValue(argv, '--input-dir'),
+  outputDir: getArgValue(argv, '--output-dir') ?? './output'
+})
+
+const args = parseArgs(process.argv)
+
+if (!args.inputDir) {
+  printUsage()
+  process.exit(1)
+}
+
+const resolvedInput = resolve(args.inputDir)
+const resolvedOutput = resolve(args.outputDir)
+
+initPaths(resolvedInput, resolvedOutput)
+loadConfig(args.configPath)
+
+const { start } = await import('./tui')
+await start()

package/main.ts

@@ -0,0 +1,62 @@
+import { error, log } from 'node:console'
+import { readFile } from 'node:fs/promises'
+import { resolve } from 'node:path'
+
+import { getPaths, initPaths } from '~/paths'
+import { runPipeline } from '~/pipeline'
+import { logger } from '~/utils'
+
+const fileListIdx = process.argv.indexOf('--file-list'),
+  baseDirIdx = process.argv.indexOf('--base-dir')
+let inputDir = process.argv[2] ?? './data'
+let inputFiles: string[] | undefined
+let baseDir = baseDirIdx === -1 ? '' : (process.argv[baseDirIdx + 1] ?? resolve('data'))
+
+if (fileListIdx !== -1) {
+  const listPath = process.argv[fileListIdx + 1]
+  if (listPath) {
+    const content = await readFile(listPath, 'utf8')
+    inputFiles = content
+      .trim()
+      .split('\n')
+      .filter(l => l.length > 0)
+    inputDir = '.'
+  }
+}
+
+if (!baseDir) baseDir = inputDir
+
+try {
+  getPaths()
+} catch {
+  initPaths(resolve(baseDir), resolve('./output'))
+}
+
+const config = {
+  baseDir,
+  concurrency: 10,
+  inputDir,
+  inputFiles,
+  outputDir: getPaths().outputDir
+}
+
+try {
+  const stats = await runPipeline(config)
+  logger.summary({
+    'Converted to MD': stats.converted,
+    'Convert Failed': stats.convertFailed,
+    Enhanced: stats.enhanced,
+    'Total Characters': stats.totalCharacters.toLocaleString(),
+    'Total Input Files': stats.totalFiles
+  })
+  if (stats.errors.length > 0) {
+    log('\n[!] Errors encountered:')
+    for (const pipelineError of stats.errors.slice(0, 10)) log(`  - ${pipelineError}`)
+    if (stats.errors.length > 10) log(`  ... and ${stats.errors.length - 10} more`)
+  }
+  log('\n[DONE] Pipeline complete!')
+  log(`[>] Markdown files saved to: ${getPaths().markdown}`)
+} catch (pipelineError) {
+  error('Pipeline failed:', pipelineError)
+  process.exit(1)
+}

package/package.json

@@ -0,0 +1,67 @@
+{
+  "name": "anymd",
+  "version": "0.0.0",
+  "description": "Convert any document (PDF, DOC, DOCX) to clean Markdown for RAG",
+  "keywords": [
+    "markdown",
+    "pdf",
+    "ocr",
+    "document",
+    "converter",
+    "rag",
+    "mlx",
+    "apple-silicon"
+  ],
+  "homepage": "https://github.com/1qh/anymd#readme",
+  "bugs": {
+    "url": "https://github.com/1qh/anymd/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/1qh/anymd.git"
+  },
+  "license": "MIT",
+  "bin": {
+    "anymd": "cli.tsx"
+  },
+  "files": [
+    "cli.tsx",
+    "main.ts",
+    "tui.tsx",
+    "src/",
+    "!src/__tests__/",
+    "scripts/",
+    "tsconfig.json"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "clean": "git clean -xdf .cache .turbo dist node_modules",
+    "doc": "bun cli.tsx --input-dir ./data --output-dir ./output",
+    "lint": "eslint",
+    "start": "bun run main.ts",
+    "test": "bun test",
+    "typecheck": "tsc --noEmit",
+    "with-env": "dotenv -e ../../.env --"
+  },
+  "dependencies": {
+    "@opentui/core": "^0.1.77",
+    "@opentui/react": "^0.1.77",
+    "markdownlint": "^0.40.0",
+    "p-map": "^7.0.4",
+    "react": "^19.2.4",
+    "yoctocolors": "^2.1.2",
+    "zod": "^4.3.6"
+  },
+  "devDependencies": {
+    "@types/react": "^19.2.13"
+  },
+  "engines": {
+    "bun": ">=1.0.0"
+  },
+  "os": [
+    "darwin"
+  ],
+  "cpu": [
+    "arm64"
+  ]
+}

package/scripts/batch-ocr.py

@@ -0,0 +1,300 @@
+import gc
+import json
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import pypdfium2 as pdfium
+import pypdfium2.raw as pdfium_c
+from PIL import Image
+
+DATA_DIR = Path('data').resolve()
+DATA_FILE = Path('data/classification.json')
+OUTPUT_BASE = Path('output/ocr-raw')
+STATUS_FILE = Path('output/ocr-progress.json')
+LOG_FILE = Path('output/ocr-log.txt')
+
+MODEL_ID = 'mlx-community/chandra-8bit'
+IMAGE_DPI = 150
+MAX_TOKENS = 8192
+
+
+def _to_output_name(pdf_path: str) -> str:
+  try:
+    rel = Path(pdf_path).resolve().relative_to(DATA_DIR)
+  except ValueError:
+    return Path(pdf_path).stem
+  return str(rel.with_suffix('')).replace('/', '--')
+
+
+OCR_PROMPT = """
+OCR this image to markdown.
+Output clean, readable markdown preserving the document structure.
+Use proper headings, lists, tables, and formatting.
+For math expressions, use LaTeX notation with $ delimiters.
+""".strip()
+
+
+def _log(msg: str) -> None:
+  line = f'[{time.strftime("%Y-%m-%d %H:%M:%S")}] {msg}'
+  print(line, flush=True)
+  with LOG_FILE.open('a', encoding='utf-8') as f:
+    f.write(line + '\n')
+
+
+def _format_duration(seconds: float) -> str:
+  h = int(seconds // 3600)
+  m = int((seconds % 3600) // 60)
+  if h > 0:
+    return f'{h}h{m:02d}m'
+  return f'{m}m{int(seconds % 60):02d}s'
+
+
+@dataclass
+class Progress:
+  done: int
+  total: int
+  errors: int
+  current_file: str
+  current_page: str
+  current_pages_total: int
+  current_file_started: float
+  elapsed: float
+  avg_per_file: float
+  recent_files: list[dict[str, object]] = field(default_factory=list)
+
+
+def _save_progress(p: Progress) -> None:
+  remaining = p.total - p.done
+  eta_seconds = remaining * p.avg_per_file if p.avg_per_file > 0 else 0
+
+  progress = {
+    'done': p.done,
+    'total': p.total,
+    'errors': p.errors,
+    'pct': round(p.done / max(p.total, 1) * 100, 1),
+    'current_file': p.current_file,
+    'current_page': p.current_page,
+    'current_pages_total': p.current_pages_total,
+    'current_file_started': p.current_file_started,
+    'elapsed': _format_duration(p.elapsed),
+    'avg_per_file': f'{p.avg_per_file:.0f}s',
+    'eta': _format_duration(eta_seconds),
+    'eta_hours': round(eta_seconds / 3600, 1),
+    'updated': time.strftime('%Y-%m-%d %H:%M:%S'),
+    'recent_files': p.recent_files[-10:],
+  }
+
+  STATUS_FILE.write_text(json.dumps(progress, indent=2) + '\n', encoding='utf-8')
+
+
+def _free_memory() -> None:
+  gc.collect()
+
+
+def _flatten_page(page: object) -> None:
+  pdfium_c.FPDFPage_Flatten(page, pdfium_c.FLAT_NORMALDISPLAY)
+
+
+NATIVE_TEXT_THRESHOLD = 50
+
+
+def _render_page(doc: pdfium.PdfDocument, page_idx: int) -> tuple[Image.Image | None, str]:
+  page_obj = doc[page_idx]
+  text = page_obj.get_textpage().get_text_range()
+  alpha_count = sum(1 for c in text if c.isalpha())
+
+  if alpha_count >= NATIVE_TEXT_THRESHOLD:
+    return None, text.strip()
+
+  min_dim = min(page_obj.get_width(), page_obj.get_height())
+  scale_dpi = max((1024 / min_dim) * 72, IMAGE_DPI)
+  _flatten_page(page_obj)
+  page_obj = doc[page_idx]
+  pil_image: Image.Image = page_obj.render(scale=scale_dpi / 72).to_pil().convert('RGB')
+  return pil_image, ''
+
+
+def _ocr_page(
+  model: object,
+  processor: object,
+  config: dict[str, object],
+  image: Image.Image,
+) -> str:
+  from mlx_vlm import generate  # noqa: PLC0415
+  from mlx_vlm.prompt_utils import apply_chat_template  # noqa: PLC0415
+
+  formatted = apply_chat_template(processor, config, OCR_PROMPT, num_images=1)
+  result = generate(
+    model,  # type: ignore[arg-type]
+    processor,  # type: ignore[arg-type]
+    formatted,  # type: ignore[arg-type]
+    image,  # type: ignore[arg-type]
+    max_tokens=MAX_TOKENS,
+    temperature=0.0,
+    verbose=False,
+  )
+  return result.text
+
+
+def _ocr_one_file(  # noqa: PLR0913, PLR0917
+  pdf: str,
+  model: object,
+  processor: object,
+  config: dict[str, object],
+  done_count: int,
+  idx: int,
+  pending_len: int,
+  total: int,
+  errors: int,
+  avg: float,
+  pipeline_start: float,
+  recent_files: list[dict[str, object]],
+) -> int:
+  unique_name = _to_output_name(pdf)
+  display_name = Path(pdf).stem
+  file_start = time.time()
+  doc = pdfium.PdfDocument(pdf)
+  try:
+    doc.init_forms()
+    num_pages = len(doc)
+    _log(f'[{idx}/{pending_len}] ({done_count + idx}/{total}) OCR {display_name} ({num_pages}p)')
+    all_markdown: list[str] = []
+
+    for p_idx in range(num_pages):
+      _save_progress(
+        Progress(
+          done=done_count + idx - 1,
+          total=total,
+          errors=errors,
+          current_file=display_name,
+          current_page=f'{p_idx + 1}/{num_pages}',
+          current_pages_total=num_pages,
+          current_file_started=file_start,
+          elapsed=time.time() - pipeline_start,
+          avg_per_file=avg,
+          recent_files=recent_files,
+        )
+      )
+
+      page_t = time.time()
+      image, native_text = _render_page(doc, p_idx)
+
+      if image is not None:
+        md = _ocr_page(model, processor, config, image)
+        image.close()
+        tag = 'ocr'
+      else:
+        md = native_text
+        tag = 'txt'
+
+      all_markdown.append(md)
+      _log(f'  p{p_idx + 1}/{num_pages} [{tag}] {time.time() - page_t:.0f}s ({len(md)} chars)')
+
+      if p_idx % 5 == 0:
+        _free_memory()
+
+    md_path = OUTPUT_BASE / f'{unique_name}.md'
+    md_path.write_text('\n\n'.join(all_markdown), encoding='utf-8')
+
+    del all_markdown
+    _free_memory()
+    return num_pages
+  finally:
+    doc.close()
+
+
+def _run_ocr() -> None:  # noqa: PLR0914
+  with DATA_FILE.open(encoding='utf-8') as f:
+    data = json.load(f)
+
+  files = data['files']['scanned'] + data['files']['mixed']
+  OUTPUT_BASE.mkdir(parents=True, exist_ok=True)
+
+  done_count = 0
+  pending = []
+  for pdf in files:
+    unique_name = _to_output_name(pdf)
+    if (OUTPUT_BASE / f'{unique_name}.md').exists():
+      done_count += 1
+    else:
+      pending.append(pdf)
+
+  total = len(files)
+  _log(f'Total: {total}, Already done: {done_count}, Pending: {len(pending)}')
+
+  if not pending:
+    _log('Nothing to OCR.')
+    _save_progress(Progress(total, total, 0, '-', '-', 0, 0, 0, 0, []))
+    return
+
+  _log(f'Loading MLX model {MODEL_ID}...')
+  t0 = time.time()
+  from mlx_vlm import load  # noqa: PLC0415
+  from mlx_vlm.utils import load_config  # noqa: PLC0415
+
+  model, processor = load(MODEL_ID)
+  config = load_config(MODEL_ID)
+  _log(f'Model loaded in {time.time() - t0:.0f}s')
+
+  errors = 0
+  file_times: list[float] = []
+  recent_files: list[dict[str, object]] = []
+  pipeline_start = time.time()
+
+  for i, pdf in enumerate(pending, 1):
+    unique_name = _to_output_name(pdf)
+
+    if (OUTPUT_BASE / f'{unique_name}.md').exists():
+      done_count += 1
+    else:
+      t1 = time.time()
+      avg = sum(file_times) / len(file_times) if file_times else 60
+
+      try:
+        num_pages = _ocr_one_file(
+          pdf, model, processor, config, done_count, i, len(pending), total, errors, avg, pipeline_start, recent_files
+        )
+        elapsed = time.time() - t1
+        file_times.append(elapsed)
+        avg = sum(file_times) / len(file_times)
+        display_name = Path(pdf).stem
+        recent_files.append({
+          'name': display_name,
+          'pages': num_pages,
+          'duration': round(elapsed, 1),
+          'per_page': round(elapsed / max(num_pages, 1), 1),
+        })
+        _log(f'  done {elapsed:.0f}s ({elapsed / max(num_pages, 1):.0f}s/p) avg={avg:.0f}s/file')
+        _save_progress(
+          Progress(
+            done_count + i,
+            total,
+            errors,
+            '-',
+            '-',
+            0,
+            0,
+            time.time() - pipeline_start,
+            avg,
+            recent_files,
+          )
+        )
+      except Exception as ocr_err:  # noqa: BLE001
+        errors += 1
+        _log(f'  ERROR {Path(pdf).stem}: {ocr_err}')
+        _free_memory()
+
+  total_time = _format_duration(time.time() - pipeline_start)
+  _log(f'OCR complete. Done: {done_count + len(pending)}, Errors: {errors}, Time: {total_time}')
+
+
+def main() -> None:
+  OUTPUT_BASE.mkdir(parents=True, exist_ok=True)
+  _log('=== Batch OCR Start (MLX) ===')
+  _run_ocr()
+  _log('=== OCR DONE ===')
+
+
+if __name__ == '__main__':
+  main()

package/scripts/docx-to-md.py

@@ -0,0 +1,66 @@
+import json
+import sys
+import time
+from pathlib import Path
+
+from markitdown import MarkItDown
+
+MIN_ARGS = 2
+
+
+def _emit(data: dict[str, object]) -> None:
+  print(json.dumps(data), flush=True)
+
+
+def _convert_one(converter: MarkItDown, input_path: str, out_path: str, index: int, total: int) -> None:
+  t1 = time.time()
+  try:
+    result = converter.convert(input_path)
+    md = result.text_content
+    Path(out_path).parent.mkdir(parents=True, exist_ok=True)
+    Path(out_path).write_text(md, encoding='utf-8')
+    _emit({
+      'type': 'converted',
+      'index': index,
+      'total': total,
+      'file': Path(input_path).name,
+      'seconds': round(time.time() - t1, 1),
+      'chars': len(md),
+    })
+  except Exception as exc:  # noqa: BLE001
+    _emit({
+      'type': 'error',
+      'index': index,
+      'total': total,
+      'file': Path(input_path).name,
+      'seconds': round(time.time() - t1, 1),
+      'error': str(exc),
+    })
+
+
+def main() -> None:
+  if len(sys.argv) < MIN_ARGS:
+    print('Usage: docx-to-md.py <manifest.json>', file=sys.stderr)
+    sys.exit(1)
+
+  manifest_path = sys.argv[1]
+  manifest: list[dict[str, str]] = json.loads(Path(manifest_path).read_text('utf-8'))
+
+  if not manifest:
+    _emit({'type': 'done', 'total': 0})
+    return
+
+  t0 = time.time()
+  _emit({'type': 'loading'})
+  converter = MarkItDown()
+  _emit({'type': 'loaded', 'seconds': round(time.time() - t0, 1)})
+
+  total = len(manifest)
+  for i, entry in enumerate(manifest):
+    _convert_one(converter, entry['input'], entry['output'], i, total)
+
+  _emit({'type': 'done', 'total': total})
+
+
+if __name__ == '__main__':
+  main()