anymd 0.0.6 → 0.0.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "anymd",
-  "version": "0.0.6",
+  "version": "0.0.8",
   "description": "Convert any document (PDF, DOC, DOCX) to clean Markdown for RAG",
   "keywords": [
     "markdown",

package/scripts/pdf-to-md.py

@@ -6,14 +6,30 @@ from pathlib import Path
 from marker.converters.pdf import PdfConverter
 from marker.models import create_model_dict
 from marker.output import text_from_rendered
+from markitdown import MarkItDown
 
 MIN_ARGS = 2
+MIN_FALLBACK_CHARS = 10
+
+_mid = MarkItDown()
 
 
 def _emit(data: dict[str, object]) -> None:
   print(json.dumps(data), flush=True)
 
 
+def _markitdown_fallback(pdf_path: str) -> str | None:
+  try:
+    result = _mid.convert(pdf_path)
+    text = result.text_content.strip()
+  except Exception:  # noqa: BLE001
+    return None
+  else:
+    if len(text) < MIN_FALLBACK_CHARS:
+      return None
+    return text
+
+
 def _convert_one(converter: PdfConverter, pdf_path: str, out_path: str, index: int, total: int) -> None:
   t1 = time.time()
   try:
@@ -29,15 +45,28 @@ def _convert_one(converter: PdfConverter, pdf_path: str, out_path: str, index: i
       'seconds': round(time.time() - t1, 1),
       'chars': len(md),
     })
-  except Exception as exc:  # noqa: BLE001
-    _emit({
-      'type': 'error',
-      'index': index,
-      'total': total,
-      'file': Path(pdf_path).name,
-      'seconds': round(time.time() - t1, 1),
-      'error': str(exc),
-    })
+  except Exception as marker_exc:  # noqa: BLE001
+    fallback_md = _markitdown_fallback(pdf_path)
+    if fallback_md:
+      Path(out_path).parent.mkdir(parents=True, exist_ok=True)
+      Path(out_path).write_text(fallback_md, encoding='utf-8')
+      _emit({
+        'type': 'converted',
+        'index': index,
+        'total': total,
+        'file': Path(pdf_path).name,
+        'seconds': round(time.time() - t1, 1),
+        'chars': len(fallback_md),
+      })
+    else:
+      _emit({
+        'type': 'error',
+        'index': index,
+        'total': total,
+        'file': Path(pdf_path).name,
+        'seconds': round(time.time() - t1, 1),
+        'error': str(marker_exc),
+      })
 
 
 def main() -> None:

package/src/bootstrap.ts

@@ -10,7 +10,7 @@ interface BootstrapCallbacks {
 }
 
 const REQUIRED_PACKAGES = ['marker', 'markitdown', 'mammoth', 'mlx_vlm', 'pypdfium2', 'torchvision']
-const PIP_PACKAGES = ['marker-pdf', 'markitdown[docx]', 'mlx-vlm', 'pypdfium2', 'torchvision']
+const PIP_PACKAGES = ['marker-pdf', 'markitdown[docx,pdf]', 'mlx-vlm', 'pypdfium2', 'torchvision']
 const CHANDRA_MODEL_ID = 'mlx-community/chandra-8bit'
 
 const checkImportable = async (py: string, pkg: string): Promise<boolean> => {

package/tui.tsx

@@ -407,6 +407,7 @@ const SidebarStep = ({
 }
 
 const ERROR_PATTERN = /\b(?:ERROR|Error:|Failed:|failed|FAILED|\u2716|exception|traceback)/iu
+const NOISY_PATTERN = /index \d+ is out of bounds/iu
 
 const PreflightBanner = ({ errors, warnings }: { errors: string[]; warnings: string[] }) => {
   if (errors.length === 0 && warnings.length === 0) return null
@@ -772,7 +773,8 @@ const App = () => {
         procRef.current = spawned.proc
 
         const onLine = (line: string): void => {
-          dispatch({ line, type: 'APPEND_OUTPUT' })
+          const noisy = NOISY_PATTERN.test(line)
+          if (!noisy) dispatch({ line, type: 'APPEND_OUTPUT' })
           if (key === 'pipeline') appendPipelineLog(line)
           if (ERROR_PATTERN.test(line)) appendErrorLog(key, line)
         }