anymd 0.0.10 → 0.0.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "anymd",
-  "version": "0.0.10",
+  "version": "0.0.11",
   "description": "Convert any document (PDF, DOC, DOCX) to clean Markdown for RAG",
   "keywords": [
     "markdown",
@@ -46,9 +46,13 @@
   "dependencies": {
     "markdownlint": "^0.40.0",
     "p-map": "^7.0.4",
+    "turndown": "^7.2.2",
     "yoctocolors": "^2.1.2",
     "zod": "^4.3.6"
   },
+  "devDependencies": {
+    "@types/turndown": "^5.0.6"
+  },
   "engines": {
     "bun": ">=1.0.0"
   },

package/scripts/batch-ocr.py

@@ -24,7 +24,7 @@ OUTPUT_BASE = Path(_get_arg('--output-base', 'output/ocr-raw'))
 STATUS_FILE = Path(_get_arg('--status-file', 'output/ocr-progress.json'))
 LOG_FILE = Path(_get_arg('--log-file', 'output/ocr-log.txt'))
 
-MODEL_ID = 'mlx-community/chandra-8bit'
+MODEL_ID = 'mlx-community/chandra-4bit'
 IMAGE_DPI = 150
 MAX_TOKENS = 8192
 

package/src/md-enhancer.ts

@@ -2,12 +2,21 @@
 import { readFile, writeFile } from 'node:fs/promises'
 import { basename, join } from 'node:path'
 import pMap from 'p-map'
+import TurndownService from 'turndown'
 
 import type { CleanResult } from '~/types'
 
 import { ensureDir, loadExistingMdFiles, logger } from '~/utils'
 
-const BOLD_LINE_REGEX = /^\*\*(?<content>.+)\*\*$/u,
+const td = new TurndownService({ bulletListMarker: '-', emDelimiter: '*', headingStyle: 'atx' })
+td.remove('style')
+const turndown = td,
+  HTML_DETECT_REGEX = /<\/?[a-z][a-z0-9]*[^>]*>/iu,
+  stripHtml = (text: string): string => {
+    if (!HTML_DETECT_REGEX.test(text)) return text
+    return turndown.turndown(text)
+  },
+  BOLD_LINE_REGEX = /^\*\*(?<content>.+)\*\*$/u,
   PHAN_REGEX = /^(?:Phần|PHẦN)\s+/u,
   CHUONG_REGEX = /^(?:Chương|CHƯƠNG)\s+/u,
   MUC_REGEX = /^(?:Mục|MỤC)\s+\d/u,
@@ -18,6 +27,7 @@ const BOLD_LINE_REGEX = /^\*\*(?<content>.+)\*\*$/u,
   HEADER_TABLE_LINE_REGEX = /^\|.*\|$/u,
   TABLE_SEP_REGEX = /^\|\s*-+/u,
   MULTIPLE_BLANKS_REGEX = /\n{3,}/gu,
+  MULTIPLE_SPACES_REGEX = / {2,}/gu,
   // oxlint-disable-next-line no-control-regex
   // eslint-disable-next-line no-control-regex
   CONTROL_CHARS_REGEX = /[\u0000-\u0008\u000B\u000C\u000E-\u001F]/gu,
@@ -85,9 +95,11 @@ const BOLD_LINE_REGEX = /^\*\*(?<content>.+)\*\*$/u,
     return enhanced
   },
   enhanceMarkdown = (text: string): string => {
-    const enhanced = processLines(text.split('\n'))
+    const cleaned = stripHtml(text)
+    const enhanced = processLines(cleaned.split('\n'))
     let output = enhanced.join('\n')
     output = output.replace(MULTIPLE_BLANKS_REGEX, '\n\n')
+    output = output.replace(MULTIPLE_SPACES_REGEX, ' ')
     output = output.trim()
     return output
   },