@geminilight/mindos 0.6.59 → 0.6.61

package/_standalone/scripts/extract-pdf.cjs

@@ -13,7 +13,6 @@
 'use strict';
 
 const fs = require('fs');
-const path = require('path');
 
 // Suppress pdfjs-dist warnings (loadFont etc.) that go to stdout/stderr
 // and would corrupt our JSON output.
@@ -22,6 +21,115 @@ const _log = console.log;
 console.warn = () => {};
 console.log = () => {};
 
+// ---------------------------------------------------------------------------
+// CJK-aware smart text joining
+// ---------------------------------------------------------------------------
+
+/**
+ * Detect whether a character falls in CJK / fullwidth ranges where
+ * inter-item spaces are NOT desired.
+ */
+function isCJK(ch) {
+  const c = ch.codePointAt(0);
+  return (
+    (c >= 0x2E80 && c <= 0x2FDF) ||   // CJK Radicals Supplement, Kangxi Radicals
+    (c >= 0x4E00 && c <= 0x9FFF) ||   // CJK Unified Ideographs
+    (c >= 0x3400 && c <= 0x4DBF) ||   // CJK Extension A
+    (c >= 0xF900 && c <= 0xFAFF) ||   // CJK Compatibility Ideographs
+    (c >= 0x3040 && c <= 0x309F) ||   // Hiragana
+    (c >= 0x30A0 && c <= 0x30FF) ||   // Katakana
+    (c >= 0xAC00 && c <= 0xD7AF) ||   // Hangul Syllables
+    (c >= 0x20000 && c <= 0x2FA1F)    // CJK Extensions B-F + Supplement
+  );
+}
+
+/** True for CJK fullwidth punctuation that already occupies its own space. */
+function isCJKPunct(ch) {
+  const c = ch.codePointAt(0);
+  return (
+    (c >= 0x3000 && c <= 0x303F) ||    // 。、「」【】
+    (c >= 0xFF01 && c <= 0xFF0F) ||    // ！＂＃＄
+    (c >= 0xFF1A && c <= 0xFF20) ||    // ：；＜＝＞
+    (c >= 0xFF3B && c <= 0xFF40) ||    // ［＼］
+    (c >= 0xFF5B && c <= 0xFF65) ||    // ｛｜｝～
+    (c >= 0xFE50 && c <= 0xFE6B)      // Small Form Variants
+  );
+}
+
+/**
+ * Decide whether a space should be inserted between two adjacent characters
+ * that come from separate pdfjs-dist TextItem objects.
+ *
+ * Rules:
+ *   CJK + CJK  → no space  (中文相邻不加)
+ *   CJK + Punct → no space  (标点紧跟)
+ *   Punct + CJK → no space
+ *   CJK + Latin → space     (中英之间加)
+ *   Latin + CJK → space
+ *   Latin + Latin → space    (英文单词间加)
+ *   Punct + Latin → no space (e.g. `(hello`)
+ *   Latin + Punct → no space (e.g. `hello.`)
+ */
+function shouldAddSpace(left, right) {
+  if (!left || !right) return false;
+
+  const lCJK = isCJK(left);
+  const rCJK = isCJK(right);
+  const lPunct = isCJKPunct(left) || /^[^\w\s]$/.test(left);
+  const rPunct = isCJKPunct(right) || /^[^\w\s]$/.test(right);
+
+  // Any side is punctuation → no space
+  if (lPunct || rPunct) return false;
+  // Both CJK → no space
+  if (lCJK && rCJK) return false;
+  // Mixed CJK/Latin → space
+  if (lCJK !== rCJK) return true;
+  // Both non-CJK (Latin/digit) → space between words
+  return true;
+}
+
+/**
+ * Join pdfjs-dist TextItem[] into clean text, respecting CJK spacing
+ * rules and `hasEOL` line-break markers.
+ */
+function smartJoinTextItems(items) {
+  const lines = [];
+  let cur = '';
+
+  for (const item of items) {
+    const str = item.str ?? '';
+
+    if (str === '') {
+      // Empty str with hasEOL → line break
+      if (item.hasEOL && cur.trim()) {
+        lines.push(cur.trim());
+        cur = '';
+      }
+      continue;
+    }
+
+    if (cur === '') {
+      cur = str;
+    } else {
+      const left = cur[cur.length - 1];
+      const right = str[0];
+      cur += shouldAddSpace(left, right) ? ' ' + str : str;
+    }
+
+    if (item.hasEOL) {
+      if (cur.trim()) lines.push(cur.trim());
+      cur = '';
+    }
+  }
+
+  if (cur.trim()) lines.push(cur.trim());
+  return lines.join('\n');
+}
+
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+
 async function main() {
   const pdfPath = process.argv[2];
   if (!pdfPath || !fs.existsSync(pdfPath)) {
@@ -43,8 +151,7 @@ async function main() {
   for (let i = 1; i <= numPages; i++) {
     const page = await doc.getPage(i);
     const content = await page.getTextContent();
-    const line = content.items.map((item) => item.str ?? '').join(' ');
-    pageTexts.push(line);
+    pageTexts.push(smartJoinTextItems(content.items));
   }
 
   process.stdout.write(JSON.stringify({ text: pageTexts.join('\n\n'), pages: numPages }));