@geminilight/mindos 0.6.58 → 0.6.60

package/_standalone/lib/stores/locale-store.ts

@@ -22,23 +22,37 @@ function getLocaleSnapshot(): Locale {
   return navigator.language.startsWith('zh') ? 'zh' : 'en';
 }
 
+/**
+ * Read the locale that the inline <script> in layout.tsx already resolved.
+ * Returns 'en' to match SSR default — the LocaleStoreInit component
+ * reconciles to the real locale synchronously before first commit.
+ */
+function getPreHydrateLocale(): Locale {
+  return 'en';
+}
+
+const initialLocale = getPreHydrateLocale();
+
 export const useLocaleStore = create<LocaleStoreState>((set) => ({
-  locale: 'en',
-  t: messages['en'] as unknown as Messages,
+  locale: initialLocale,
+  t: messages[initialLocale] as unknown as Messages,
 
   setLocale: (l: Locale) => {
     document.cookie = `locale=${l};path=/;max-age=31536000;SameSite=Lax`;
     document.documentElement.lang = l === 'zh' ? 'zh' : 'en';
+    (window as any).__mindos_locale__ = l;
     set({ locale: l, t: messages[l] as unknown as Messages });
     window.dispatchEvent(new Event('mindos-locale-change'));
   },
 
   _init: (ssrLocale: Locale) => {
-    // Hydrate with client value (or SSR fallback)
-    const clientLocale = typeof window !== 'undefined' ? getLocaleSnapshot() : ssrLocale;
-    set({ locale: clientLocale, t: messages[clientLocale] as unknown as Messages });
+    // Reconcile: if client localStorage disagrees with current store, update once
+    const clientLocale = getLocaleSnapshot();
+    const current = useLocaleStore.getState().locale;
+    if (clientLocale !== current) {
+      set({ locale: clientLocale, t: messages[clientLocale] as unknown as Messages });
+    }
 
-    // Listen for external changes (e.g. localStorage change in another tab, or pre-hydration script)
     const handler = () => {
       const l = getLocaleSnapshot();
       set({ locale: l, t: messages[l] as unknown as Messages });

package/_standalone/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "wiki-app",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "wiki-app",
-      "version": "0.1.0",
+      "version": "0.1.1",
       "hasInstallScript": true,
       "dependencies": {
         "@base-ui/react": "^1.2.0",

package/_standalone/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wiki-app",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "private": true,
   "scripts": {
     "generate": "npx tsx scripts/generate-explore.ts",

package/_standalone/scripts/extract-pdf.cjs

@@ -13,7 +13,6 @@
 'use strict';
 
 const fs = require('fs');
-const path = require('path');
 
 // Suppress pdfjs-dist warnings (loadFont etc.) that go to stdout/stderr
 // and would corrupt our JSON output.
@@ -22,6 +21,115 @@ const _log = console.log;
 console.warn = () => {};
 console.log = () => {};
 
+// ---------------------------------------------------------------------------
+// CJK-aware smart text joining
+// ---------------------------------------------------------------------------
+
+/**
+ * Detect whether a character falls in CJK / fullwidth ranges where
+ * inter-item spaces are NOT desired.
+ */
+function isCJK(ch) {
+  const c = ch.codePointAt(0);
+  return (
+    (c >= 0x2E80 && c <= 0x2FDF) ||   // CJK Radicals Supplement, Kangxi Radicals
+    (c >= 0x4E00 && c <= 0x9FFF) ||   // CJK Unified Ideographs
+    (c >= 0x3400 && c <= 0x4DBF) ||   // CJK Extension A
+    (c >= 0xF900 && c <= 0xFAFF) ||   // CJK Compatibility Ideographs
+    (c >= 0x3040 && c <= 0x309F) ||   // Hiragana
+    (c >= 0x30A0 && c <= 0x30FF) ||   // Katakana
+    (c >= 0xAC00 && c <= 0xD7AF) ||   // Hangul Syllables
+    (c >= 0x20000 && c <= 0x2FA1F)    // CJK Extensions B-F + Supplement
+  );
+}
+
+/** True for CJK fullwidth punctuation that already occupies its own space. */
+function isCJKPunct(ch) {
+  const c = ch.codePointAt(0);
+  return (
+    (c >= 0x3000 && c <= 0x303F) ||    // 。、「」【】
+    (c >= 0xFF01 && c <= 0xFF0F) ||    // ！＂＃＄
+    (c >= 0xFF1A && c <= 0xFF20) ||    // ：；＜＝＞
+    (c >= 0xFF3B && c <= 0xFF40) ||    // ［＼］
+    (c >= 0xFF5B && c <= 0xFF65) ||    // ｛｜｝～
+    (c >= 0xFE50 && c <= 0xFE6B)      // Small Form Variants
+  );
+}
+
+/**
+ * Decide whether a space should be inserted between two adjacent characters
+ * that come from separate pdfjs-dist TextItem objects.
+ *
+ * Rules:
+ *   CJK + CJK  → no space  (中文相邻不加)
+ *   CJK + Punct → no space  (标点紧跟)
+ *   Punct + CJK → no space
+ *   CJK + Latin → space     (中英之间加)
+ *   Latin + CJK → space
+ *   Latin + Latin → space    (英文单词间加)
+ *   Punct + Latin → no space (e.g. `(hello`)
+ *   Latin + Punct → no space (e.g. `hello.`)
+ */
+function shouldAddSpace(left, right) {
+  if (!left || !right) return false;
+
+  const lCJK = isCJK(left);
+  const rCJK = isCJK(right);
+  const lPunct = isCJKPunct(left) || /^[^\w\s]$/.test(left);
+  const rPunct = isCJKPunct(right) || /^[^\w\s]$/.test(right);
+
+  // Any side is punctuation → no space
+  if (lPunct || rPunct) return false;
+  // Both CJK → no space
+  if (lCJK && rCJK) return false;
+  // Mixed CJK/Latin → space
+  if (lCJK !== rCJK) return true;
+  // Both non-CJK (Latin/digit) → space between words
+  return true;
+}
+
+/**
+ * Join pdfjs-dist TextItem[] into clean text, respecting CJK spacing
+ * rules and `hasEOL` line-break markers.
+ */
+function smartJoinTextItems(items) {
+  const lines = [];
+  let cur = '';
+
+  for (const item of items) {
+    const str = item.str ?? '';
+
+    if (str === '') {
+      // Empty str with hasEOL → line break
+      if (item.hasEOL && cur.trim()) {
+        lines.push(cur.trim());
+        cur = '';
+      }
+      continue;
+    }
+
+    if (cur === '') {
+      cur = str;
+    } else {
+      const left = cur[cur.length - 1];
+      const right = str[0];
+      cur += shouldAddSpace(left, right) ? ' ' + str : str;
+    }
+
+    if (item.hasEOL) {
+      if (cur.trim()) lines.push(cur.trim());
+      cur = '';
+    }
+  }
+
+  if (cur.trim()) lines.push(cur.trim());
+  return lines.join('\n');
+}
+
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+
 async function main() {
   const pdfPath = process.argv[2];
   if (!pdfPath || !fs.existsSync(pdfPath)) {
@@ -43,8 +151,7 @@ async function main() {
   for (let i = 1; i <= numPages; i++) {
     const page = await doc.getPage(i);
     const content = await page.getTextContent();
-    const line = content.items.map((item) => item.str ?? '').join(' ');
-    pageTexts.push(line);
+    pageTexts.push(smartJoinTextItems(content.items));
   }
 
   process.stdout.write(JSON.stringify({ text: pageTexts.join('\n\n'), pages: numPages }));