@geminilight/mindos 0.6.59 → 0.6.60

package/_standalone/hooks/useAskChat.ts

@@ -1,7 +1,7 @@
 'use client';
 
 import { useRef, useState, useCallback } from 'react';
-import type { Message, ImagePart, AskMode } from '@/lib/types';
+import type { Message, ImagePart, AskMode, LocalAttachment } from '@/lib/types';
 import type { ProviderId } from '@/lib/agent/providers';
 import { consumeUIMessageStream } from '@/lib/agent/stream-consumer';
 import { isRetryableError, retryDelay, sleep } from '@/lib/agent/reconnect';
@@ -18,7 +18,7 @@ export interface AskChatRefs {
     setMessages: React.Dispatch<React.SetStateAction<Message[]>>;
   }>;
   uploadRef: React.RefObject<{
-    localAttachments: { name: string; content: string }[];
+    localAttachments: LocalAttachment[];
   }>;
   selectedSkillRef: React.RefObject<{ name: string } | null>;
   selectedAcpAgentRef: React.RefObject<unknown>;
@@ -103,12 +103,14 @@ export function useAskChat({
       messages: requestMessages,
       currentFile,
       attachedFiles: refs.attachedFilesRef.current,
-      uploadedFiles: upl.localAttachments.map(f => ({
-        name: f.name,
-        content: f.content.length > 20_000
-          ? f.content.slice(0, 20_000) + '\n\n[...truncated to first ~20000 chars]'
-          : f.content,
-      })),
+      uploadedFiles: upl.localAttachments
+        .filter(f => f.status !== 'loading')
+        .map(f => ({
+          name: f.name,
+          content: f.content.length > 80_000
+            ? f.content.slice(0, 80_000) + '\n\n[...truncated to first ~80000 chars]'
+            : f.content,
+        })),
       selectedAcpAgent: acpAgent,
       mode: chatMode,
       providerOverride: providerOverride ?? undefined,

package/_standalone/hooks/useFileUpload.ts

@@ -22,28 +22,76 @@ function uint8ToBase64(bytes: Uint8Array): string {
   return btoa(binary);
 }
 
-async function extractPdfText(file: File): Promise<string> {
-  const buffer = await file.arrayBuffer();
-  const dataBase64 = uint8ToBase64(new Uint8Array(buffer));
+/**
+ * Extract text from a PDF file via the backend API.
+ * Returns a LocalAttachment with `status` reflecting the outcome.
+ */
+async function extractPdfToAttachment(file: File): Promise<LocalAttachment> {
+  const name = file.name;
 
-  const res = await fetch('/api/extract-pdf', {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json' },
-    body: JSON.stringify({ name: file.name, dataBase64 }),
-  });
-
-  let payload: { text?: string; extracted?: boolean; error?: string } = {};
   try {
-    payload = await res.json();
-  } catch {
-    // ignore JSON parse error
-  }
+    const buffer = await file.arrayBuffer();
+    const dataBase64 = uint8ToBase64(new Uint8Array(buffer));
 
-  if (!res.ok) {
-    throw new Error(payload.error || `PDF extraction failed (${res.status})`);
-  }
+    const res = await fetch('/api/extract-pdf', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ name, dataBase64 }),
+    });
+
+    let payload: {
+      text?: string;
+      extracted?: boolean;
+      error?: string;
+      truncated?: boolean;
+      totalChars?: number;
+      pagesParsed?: number;
+    } = {};
+    try {
+      payload = await res.json();
+    } catch {
+      throw new Error('Failed to parse extraction response');
+    }
+
+    if (!res.ok) {
+      throw new Error(payload.error || `PDF extraction failed (${res.status})`);
+    }
+
+    const extracted = payload.extracted ? (payload.text || '') : '';
+
+    if (!extracted) {
+      return {
+        name,
+        content: `[PDF: ${name}] Could not extract readable text (possibly scanned/image PDF).`,
+        status: 'error',
+        error: 'Scanned or image-only PDF — no extractable text',
+      };
+    }
+
+    const att: LocalAttachment = {
+      name,
+      content: `[PDF TEXT EXTRACTED: ${name}]\n\n${extracted}`,
+      status: 'success',
+    };
+
+    if (payload.truncated && payload.totalChars) {
+      att.truncatedInfo = {
+        totalChars: payload.totalChars,
+        includedChars: extracted.length,
+        totalPages: payload.pagesParsed ?? 0,
+      };
+    }
 
-  return payload.extracted ? (payload.text || '') : '';
+    return att;
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : 'Unknown error';
+    return {
+      name,
+      content: `[PDF: ${name}] Failed to extract text from this PDF.`,
+      status: 'error',
+      error: msg,
+    };
+  }
 }
 
 export function useFileUpload() {
@@ -73,37 +121,43 @@ export function useFileUpload() {
       setUploadError('');
     }
 
-    const loaded = await Promise.all(
-      accepted.map(async (f) => {
-        const ext = getExt(f.name);
-        if (ext === '.pdf') {
-          try {
-            const extracted = await extractPdfText(f);
-            return {
-              name: f.name,
-              content: extracted
-                ? `[PDF TEXT EXTRACTED: ${f.name}]\n\n${extracted}`
-                : `[PDF: ${f.name}] Could not extract readable text (possibly scanned/image PDF).`,
-            };
-          } catch {
-            return {
-              name: f.name,
-              content: `[PDF: ${f.name}] Failed to extract text from this PDF.`,
-            };
-          }
-        }
-        return { name: f.name, content: await f.text() };
-      }),
-    );
+    // Phase 1: Immediately add all files — PDFs start in 'loading' state
+    const pdfFiles: File[] = [];
+    const immediateItems: LocalAttachment[] = [];
+
+    for (const f of accepted) {
+      if (getExt(f.name) === '.pdf') {
+        immediateItems.push({ name: f.name, content: '', status: 'loading' });
+        pdfFiles.push(f);
+      } else {
+        immediateItems.push({
+          name: f.name,
+          content: await f.text(),
+          status: 'success',
+        });
+      }
+    }
 
     setLocalAttachments((prev) => {
       const merged = [...prev];
-      for (const item of loaded) {
-        if (!merged.some((m) => m.name === item.name && m.content === item.content))
-          merged.push(item);
+      for (const item of immediateItems) {
+        if (!merged.some((m) => m.name === item.name)) merged.push(item);
       }
       return merged;
     });
+
+    // Phase 2: Extract PDFs in parallel, then update each one in-place
+    if (pdfFiles.length > 0) {
+      const results = await Promise.all(pdfFiles.map(extractPdfToAttachment));
+
+      setLocalAttachments((prev) =>
+        prev.map((att) => {
+          if (att.status !== 'loading') return att;
+          const result = results.find((r) => r.name === att.name);
+          return result ?? att;
+        }),
+      );
+    }
   }, []);
 
   const removeAttachment = useCallback((idx: number) => {

package/_standalone/hooks/useImageUpload.ts

@@ -86,7 +86,7 @@ export function useImageUpload() {
       }
       try {
         const { data, mimeType } = await compressImage(file);
-        results.push({ type: 'image', data, mimeType });
+        results.push({ type: 'image', data, mimeType, fileName: file.name || undefined });
       } catch {
         setImageError(`Failed to process image: ${file.name}`);
       }

package/_standalone/lib/inbox-upload.ts

@@ -91,3 +91,89 @@ export async function quickDropToInbox(
     toast.error(t.inbox.saveFailed, 4000);
   }
 }
+
+/**
+ * Drop files into a specific directory via /api/file/import.
+ * Used when user drags external files onto a directory in the file tree.
+ */
+export async function quickDropToDirectory(
+  files: File[],
+  targetDir: string,
+  t: ReturnType<typeof useLocale>['t'],
+) {
+  const payload: Array<{ name: string; content: string; encoding?: string }> = [];
+  let oversizedCount = 0;
+
+  const BINARY_EXTS = new Set([
+    'pdf', 'png', 'jpg', 'jpeg', 'gif', 'webp', 'bmp', 'ico',
+    'mp3', 'wav', 'm4a', 'ogg', 'flac', 'aac', 'mp4', 'webm', 'mov', 'mkv',
+  ]);
+
+  for (const file of files) {
+    if (file.size > MAX_FILE_SIZE) {
+      oversizedCount++;
+      continue;
+    }
+    try {
+      const ext = (file.name.split('.').pop() ?? '').toLowerCase();
+      if (BINARY_EXTS.has(ext)) {
+        const buf = await file.arrayBuffer();
+        payload.push({ name: file.name, content: arrayBufferToBase64(buf), encoding: 'base64' });
+      } else {
+        const text = await file.text();
+        payload.push({ name: file.name, content: text });
+      }
+    } catch {
+      /* skip unreadable files */
+    }
+  }
+
+  if (payload.length === 0) {
+    if (oversizedCount > 0) {
+      toast.error(t.inbox.tooLarge(oversizedCount), 4000);
+    } else if (files.length > 0) {
+      toast.error(t.inbox.saveFailed, 4000);
+    }
+    return;
+  }
+
+  const dirName = targetDir.split('/').pop() || targetDir;
+
+  try {
+    const res = await fetch('/api/file/import', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({
+        files: payload,
+        targetSpace: targetDir,
+        organize: false,
+        conflict: 'rename',
+      }),
+    });
+
+    if (!res.ok) {
+      const data = await res.json().catch(() => ({}));
+      console.error('[TreeDrop] Save failed:', data.error);
+      toast.error(t.inbox.saveFailed, 4000);
+      return;
+    }
+
+    const result = await res.json();
+    const saved = result.created?.length ?? 0;
+    const skipped = result.skipped?.length ?? 0;
+
+    if (saved > 0 && (skipped + oversizedCount) > 0) {
+      toast.success(t.inbox.savedWithSkipped(saved, skipped + oversizedCount), 4000);
+    } else if (saved > 0) {
+      toast.success(t.inbox.savedToDir(saved, dirName), 3000);
+    } else {
+      if (oversizedCount > 0) toast.error(t.inbox.tooLarge(oversizedCount), 4000);
+      if (skipped > 0) toast.error(t.inbox.savedWithSkipped(0, skipped), 4000);
+    }
+
+    window.dispatchEvent(new Event('mindos:files-changed'));
+  } catch (err) {
+    console.error('[TreeDrop] Network error:', err);
+    toast.error(t.inbox.saveFailed, 4000);
+  }
+}

package/_standalone/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "wiki-app",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "wiki-app",
-      "version": "0.1.0",
+      "version": "0.1.1",
       "hasInstallScript": true,
       "dependencies": {
         "@base-ui/react": "^1.2.0",

package/_standalone/package.json

@@ -1,6 +1,6 @@
 {
   "name": "wiki-app",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "private": true,
   "scripts": {
     "generate": "npx tsx scripts/generate-explore.ts",

package/_standalone/scripts/extract-pdf.cjs

@@ -13,7 +13,6 @@
 'use strict';
 
 const fs = require('fs');
-const path = require('path');
 
 // Suppress pdfjs-dist warnings (loadFont etc.) that go to stdout/stderr
 // and would corrupt our JSON output.
@@ -22,6 +21,115 @@ const _log = console.log;
 console.warn = () => {};
 console.log = () => {};
 
+// ---------------------------------------------------------------------------
+// CJK-aware smart text joining
+// ---------------------------------------------------------------------------
+
+/**
+ * Detect whether a character falls in CJK / fullwidth ranges where
+ * inter-item spaces are NOT desired.
+ */
+function isCJK(ch) {
+  const c = ch.codePointAt(0);
+  return (
+    (c >= 0x2E80 && c <= 0x2FDF) ||   // CJK Radicals Supplement, Kangxi Radicals
+    (c >= 0x4E00 && c <= 0x9FFF) ||   // CJK Unified Ideographs
+    (c >= 0x3400 && c <= 0x4DBF) ||   // CJK Extension A
+    (c >= 0xF900 && c <= 0xFAFF) ||   // CJK Compatibility Ideographs
+    (c >= 0x3040 && c <= 0x309F) ||   // Hiragana
+    (c >= 0x30A0 && c <= 0x30FF) ||   // Katakana
+    (c >= 0xAC00 && c <= 0xD7AF) ||   // Hangul Syllables
+    (c >= 0x20000 && c <= 0x2FA1F)    // CJK Extensions B-F + Supplement
+  );
+}
+
+/** True for CJK fullwidth punctuation that already occupies its own space. */
+function isCJKPunct(ch) {
+  const c = ch.codePointAt(0);
+  return (
+    (c >= 0x3000 && c <= 0x303F) ||    // 。、「」【】
+    (c >= 0xFF01 && c <= 0xFF0F) ||    // ！＂＃＄
+    (c >= 0xFF1A && c <= 0xFF20) ||    // ：；＜＝＞
+    (c >= 0xFF3B && c <= 0xFF40) ||    // ［＼］
+    (c >= 0xFF5B && c <= 0xFF65) ||    // ｛｜｝～
+    (c >= 0xFE50 && c <= 0xFE6B)      // Small Form Variants
+  );
+}
+
+/**
+ * Decide whether a space should be inserted between two adjacent characters
+ * that come from separate pdfjs-dist TextItem objects.
+ *
+ * Rules:
+ *   CJK + CJK  → no space  (中文相邻不加)
+ *   CJK + Punct → no space  (标点紧跟)
+ *   Punct + CJK → no space
+ *   CJK + Latin → space     (中英之间加)
+ *   Latin + CJK → space
+ *   Latin + Latin → space    (英文单词间加)
+ *   Punct + Latin → no space (e.g. `(hello`)
+ *   Latin + Punct → no space (e.g. `hello.`)
+ */
+function shouldAddSpace(left, right) {
+  if (!left || !right) return false;
+
+  const lCJK = isCJK(left);
+  const rCJK = isCJK(right);
+  const lPunct = isCJKPunct(left) || /^[^\w\s]$/.test(left);
+  const rPunct = isCJKPunct(right) || /^[^\w\s]$/.test(right);
+
+  // Any side is punctuation → no space
+  if (lPunct || rPunct) return false;
+  // Both CJK → no space
+  if (lCJK && rCJK) return false;
+  // Mixed CJK/Latin → space
+  if (lCJK !== rCJK) return true;
+  // Both non-CJK (Latin/digit) → space between words
+  return true;
+}
+
+/**
+ * Join pdfjs-dist TextItem[] into clean text, respecting CJK spacing
+ * rules and `hasEOL` line-break markers.
+ */
+function smartJoinTextItems(items) {
+  const lines = [];
+  let cur = '';
+
+  for (const item of items) {
+    const str = item.str ?? '';
+
+    if (str === '') {
+      // Empty str with hasEOL → line break
+      if (item.hasEOL && cur.trim()) {
+        lines.push(cur.trim());
+        cur = '';
+      }
+      continue;
+    }
+
+    if (cur === '') {
+      cur = str;
+    } else {
+      const left = cur[cur.length - 1];
+      const right = str[0];
+      cur += shouldAddSpace(left, right) ? ' ' + str : str;
+    }
+
+    if (item.hasEOL) {
+      if (cur.trim()) lines.push(cur.trim());
+      cur = '';
+    }
+  }
+
+  if (cur.trim()) lines.push(cur.trim());
+  return lines.join('\n');
+}
+
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+
 async function main() {
   const pdfPath = process.argv[2];
   if (!pdfPath || !fs.existsSync(pdfPath)) {
@@ -43,8 +151,7 @@ async function main() {
   for (let i = 1; i <= numPages; i++) {
     const page = await doc.getPage(i);
     const content = await page.getTextContent();
-    const line = content.items.map((item) => item.str ?? '').join(' ');
-    pageTexts.push(line);
+    pageTexts.push(smartJoinTextItems(content.items));
   }
 
   process.stdout.write(JSON.stringify({ text: pageTexts.join('\n\n'), pages: numPages }));