larkup 0.1.14 → 0.1.21

package/app/api/doc-editor/edit/route.ts

@@ -0,0 +1,88 @@
+import { NextResponse } from 'next/server';
+import { applyFieldEdits, applyContentEdits, getSession } from '@larkup/tool-doc-editor';
+import type { FieldEdit, ContentEdit } from '@larkup/tool-doc-editor';
+
+export const runtime = 'nodejs';
+export const dynamic = 'force-dynamic';
+
+/**
+ * POST /api/doc-editor/edit
+ *
+ * Applies edits (field fills or content changes) to a document session.
+ * Returns the updated parsed document and a flag for live preview refresh.
+ */
+export async function POST(req: Request) {
+  try {
+    const body = await req.json();
+    const { sessionId, fieldEdits, contentEdits } = body as {
+      sessionId: string;
+      fieldEdits?: FieldEdit[];
+      contentEdits?: ContentEdit[];
+    };
+
+    if (!sessionId) {
+      return NextResponse.json({ error: 'sessionId required' }, { status: 400 });
+    }
+
+    const session = getSession(sessionId);
+    if (!session) {
+      return NextResponse.json({ error: 'Session not found' }, { status: 404 });
+    }
+
+    // Apply field edits if provided
+    if (fieldEdits && fieldEdits.length > 0) {
+      const result = await applyFieldEdits(sessionId, fieldEdits);
+
+      if (!result.success) {
+        return NextResponse.json(
+          { error: result.error, fields: session.parsed.fields },
+          { status: 500 },
+        );
+      }
+
+      return NextResponse.json({
+        success: true,
+        updatedFields: result.updatedFields,
+        fields: result.parsed.fields,
+        pages: result.parsed.pages.map((p) => ({
+          index: p.index,
+          text: p.text.slice(0, 2000),
+          html: p.html,
+          fields: p.fields,
+          dimensions: p.dimensions,
+        })),
+        totalPages: result.parsed.totalPages,
+        // Include base64 for live preview
+        fileBase64: result.fileBase64,
+      });
+    }
+
+    // Apply content edits if provided
+    if (contentEdits && contentEdits.length > 0) {
+      const result = await applyContentEdits(sessionId, contentEdits);
+
+      if (!result.success) {
+        return NextResponse.json({ error: result.error }, { status: 500 });
+      }
+
+      return NextResponse.json({
+        success: true,
+        updatedFields: result.updatedFields,
+        pages: result.parsed.pages.map((p) => ({
+          index: p.index,
+          text: p.text.slice(0, 2000),
+          html: p.html,
+          fields: p.fields,
+          dimensions: p.dimensions,
+        })),
+        totalPages: result.parsed.totalPages,
+        fileBase64: result.fileBase64,
+      });
+    }
+
+    return NextResponse.json({ error: 'No edits provided' }, { status: 400 });
+  } catch (err) {
+    const message = err instanceof Error ? err.message : 'Edit failed';
+    return NextResponse.json({ error: message }, { status: 500 });
+  }
+}

package/app/api/doc-editor/export/route.ts

@@ -0,0 +1,33 @@
+import { NextResponse } from 'next/server';
+import { exportDocument } from '@larkup/tool-doc-editor';
+
+export const runtime = 'nodejs';
+export const dynamic = 'force-dynamic';
+
+/**
+ * GET /api/doc-editor/export?sessionId=xxx
+ *
+ * Downloads the current (possibly modified) document as a binary file.
+ */
+export async function GET(req: Request) {
+  const url = new URL(req.url);
+  const sessionId = url.searchParams.get('sessionId');
+
+  if (!sessionId) {
+    return NextResponse.json({ error: 'sessionId required' }, { status: 400 });
+  }
+
+  const result = exportDocument(sessionId);
+  if (!result) {
+    return NextResponse.json({ error: 'Session not found' }, { status: 404 });
+  }
+
+  return new NextResponse(result.buffer, {
+    status: 200,
+    headers: {
+      'Content-Type': result.mimeType,
+      'Content-Disposition': `attachment; filename="${encodeURIComponent(result.fileName)}"`,
+      'Content-Length': String(result.buffer.length),
+    },
+  });
+}

package/app/api/doc-editor/extract-text/route.ts

@@ -0,0 +1,41 @@
+import { NextResponse } from 'next/server';
+import { parseDocument } from '@larkup/tool-doc-editor';
+
+export async function POST(req: Request) {
+  try {
+    const formData = await req.formData();
+    const files = formData.getAll('files') as File[];
+
+    if (!files || files.length === 0) {
+      return NextResponse.json({ error: 'No files provided' }, { status: 400 });
+    }
+
+    const results: { fileName: string; content: string; error?: string }[] = [];
+
+    for (const file of files) {
+      try {
+        const arrayBuffer = await file.arrayBuffer();
+        const buffer = Buffer.from(arrayBuffer);
+        const parsed = await parseDocument(buffer, file.name, file.type);
+
+        // Extract text content, truncated to avoid massive payloads
+        const content = parsed.rawText || parsed.pages.map((p) => p.text).join('\n\n');
+        results.push({
+          fileName: file.name,
+          content: content.substring(0, 15000), // Cap at 15K chars per file
+        });
+      } catch (err: any) {
+        results.push({
+          fileName: file.name,
+          content: '',
+          error: err.message || 'Failed to extract text',
+        });
+      }
+    }
+
+    return NextResponse.json({ results });
+  } catch (err: any) {
+    console.error('Extract text error:', err);
+    return NextResponse.json({ error: 'Internal Server Error' }, { status: 500 });
+  }
+}

package/app/api/doc-editor/parse/route.ts

@@ -0,0 +1,66 @@
+import { NextResponse } from 'next/server';
+import { createSession, enrichPDFWithText, getSession } from '@larkup/tool-doc-editor';
+// @ts-expect-error — import from lib to avoid pdf-parse's test-file-loading bug
+import pdfParse from 'pdf-parse/lib/pdf-parse.js';
+
+export const runtime = 'nodejs';
+export const dynamic = 'force-dynamic';
+
+/**
+ * POST /api/doc-editor/parse
+ *
+ * Accepts a file upload (FormData), parses it, creates an editor session,
+ * and returns the parsed document structure.
+ */
+export async function POST(req: Request) {
+  try {
+    const formData = await req.formData();
+    const file = formData.get('file') as File | null;
+
+    if (!file) {
+      return NextResponse.json({ error: 'No file provided' }, { status: 400 });
+    }
+
+    const buffer = Buffer.from(await file.arrayBuffer());
+    const session = await createSession(buffer, file.name, file.type);
+
+    // For PDFs, enrich with full text extraction via pdf-parse
+    // Skip heavy text extraction for long PDFs (e.g. > 5 pages) to just "view" them quickly
+    if (session.type === 'pdf' && session.parsed.rawText === '') {
+      if (session.parsed.totalPages <= 5) {
+        try {
+          const pdfData = await pdfParse(buffer);
+          const enrichedParsed = enrichPDFWithText(session.parsed, pdfData.text);
+          session.parsed = enrichedParsed;
+        } catch {
+          // pdf-parse failed — continue with whatever we have
+        }
+      } else {
+        session.parsed.rawText = `[Long PDF - Text extraction skipped for performance. You can still view it.]`;
+      }
+    }
+
+    // Return a lightweight response (no base64 file data)
+    return NextResponse.json({
+      sessionId: session.id,
+      fileName: session.fileName,
+      type: session.type,
+      mimeType: session.mimeType,
+      fileSize: session.parsed.fileSize,
+      totalPages: session.parsed.totalPages,
+      fields: session.parsed.fields,
+      pages: session.parsed.pages.map((p) => ({
+        index: p.index,
+        text: p.text.slice(0, 2000), // Limit text for response size
+        html: p.html,
+        fields: p.fields,
+        dimensions: p.dimensions,
+      })),
+      rawText: session.parsed.rawText.slice(0, 5000),
+      metadata: session.parsed.metadata,
+    });
+  } catch (err) {
+    const message = err instanceof Error ? err.message : 'Failed to parse file';
+    return NextResponse.json({ error: message }, { status: 500 });
+  }
+}

package/app/api/doc-editor/restore/route.ts

@@ -0,0 +1,74 @@
+import { NextResponse } from 'next/server';
+import { restoreSession, parseDocument } from '@larkup/tool-doc-editor';
+import type { EditorSession } from '@larkup/tool-doc-editor';
+
+export async function POST(req: Request) {
+  try {
+    const { sessionId, fileBase64, parsedDocument, fileName, mimeType, activeFileType } =
+      await req.json();
+
+    if (!sessionId || !fileBase64) {
+      return NextResponse.json({ error: 'Missing sessionId or fileBase64' }, { status: 400 });
+    }
+
+    const buffer = Buffer.from(fileBase64, 'base64');
+
+    // Re-parse the actual file to get proper fields and pages.
+    // This is critical: if we don't have a parsedDocument with fields,
+    // any subsequent fill attempts will fail because field ID resolution
+    // relies on session.parsed.fields being populated.
+    let parsed;
+    if (parsedDocument && parsedDocument.fields && parsedDocument.fields.length > 0) {
+      parsed = parsedDocument;
+    } else {
+      try {
+        const resolvedMime = mimeType || 'application/pdf';
+        const resolvedName = fileName || 'Document';
+        parsed = await parseDocument(buffer, resolvedName, resolvedMime);
+        parsed.sessionId = sessionId;
+      } catch (parseErr: any) {
+        console.warn('[doc-editor/restore] Re-parse failed, using empty parsed:', parseErr.message);
+        parsed = {
+          sessionId,
+          fileName: fileName || 'Document',
+          type: activeFileType || 'pdf',
+          mimeType: mimeType || 'application/pdf',
+          fileSize: buffer.length,
+          pages: [],
+          fields: [],
+          rawText: '',
+          metadata: {},
+          totalPages: 1,
+        };
+      }
+    }
+
+    const session: EditorSession = {
+      id: sessionId,
+      fileName: fileName || parsed.fileName || 'Document',
+      type: activeFileType || parsed.type || 'pdf',
+      mimeType: mimeType || parsed.mimeType || 'application/pdf',
+      originalFileBase64: fileBase64,
+      currentFileBase64: fileBase64,
+      parsed: parsed,
+      edits: [],
+      createdAt: new Date().toISOString(),
+      updatedAt: new Date().toISOString(),
+    };
+
+    restoreSession(session);
+    console.log(
+      `[doc-editor/restore] Session ${sessionId} restored with ${
+        parsed.fields?.length ?? 0
+      } fields`,
+    );
+
+    return NextResponse.json({ success: true, message: 'Session restored in backend' });
+  } catch (error: any) {
+    console.error('Error restoring session:', error);
+    return NextResponse.json(
+      { error: error.message || 'Failed to restore session' },
+      { status: 500 },
+    );
+  }
+}

package/app/api/document/sign/route.ts

@@ -0,0 +1,56 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { applySignature } from '@larkup/tool-doc-editor';
+import { z } from 'zod';
+
+const requestSchema = z.object({
+  sessionId: z.string(),
+  signatureData: z.object({
+    mode: z.enum(['type', 'draw', 'upload']),
+    imagePayload: z.string().optional(),
+    text: z.string().optional(),
+    font: z.string().optional(),
+    color: z.string().optional(),
+    date: z.string().optional(),
+    extraText: z.string().optional(),
+    pageIndex: z.number(),
+    x: z.number().optional(),
+    y: z.number().optional(),
+    scale: z.number().optional(),
+    base64Override: z.string().optional(),
+    detectedContext: z.string().optional(),
+    placementNote: z.string().optional(),
+  }),
+});
+
+export async function POST(req: NextRequest) {
+  try {
+    const body = await req.json();
+    const result = requestSchema.safeParse(body);
+
+    if (!result.success) {
+      return NextResponse.json(
+        { error: 'Invalid request data', details: result.error.issues },
+        { status: 400 },
+      );
+    }
+
+    const { sessionId, signatureData } = result.data;
+    const editResult = await applySignature(sessionId, signatureData as any);
+
+    if (!editResult.success) {
+      return NextResponse.json({ error: editResult.error }, { status: 500 });
+    }
+
+    // Since EditResult no longer returns `document` but `parsed` and `fileBase64`, we return those.
+    return NextResponse.json({
+      document: {
+        parsed: (editResult as any).parsed,
+        fileBase64: (editResult as any).fileBase64,
+        updatedFields: (editResult as any).updatedFields,
+      },
+    });
+  } catch (err: any) {
+    console.error('Sign doc error:', err);
+    return NextResponse.json({ error: 'Internal Server Error' }, { status: 500 });
+  }
+}

package/app/api/documents/route.ts

@@ -1,4 +1,4 @@
-import { NextResponse } from "next/server"
+import { NextResponse } from 'next/server';
 import {
   addDocument,
   clearDocuments,
@@ -7,18 +7,49 @@ import {
   deleteDocuments,
   readDocuments,
   updateDocument,
-} from "@larkup/core/documents-store"
-import type { DocumentSource } from "@larkup/core/types"
+} from '@larkup/core/documents-store';
+import { readConfig } from '@larkup/core/config-store';
+import { readRun, patchRun } from '@larkup/core/index-store';
+import { createAdapter } from '@larkup/vector-stores/factory';
+import type { DocumentSource } from '@larkup/core/types';
 
-export const runtime = "nodejs"
-export const dynamic = "force-dynamic"
+export const runtime = 'nodejs';
+export const dynamic = 'force-dynamic';
 
 /** GET → the full corpus plus summary stats. */
 export async function GET() {
-  const [documents, stats] = await Promise.all([readDocuments(), corpusStats()])
-  return NextResponse.json({ documents, stats })
+  const [storedDocuments, stats] = await Promise.all([readDocuments(), corpusStats()]);
+
+  // Media transcripts/timelines are staged in the document store so the
+  // indexer can embed them. Do not expose those temporary records in the
+  // Knowledge Base until the indexer has actually completed successfully.
+  // If indexing fails, the media worker removes the staged record again.
+  const documents = storedDocuments.filter(
+    (document) => document.source !== 'media' || document.status === 'indexed',
+  );
+  const visibleChars = documents.reduce((total, document) => total + document.charCount, 0);
+  const visibleStats = {
+    ...stats,
+    docCount: documents.length,
+    charCount: visibleChars,
+    bySource: documents.reduce<Record<string, number>>((counts, document) => {
+      counts[document.source] = (counts[document.source] ?? 0) + 1;
+      return counts;
+    }, {}),
+  };
+
+  // Sort documents by createdAt descending (newest first)
+  const sortedDocuments = documents.sort((a, b) => {
+    return new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime();
+  });
+
+  return NextResponse.json({ documents: sortedDocuments, stats: visibleStats });
 }
 
+import { randomUUID } from 'crypto';
+import fs from 'fs/promises';
+import path from 'path';
+
 /**
  * POST → ingest pasted text or an uploaded file's contents.
  * Body: { title, content, source: "paste" | "upload", url? }
@@ -26,26 +57,48 @@ export async function GET() {
 export async function POST(req: Request) {
   try {
     const body = (await req.json()) as {
-      title?: string
-      content?: string
-      source?: DocumentSource
-      url?: string
-      metadata?: Record<string, any>
-    }
+      title?: string;
+      content?: string;
+      source?: DocumentSource;
+      url?: string;
+      metadata?: Record<string, any>;
+    };
     if (!body.content || !body.content.trim()) {
-      return NextResponse.json({ error: "Content is empty." }, { status: 400 })
+      return NextResponse.json({ error: 'Content is empty.' }, { status: 400 });
     }
+
+    // Intercept imageBase64 and write it to disk
+    if (body.metadata && body.metadata.isImage && body.metadata.imageBase64) {
+      try {
+        const base64Data = body.metadata.imageBase64.replace(/^data:image\/\w+;base64,/, '');
+        const buffer = Buffer.from(base64Data, 'base64');
+        const filename = `${randomUUID()}.png`;
+        const uploadsDir = path.join(process.cwd(), '.larkup', 'uploads');
+        await fs.mkdir(uploadsDir, { recursive: true });
+        await fs.writeFile(path.join(uploadsDir, filename), buffer);
+
+        // Remove base64 string and save the URL instead
+        delete body.metadata.imageBase64;
+        body.metadata.imageUrl = `/api/uploads/${filename}`;
+
+        // Use imageUrl as the url for the document
+        body.url = body.metadata.imageUrl;
+      } catch (err) {
+        console.error('Failed to save extracted image to disk', err);
+      }
+    }
+
     const doc = await addDocument({
-      title: body.title ?? "Untitled",
+      title: body.title ?? 'Untitled',
       content: body.content,
-      source: body.source === "upload" ? "upload" : "paste",
+      source: body.source === 'upload' ? 'upload' : 'paste',
       url: body.url,
       metadata: body.metadata,
-    })
-    return NextResponse.json({ document: doc }, { status: 201 })
+    });
+    return NextResponse.json({ document: doc }, { status: 201 });
   } catch (err) {
-    const message = err instanceof Error ? err.message : "Failed to add document."
-    return NextResponse.json({ error: message }, { status: 500 })
+    const message = err instanceof Error ? err.message : 'Failed to add document.';
+    return NextResponse.json({ error: message }, { status: 500 });
   }
 }
 
@@ -54,37 +107,134 @@ export async function POST(req: Request) {
  * Body: { id, title?, content?, url? }
  */
 export async function PATCH(req: Request) {
-  let body: { id?: string; title?: string; content?: string; url?: string; metadata?: Record<string, any> }
+  let body: {
+    id?: string;
+    title?: string;
+    content?: string;
+    url?: string;
+    metadata?: Record<string, any>;
+  };
   try {
-    body = await req.json()
+    body = await req.json();
   } catch {
-    return NextResponse.json({ error: "Invalid JSON body." }, { status: 400 })
+    return NextResponse.json({ error: 'Invalid JSON body.' }, { status: 400 });
   }
   if (!body.id) {
-    return NextResponse.json({ error: "id is required." }, { status: 400 })
+    return NextResponse.json({ error: 'id is required.' }, { status: 400 });
   }
   if (body.content !== undefined && !body.content.trim()) {
-    return NextResponse.json({ error: "Content is empty." }, { status: 400 })
+    return NextResponse.json({ error: 'Content is empty.' }, { status: 400 });
   }
   const doc = await updateDocument(body.id, {
     title: body.title,
     content: body.content,
     url: body.url,
     metadata: body.metadata,
-  })
+  });
   if (!doc) {
-    return NextResponse.json({ error: "Document not found." }, { status: 404 })
+    return NextResponse.json({ error: 'Document not found.' }, { status: 404 });
   }
-  return NextResponse.json({ document: doc })
+  return NextResponse.json({ document: doc });
 }
 
 /** DELETE ?id=<id> removes one doc; DELETE ?ids=1,2 removes many; DELETE with no id clears the corpus. */
 export async function DELETE(req: Request) {
-  const url = new URL(req.url)
-  const id = url.searchParams.get("id")
-  const ids = url.searchParams.get("ids")
-  if (id) await deleteDocument(id)
-  else if (ids) await deleteDocuments(ids.split(","))
-  else await clearDocuments()
-  return NextResponse.json({ ok: true })
+  const url = new URL(req.url);
+  const id = url.searchParams.get('id');
+  const ids = url.searchParams.get('ids');
+
+  // Clean up associated media assets
+  const docs = await readDocuments();
+  let toDeleteDocs = [];
+  if (id) {
+    toDeleteDocs = docs.filter((d) => d.id === id);
+  } else if (ids) {
+    const idList = ids.split(',');
+    toDeleteDocs = docs.filter((d) => idList.includes(d.id));
+  } else {
+    toDeleteDocs = docs;
+  }
+
+  for (const doc of toDeleteDocs) {
+    if (doc.metadata?.mediaAssetId) {
+      const { getMediaAsset, deleteMediaAsset } = await import('@larkup/core/media-store');
+      const { createStorageProvider } = await import('@larkup/marketplace/storage');
+      const asset = await getMediaAsset(doc.metadata.mediaAssetId);
+      if (asset) {
+        const storage = createStorageProvider();
+        await storage.delete(asset.storageUri).catch(() => {});
+        if (asset.thumbnailUri) await storage.delete(asset.thumbnailUri).catch(() => {});
+        await deleteMediaAsset(doc.metadata.mediaAssetId);
+      }
+    }
+
+    const cleanupUrls = [doc.url, doc.metadata?.imageUrl].filter(Boolean);
+    for (const docUrl of cleanupUrls) {
+      if (docUrl?.startsWith('/api/media/')) {
+        const assetId = docUrl.split('/api/media/')[1]?.split('?')[0];
+        if (assetId) {
+          const { getMediaAsset, deleteMediaAsset } = await import('@larkup/core/media-store');
+          const { createStorageProvider } = await import('@larkup/marketplace/storage');
+          const asset = await getMediaAsset(assetId);
+          if (asset) {
+            const storage = createStorageProvider();
+            await storage.delete(asset.storageUri).catch(() => {});
+            if (asset.thumbnailUri) await storage.delete(asset.thumbnailUri).catch(() => {});
+            await deleteMediaAsset(assetId);
+          }
+        }
+      } else if (docUrl?.startsWith('/api/uploads/')) {
+        const filename = docUrl.split('/api/uploads/')[1]?.split('?')[0];
+        if (filename) {
+          const uploadsDir = path.join(process.cwd(), '.larkup', 'uploads');
+          await fs.unlink(path.join(uploadsDir, filename)).catch(() => {});
+        }
+      }
+    }
+  }
+
+  let deletedIds: string[] = [];
+
+  if (id) {
+    await deleteDocument(id);
+    deletedIds = [id];
+  } else if (ids) {
+    const idList = ids.split(',');
+    await deleteDocuments(idList);
+    deletedIds = idList;
+  } else {
+    await clearDocuments();
+
+    try {
+      const config = await readConfig();
+      const adapter = await createAdapter(config);
+      await adapter.init(0);
+      await adapter.reset();
+    } catch {}
+
+    try {
+      const run = await readRun();
+      if (run) {
+        await patchRun({
+          status: 'failed',
+          error: 'Corpus cleared — index invalidated.',
+          finishedAt: new Date().toISOString(),
+        });
+      }
+    } catch {
+      // best-effort
+    }
+  }
+
+  if (deletedIds.length > 0) {
+    try {
+      const config = await readConfig();
+      const adapter = await createAdapter(config);
+      await adapter.init(0);
+      await adapter.deleteByDocumentIds(deletedIds);
+    } catch (err) {
+      console.error('Failed to delete vectors for specific documents:', err);
+    }
+  }
+  return NextResponse.json({ ok: true });
 }