@uploadista/flow-documents-nodes 0.0.16-beta.2

package/src/convert-to-markdown-node.ts

@@ -0,0 +1,156 @@
+import { UploadistaError } from "@uploadista/core/errors";
+import {
+  completeNodeExecution,
+  createFlowNode,
+  DocumentAiPlugin,
+  DocumentPlugin,
+  NodeType,
+  resolveUploadMetadata,
+} from "@uploadista/core/flow";
+import { uploadFileSchema } from "@uploadista/core/types";
+import { UploadServer } from "@uploadista/core/upload";
+import { Effect, Either } from "effect";
+
+export type ConvertToMarkdownNodeParams = {
+  credentialId?: string;
+  resolution?: "tiny" | "small" | "base" | "gundam" | "large";
+};
+
+export function createConvertToMarkdownNode(
+  id: string,
+  params: ConvertToMarkdownNodeParams = {},
+) {
+  return Effect.gen(function* () {
+    const documentService = yield* DocumentPlugin;
+    const documentAiService = yield* DocumentAiPlugin;
+    const uploadServer = yield* UploadServer;
+
+    return yield* createFlowNode({
+      id,
+      name: "Convert to Markdown",
+      description:
+        "Convert documents to Markdown format (intelligently uses OCR if needed)",
+      type: NodeType.process,
+      inputSchema: uploadFileSchema,
+      outputSchema: uploadFileSchema,
+      run: ({ data: file, flowId, jobId, clientId }) => {
+        return Effect.gen(function* () {
+          const flow = {
+            flowId,
+            nodeId: id,
+            jobId,
+          };
+
+          yield* Effect.logInfo(`Converting file ${file.id} to Markdown`);
+
+          // Read file bytes from upload server
+          const fileBytes = yield* uploadServer.read(file.id, clientId);
+
+          // Try to extract text first (for searchable PDFs)
+          const extractResult = yield* documentService
+            .extractText(fileBytes)
+            .pipe(Effect.either);
+
+          let markdown: string;
+          let markdownSource: "text" | "ocr";
+
+          if (
+            Either.isRight(extractResult) &&
+            extractResult.right.trim().length > 0
+          ) {
+            // Successfully extracted text from searchable PDF
+            const text = extractResult.right;
+
+            yield* Effect.logInfo(
+              `Successfully extracted ${text.length} characters from searchable PDF`,
+            );
+
+            // Simple text-to-markdown conversion
+            // In a real implementation, this could be more sophisticated
+            markdown = text
+              .split("\n\n")
+              .map((para: string) => para.trim())
+              .filter((para: string) => para.length > 0)
+              .join("\n\n");
+
+            markdownSource = "text";
+
+            yield* Effect.logInfo(
+              `Converted text to Markdown (${markdown.length} characters)`,
+            );
+          } else {
+            // Text extraction failed or returned empty - use OCR
+            yield* Effect.logInfo(
+              "Text extraction failed or returned empty, falling back to OCR",
+            );
+
+            const fileUrl = file.url;
+
+            if (!fileUrl) {
+              return yield* UploadistaError.fromCode("FLOW_NODE_ERROR", {
+                cause: "URL is required for OCR-based markdown conversion",
+              }).toEffect();
+            }
+
+            // Build context for DocumentAI plugin
+            const context = {
+              clientId,
+              credentialId: params.credentialId,
+            };
+
+            // Perform OCR with markdown conversion
+            const ocrResult = yield* documentAiService
+              .performOCR(
+                fileUrl,
+                {
+                  taskType: "convertToMarkdown",
+                  resolution: params.resolution || "gundam",
+                },
+                context,
+              )
+              .pipe(
+                Effect.catchAll((error) =>
+                  Effect.gen(function* () {
+                    yield* Effect.logError("Failed to perform OCR", error);
+                    return yield* UploadistaError.fromCode("FLOW_NODE_ERROR", {
+                      cause:
+                        error instanceof Error
+                          ? error.message
+                          : "Failed to perform OCR for markdown conversion",
+                    }).toEffect();
+                  }),
+                ),
+              );
+
+            markdown = ocrResult.extractedText;
+            markdownSource = "ocr";
+
+            yield* Effect.logInfo(
+              `Successfully converted scanned document to Markdown using OCR (${markdown.length} characters)`,
+            );
+          }
+
+          const { metadata } = resolveUploadMetadata(file.metadata);
+
+          // Add markdown to metadata
+          const newMetadata = {
+            ...file.metadata,
+            ...metadata,
+            markdown,
+            markdownSource,
+          };
+
+          yield* Effect.logInfo(
+            `Successfully converted file ${file.id} to Markdown via ${markdownSource}`,
+          );
+
+          return completeNodeExecution({
+            ...file,
+            metadata: newMetadata,
+            flow,
+          });
+        });
+      },
+    });
+  });
+}

package/src/describe-document-node.ts

@@ -0,0 +1,92 @@
+import { UploadistaError } from "@uploadista/core/errors";
+import {
+  completeNodeExecution,
+  createFlowNode,
+  DocumentPlugin,
+  NodeType,
+  resolveUploadMetadata,
+} from "@uploadista/core/flow";
+import { uploadFileSchema } from "@uploadista/core/types";
+import { UploadServer } from "@uploadista/core/upload";
+import { Effect } from "effect";
+
+export type DescribeDocumentNodeParams = Record<string, never>; // No parameters needed
+
+export function createDescribeDocumentNode(
+  id: string,
+  _params: DescribeDocumentNodeParams = {},
+) {
+  return Effect.gen(function* () {
+    const documentService = yield* DocumentPlugin;
+    const uploadServer = yield* UploadServer;
+
+    return yield* createFlowNode({
+      id,
+      name: "Describe Document",
+      description: "Extract metadata from PDF documents",
+      type: NodeType.process,
+      inputSchema: uploadFileSchema,
+      outputSchema: uploadFileSchema,
+      run: ({ data: file, flowId, jobId, clientId }) => {
+        return Effect.gen(function* () {
+          const flow = {
+            flowId,
+            nodeId: id,
+            jobId,
+          };
+
+          yield* Effect.logInfo(
+            `Extracting metadata from PDF file ${file.id}`,
+          );
+
+          // Read file bytes from upload server
+          const fileBytes = yield* uploadServer.read(file.id, clientId);
+
+          // Get metadata with error handling
+          const documentMetadata = yield* documentService
+            .getMetadata(fileBytes)
+            .pipe(
+              Effect.catchAll((error) =>
+                Effect.gen(function* () {
+                  yield* Effect.logError("Failed to extract metadata", error);
+                  return yield* UploadistaError.fromCode("FLOW_NODE_ERROR", {
+                    cause:
+                      error instanceof Error
+                        ? error.message
+                        : "Failed to extract metadata",
+                  }).toEffect();
+                }),
+              ),
+            );
+
+          const { metadata } = resolveUploadMetadata(file.metadata);
+
+          // Add document metadata to file metadata (filter out null values)
+          const newMetadata = {
+            ...file.metadata,
+            ...metadata,
+            pageCount: documentMetadata.pageCount,
+            format: documentMetadata.format,
+            ...(documentMetadata.author && { author: documentMetadata.author }),
+            ...(documentMetadata.title && { title: documentMetadata.title }),
+            ...(documentMetadata.subject && { subject: documentMetadata.subject }),
+            ...(documentMetadata.creator && { creator: documentMetadata.creator }),
+            ...(documentMetadata.creationDate && { creationDate: documentMetadata.creationDate }),
+            ...(documentMetadata.modifiedDate && { modifiedDate: documentMetadata.modifiedDate }),
+            fileSize: documentMetadata.fileSize,
+          };
+
+          yield* Effect.logInfo(
+            `Successfully extracted metadata from file ${file.id}: ${documentMetadata.pageCount} pages`,
+          );
+
+          return completeNodeExecution({
+            ...file,
+            metadata: newMetadata,
+            flow,
+          });
+        });
+      },
+    });
+  });
+}

package/src/extract-text-node.ts

@@ -0,0 +1,90 @@
+import { UploadistaError } from "@uploadista/core/errors";
+import {
+  completeNodeExecution,
+  createFlowNode,
+  DocumentPlugin,
+  NodeType,
+  resolveUploadMetadata,
+} from "@uploadista/core/flow";
+import { uploadFileSchema } from "@uploadista/core/types";
+import { UploadServer } from "@uploadista/core/upload";
+import { Effect } from "effect";
+
+export type ExtractTextNodeParams = Record<string, never>; // No parameters needed
+
+export function createExtractTextNode(
+  id: string,
+  _params: ExtractTextNodeParams = {},
+) {
+  return Effect.gen(function* () {
+    const documentService = yield* DocumentPlugin;
+    const uploadServer = yield* UploadServer;
+
+    return yield* createFlowNode({
+      id,
+      name: "Extract Text",
+      description: "Extract text from searchable PDF documents",
+      type: NodeType.process,
+      inputSchema: uploadFileSchema,
+      outputSchema: uploadFileSchema,
+      run: ({ data: file, flowId, jobId, clientId }) => {
+        return Effect.gen(function* () {
+          const flow = {
+            flowId,
+            nodeId: id,
+            jobId,
+          };
+
+          yield* Effect.logInfo(
+            `Extracting text from PDF file ${file.id}`,
+          );
+
+          // Read file bytes from upload server
+          const fileBytes = yield* uploadServer.read(file.id, clientId);
+
+          // Extract text with error handling
+          const extractedText = yield* documentService
+            .extractText(fileBytes)
+            .pipe(
+              Effect.catchAll((error) =>
+                Effect.gen(function* () {
+                  yield* Effect.logError("Failed to extract text", error);
+                  return yield* UploadistaError.fromCode("FLOW_NODE_ERROR", {
+                    cause:
+                      error instanceof Error
+                        ? error.message
+                        : "Failed to extract text",
+                  }).toEffect();
+                }),
+              ),
+            );
+
+          const { metadata } = resolveUploadMetadata(file.metadata);
+
+          // Add extracted text to metadata
+          const newMetadata = {
+            ...file.metadata,
+            ...metadata,
+            extractedText,
+          };
+
+          if (!extractedText || extractedText.trim().length === 0) {
+            yield* Effect.logWarning(
+              `No text extracted from file ${file.id}. This might be a scanned document. Consider using the OCR node instead.`,
+            );
+          } else {
+            yield* Effect.logInfo(
+              `Successfully extracted ${extractedText.length} characters from file ${file.id}`,
+            );
+          }
+
+          return completeNodeExecution({
+            ...file,
+            metadata: newMetadata,
+            flow,
+          });
+        });
+      },
+    });
+  });
+}

package/src/index.ts

@@ -0,0 +1,27 @@
+// Document processing nodes
+
+export {
+  type ConvertToMarkdownNodeParams,
+  createConvertToMarkdownNode,
+} from "./convert-to-markdown-node";
+export {
+  createDescribeDocumentNode,
+  type DescribeDocumentNodeParams,
+} from "./describe-document-node";
+export {
+  createExtractTextNode,
+  type ExtractTextNodeParams,
+} from "./extract-text-node";
+
+export {
+  createMergePdfNode,
+  type MergePdfNodeParams,
+} from "./merge-pdf-node";
+export {
+  createOcrNode,
+  type OcrNodeParams,
+} from "./ocr-node";
+export {
+  createSplitPdfNode,
+  type SplitPdfNodeParams,
+} from "./split-pdf-node";

package/src/merge-pdf-node.ts

@@ -0,0 +1,144 @@
+import { UploadistaError } from "@uploadista/core/errors";
+import {
+  completeNodeExecution,
+  createFlowNode,
+  DocumentPlugin,
+  NodeType,
+  resolveUploadMetadata,
+} from "@uploadista/core/flow";
+import { uploadFileSchema } from "@uploadista/core/types";
+import { UploadServer } from "@uploadista/core/upload";
+import { Effect } from "effect";
+import { z } from "zod";
+
+export type MergePdfNodeParams = {
+  inputCount?: number;
+};
+
+// Schema for multiple file inputs
+const multipleFilesSchema = z.array(uploadFileSchema);
+
+export function createMergePdfNode(
+  id: string,
+  _params: MergePdfNodeParams = {},
+) {
+  return Effect.gen(function* () {
+    const documentService = yield* DocumentPlugin;
+    const uploadServer = yield* UploadServer;
+
+    return yield* createFlowNode({
+      id,
+      name: "Merge PDFs",
+      description: "Merge multiple PDF documents into one",
+      type: NodeType.process,
+      inputSchema: multipleFilesSchema,
+      outputSchema: uploadFileSchema,
+      run: ({ data: files, flowId, jobId, clientId }) => {
+        return Effect.gen(function* () {
+          const flow = {
+            flowId,
+            nodeId: id,
+            jobId,
+          };
+
+          // Validate that we have an array of files
+          if (!Array.isArray(files)) {
+            return yield* UploadistaError.fromCode("FLOW_NODE_ERROR", {
+              cause: "Merge PDF node requires an array of files from a Merge utility node",
+            }).toEffect();
+          }
+
+          if (files.length === 0) {
+            return yield* UploadistaError.fromCode("FLOW_NODE_ERROR", {
+              cause: "At least one PDF file is required for merging",
+            }).toEffect();
+          }
+
+          // Read buffers from all files
+          const pdfBuffers: Uint8Array[] = [];
+          let totalPages = 0;
+
+          for (const file of files) {
+            // Read file bytes from upload server
+            const fileBytes = yield* uploadServer.read(file.id, clientId);
+            pdfBuffers.push(fileBytes);
+
+            // Sum up page counts if available
+            const fileMetadata = resolveUploadMetadata(file.metadata).metadata;
+            if (fileMetadata?.pageCount && typeof fileMetadata.pageCount === 'number') {
+              totalPages += fileMetadata.pageCount;
+            }
+          }
+
+          yield* Effect.logInfo(
+            `Merging ${files.length} PDF files`,
+          );
+
+          // Merge PDFs with error handling
+          const mergedPdf = yield* documentService
+            .mergePdfs({ pdfs: pdfBuffers })
+            .pipe(
+              Effect.catchAll((error) =>
+                Effect.gen(function* () {
+                  yield* Effect.logError("Failed to merge PDFs", error);
+                  return yield* UploadistaError.fromCode("FLOW_NODE_ERROR", {
+                    cause:
+                      error instanceof Error
+                        ? error.message
+                        : "Failed to merge PDFs",
+                  }).toEffect();
+                }),
+              ),
+            );
+
+          // Use metadata from first file as base
+          const firstFile = files[0];
+          const { metadata } = resolveUploadMetadata(firstFile.metadata);
+
+          // Create a stream from the merged PDF bytes
+          const stream = new ReadableStream({
+            start(controller) {
+              controller.enqueue(mergedPdf);
+              controller.close();
+            },
+          });
+
+          // Upload the merged PDF back to the upload server
+          const result = yield* uploadServer.upload(
+            {
+              storageId: firstFile.storage.id,
+              size: mergedPdf.byteLength,
+              type: "application/pdf",
+              fileName: `merged-${files.length}-documents.pdf`,
+              lastModified: 0,
+              metadata: JSON.stringify({
+                ...metadata,
+                pageCount: totalPages,
+                mergedFrom: files.length,
+              }),
+              flow,
+            },
+            clientId,
+            stream,
+          );
+
+          const newMetadata = {
+            ...metadata,
+            pageCount: totalPages,
+            mergedFrom: files.length,
+            fileName: `merged-${files.length}-documents.pdf`,
+          };
+
+          yield* Effect.logInfo(
+            `Successfully merged ${files.length} PDFs into one document with ${totalPages} pages`,
+          );
+
+          return completeNodeExecution({
+            ...result,
+            metadata: newMetadata,
+          });
+        });
+      },
+    });
+  });
+}

package/src/ocr-node.ts

@@ -0,0 +1,111 @@
+import { UploadistaError } from "@uploadista/core/errors";
+import {
+  completeNodeExecution,
+  createFlowNode,
+  DocumentAiPlugin,
+  NodeType,
+  type OcrTaskType,
+  type OcrResolution,
+  resolveUploadMetadata,
+} from "@uploadista/core/flow";
+import { uploadFileSchema } from "@uploadista/core/types";
+import { Effect } from "effect";
+
+export type OcrNodeParams = {
+  taskType: OcrTaskType;
+  resolution?: OcrResolution;
+  credentialId?: string;
+  referenceText?: string;
+};
+
+export function createOcrNode(
+  id: string,
+  params: OcrNodeParams,
+) {
+  return Effect.gen(function* () {
+    const documentAiService = yield* DocumentAiPlugin;
+
+    return yield* createFlowNode({
+      id,
+      name: "OCR",
+      description: "Extract text from scanned documents using AI",
+      type: NodeType.process,
+      inputSchema: uploadFileSchema,
+      outputSchema: uploadFileSchema,
+      run: ({ data: file, flowId, jobId, clientId }) => {
+        return Effect.gen(function* () {
+          const flow = {
+            flowId,
+            nodeId: id,
+            jobId,
+          };
+
+          const fileUrl = file.url;
+
+          // Validate input
+          if (!fileUrl) {
+            return yield* UploadistaError.fromCode("FLOW_NODE_ERROR", {
+              cause: "URL is required for OCR operation",
+            }).toEffect();
+          }
+
+          yield* Effect.logInfo(
+            `Starting OCR for file ${file.id} with task type: ${params.taskType}`,
+          );
+
+          // Build context for DocumentAI plugin
+          const context = {
+            clientId,
+            credentialId: params.credentialId,
+          };
+
+          // Perform OCR with error handling
+          const ocrResult = yield* documentAiService
+            .performOCR(
+              fileUrl,
+              {
+                taskType: params.taskType,
+                resolution: params.resolution,
+                referenceText: params.referenceText,
+              },
+              context
+            )
+            .pipe(
+              Effect.catchAll((error) =>
+                Effect.gen(function* () {
+                  yield* Effect.logError("Failed to perform OCR", error);
+                  return yield* UploadistaError.fromCode("FLOW_NODE_ERROR", {
+                    cause:
+                      error instanceof Error
+                        ? error.message
+                        : "Failed to perform OCR",
+                  }).toEffect();
+                }),
+              ),
+            );
+
+          const { metadata } = resolveUploadMetadata(file.metadata);
+
+          // Add OCR results to metadata
+          const newMetadata = {
+            ...file.metadata,
+            ...metadata,
+            ocrText: ocrResult.extractedText,
+            ocrFormat: ocrResult.format,
+            ocrTaskType: params.taskType,
+          };
+
+          yield* Effect.logInfo(
+            `Successfully completed OCR for file ${file.id}, extracted ${ocrResult.extractedText.length} characters`,
+          );
+
+          return completeNodeExecution({
+            ...file,
+            metadata: newMetadata,
+            flow,
+          });
+        });
+      },
+    });
+  });
+}