@workglow/tasks 0.0.83 → 0.0.85

package/dist/{index.js → browser.js}

@@ -178,6 +178,7 @@ var inputSchema3 = {
     response_type: {
       anyOf: [{ type: "null" }, { enum: ["json", "text", "blob", "arraybuffer"] }],
       title: "Response Type",
+      description: "The forced type of response to return. If null, the response type is inferred from the Content-Type header.",
       default: null
     },
     timeout: {
@@ -214,6 +215,16 @@ var outputSchema3 = {
     arraybuffer: {
       title: "ArrayBuffer",
       description: "The arraybuffer response"
+    },
+    metadata: {
+      type: "object",
+      properties: {
+        contentType: { type: "string" },
+        headers: { type: "object", additionalProperties: { type: "string" } }
+      },
+      additionalProperties: false,
+      title: "Response Metadata",
+      description: "HTTP response metadata including content type and headers"
     }
   },
   additionalProperties: false
@@ -281,9 +292,17 @@ class FetchUrlJob extends Job {
       signal: context.signal
     }, async (progress) => await context.updateProgress(progress));
     if (response.ok) {
+      const contentType = response.headers.get("content-type") ?? "";
+      const headers = {};
+      response.headers.forEach((value, key) => {
+        headers[key] = value;
+      });
+      const metadata = {
+        contentType,
+        headers
+      };
       let responseType = input.response_type;
       if (!responseType) {
-        const contentType = response.headers.get("content-type") ?? "";
         if (contentType.includes("application/json")) {
           responseType = "json";
         } else if (contentType.includes("text/")) {
@@ -298,13 +317,13 @@ class FetchUrlJob extends Job {
         input.response_type = responseType;
       }
       if (responseType === "json") {
-        return { json: await response.json() };
+        return { json: await response.json(), metadata };
       } else if (responseType === "text") {
-        return { text: await response.text() };
+        return { text: await response.text(), metadata };
       } else if (responseType === "blob") {
-        return { blob: await response.blob() };
+        return { blob: await response.blob(), metadata };
       } else if (responseType === "arraybuffer") {
-        return { arraybuffer: await response.arrayBuffer() };
+        return { arraybuffer: await response.arrayBuffer(), metadata };
       }
       throw new TaskInvalidInputError(`Invalid response type: ${responseType}`);
     } else {
@@ -364,6 +383,9 @@ class FetchUrlTask extends JobQueueTask {
     } else if (responseType === "arraybuffer" && staticSchema.properties.arraybuffer) {
       properties.arraybuffer = staticSchema.properties.arraybuffer;
     }
+    if (staticSchema.properties.metadata) {
+      properties.metadata = staticSchema.properties.metadata;
+    }
     if (Object.keys(properties).length === 0) {
       return staticSchema;
     }
@@ -5195,6 +5217,309 @@ var split = (input2, config = {}) => {
   return task.run();
 };
 Workflow8.prototype.split = CreateWorkflow8(SplitTask);
+// src/task/FileLoaderTask.ts
+import {
+  CreateWorkflow as CreateWorkflow9,
+  Task as Task7,
+  TaskAbortedError as TaskAbortedError2,
+  TaskRegistry as TaskRegistry9,
+  Workflow as Workflow9
+} from "@workglow/task-graph";
+import { parse as parse2 } from "csv-parse/sync";
+var inputSchema9 = {
+  type: "object",
+  properties: {
+    url: {
+      type: "string",
+      title: "URL",
+      description: "URL to load document from (http://, https://)",
+      format: "uri"
+    },
+    format: {
+      type: "string",
+      enum: ["text", "markdown", "json", "csv", "pdf", "image", "html", "auto"],
+      title: "Format",
+      description: "File format (auto-detected from URL if 'auto')",
+      default: "auto"
+    }
+  },
+  required: ["url"],
+  additionalProperties: false
+};
+var outputSchema9 = {
+  type: "object",
+  properties: {
+    text: {
+      type: "string",
+      title: "Text",
+      description: "Text content (for text, markdown, html formats)"
+    },
+    json: {
+      title: "JSON",
+      description: "Parsed JSON object or array"
+    },
+    csv: {
+      type: "array",
+      title: "CSV",
+      description: "Parsed CSV data as array of objects"
+    },
+    image: {
+      type: "string",
+      title: "Image",
+      description: "Base64 data URL for image files",
+      format: "image:data-uri"
+    },
+    pdf: {
+      type: "string",
+      title: "PDF",
+      description: "Base64 data URL for PDF files"
+    },
+    metadata: {
+      type: "object",
+      properties: {
+        url: { type: "string" },
+        format: { type: "string" },
+        size: { type: "number" },
+        title: { type: "string" },
+        mimeType: { type: "string" }
+      },
+      additionalProperties: false,
+      title: "Metadata",
+      description: "File metadata"
+    }
+  },
+  required: ["metadata"],
+  additionalProperties: false
+};
+
+class FileLoaderTask extends Task7 {
+  static type = "FileLoaderTask";
+  static category = "Document";
+  static title = "File Loader";
+  static description = "Load documents from URLs (http://, https://)";
+  static cacheable = true;
+  static inputSchema() {
+    return inputSchema9;
+  }
+  static outputSchema() {
+    return outputSchema9;
+  }
+  async execute(input2, context) {
+    const { url, format = "auto" } = input2;
+    if (context.signal.aborted) {
+      throw new TaskAbortedError2("Task aborted");
+    }
+    await context.updateProgress(0, "Detecting file format");
+    const detectedFormat = this.detectFormat(url, format);
+    const responseType = this.detectResponseType(detectedFormat);
+    if (context.signal.aborted) {
+      throw new TaskAbortedError2("Task aborted");
+    }
+    await context.updateProgress(10, `Fetching ${detectedFormat} file from ${url}`);
+    const fetchTask = context.own(new FetchUrlTask({
+      url,
+      response_type: responseType,
+      queue: false
+    }));
+    const response = await fetchTask.run();
+    if (context.signal.aborted) {
+      throw new TaskAbortedError2("Task aborted");
+    }
+    await context.updateProgress(60, "Parsing file content");
+    const title = url.split("/").pop() || url;
+    const { text, json: json2, csv, image, pdf, size, mimeType } = await this.parseResponse(response, url, detectedFormat);
+    if (context.signal.aborted) {
+      throw new TaskAbortedError2("Task aborted");
+    }
+    await context.updateProgress(100, "File loaded successfully");
+    return {
+      text,
+      json: json2,
+      csv,
+      image,
+      pdf,
+      metadata: {
+        url,
+        format: detectedFormat,
+        size,
+        title,
+        mimeType
+      }
+    };
+  }
+  parseJsonContent(content) {
+    return JSON.parse(content);
+  }
+  parseCsvContent(content) {
+    try {
+      return parse2(content, {
+        columns: true,
+        skip_empty_lines: true,
+        trim: true
+      });
+    } catch (error) {
+      throw new Error(`Failed to parse CSV: ${error}`);
+    }
+  }
+  async parseResponse(response, url, detectedFormat) {
+    const responseMimeType = response.metadata?.contentType || "";
+    if (detectedFormat === "json") {
+      if (!response.json) {
+        throw new Error(`Failed to load JSON from ${url}`);
+      }
+      const jsonData = response.json;
+      const content2 = JSON.stringify(jsonData, null, 2);
+      return {
+        text: undefined,
+        json: jsonData,
+        csv: undefined,
+        image: undefined,
+        pdf: undefined,
+        size: content2.length,
+        mimeType: responseMimeType || "application/json"
+      };
+    }
+    if (detectedFormat === "csv") {
+      const content2 = response.text || "";
+      if (!content2) {
+        throw new Error(`Failed to load CSV from ${url}`);
+      }
+      const csvData = this.parseCsvContent(content2);
+      return {
+        text: undefined,
+        json: undefined,
+        csv: csvData,
+        image: undefined,
+        pdf: undefined,
+        size: content2.length,
+        mimeType: responseMimeType || "text/csv"
+      };
+    }
+    if (detectedFormat === "image") {
+      if (!response.blob) {
+        throw new Error(`Failed to load image from ${url}`);
+      }
+      const blob = response.blob;
+      const mimeType2 = responseMimeType || (blob.type && blob.type !== "" ? blob.type : this.getImageMimeType(url));
+      const imageData = await this.blobToBase64DataURL(blob, mimeType2);
+      return {
+        text: undefined,
+        json: undefined,
+        csv: undefined,
+        image: imageData,
+        pdf: undefined,
+        size: blob.size,
+        mimeType: mimeType2
+      };
+    }
+    if (detectedFormat === "pdf") {
+      if (!response.blob) {
+        throw new Error(`Failed to load PDF from ${url}`);
+      }
+      const blob = response.blob;
+      const mimeType2 = responseMimeType || "application/pdf";
+      const pdfData = await this.blobToBase64DataURL(blob, mimeType2);
+      return {
+        text: undefined,
+        json: undefined,
+        csv: undefined,
+        image: undefined,
+        pdf: pdfData,
+        size: blob.size,
+        mimeType: mimeType2
+      };
+    }
+    const content = response.text || "";
+    if (!content) {
+      throw new Error(`Failed to load content from ${url}`);
+    }
+    const mimeType = responseMimeType || (detectedFormat === "markdown" ? "text/markdown" : detectedFormat === "html" ? "text/html" : "text/plain");
+    return {
+      text: content,
+      json: undefined,
+      csv: undefined,
+      image: undefined,
+      pdf: undefined,
+      size: content.length,
+      mimeType
+    };
+  }
+  detectResponseType(detectedFormat) {
+    let responseType = "text";
+    if (detectedFormat === "json") {
+      responseType = "json";
+    } else if (detectedFormat === "image" || detectedFormat === "pdf") {
+      responseType = "blob";
+    } else if (detectedFormat === "csv" || detectedFormat === "text" || detectedFormat === "markdown" || detectedFormat === "html") {
+      responseType = "text";
+    }
+    return responseType;
+  }
+  detectFormat(url, format) {
+    if (format === "auto") {
+      const urlLower = url.toLowerCase();
+      if (urlLower.endsWith(".md") || urlLower.endsWith(".markdown")) {
+        return "markdown";
+      } else if (urlLower.endsWith(".json")) {
+        return "json";
+      } else if (urlLower.endsWith(".csv")) {
+        return "csv";
+      } else if (urlLower.endsWith(".pdf")) {
+        return "pdf";
+      } else if (urlLower.match(/\.(jpg|jpeg|png|gif|bmp|webp|svg|ico)$/)) {
+        return "image";
+      } else if (urlLower.endsWith(".html") || urlLower.endsWith(".htm")) {
+        return "html";
+      } else {
+        return "text";
+      }
+    }
+    return format;
+  }
+  getImageMimeType(url) {
+    const urlLower = url.toLowerCase();
+    if (urlLower.endsWith(".png"))
+      return "image/png";
+    if (urlLower.endsWith(".jpg") || urlLower.endsWith(".jpeg"))
+      return "image/jpeg";
+    if (urlLower.endsWith(".gif"))
+      return "image/gif";
+    if (urlLower.endsWith(".webp"))
+      return "image/webp";
+    if (urlLower.endsWith(".bmp"))
+      return "image/bmp";
+    if (urlLower.endsWith(".svg"))
+      return "image/svg+xml";
+    if (urlLower.endsWith(".ico"))
+      return "image/x-icon";
+    return "image/jpeg";
+  }
+  async blobToBase64DataURL(blob, mimeType) {
+    if (typeof Buffer !== "undefined") {
+      const arrayBuffer = await blob.arrayBuffer();
+      const buffer = Buffer.from(arrayBuffer);
+      return `data:${mimeType};base64,${buffer.toString("base64")}`;
+    }
+    return new Promise((resolve, reject) => {
+      const reader = new FileReader;
+      reader.onloadend = () => {
+        const result = reader.result;
+        if (result.startsWith("data:;base64,")) {
+          resolve(`data:${mimeType};base64,${result.substring(13)}`);
+        } else {
+          resolve(result);
+        }
+      };
+      reader.onerror = reject;
+      reader.readAsDataURL(blob);
+    });
+  }
+}
+TaskRegistry9.registerTask(FileLoaderTask);
+var fileLoader = (input2, config) => {
+  return new FileLoaderTask(input2, config).run();
+};
+Workflow9.prototype.fileLoader = CreateWorkflow9(FileLoaderTask);
 export {
   split,
   process,
@@ -5202,6 +5527,7 @@ export {
   lambda,
   json,
   javaScript,
+  fileLoader,
   fetchUrl,
   delay,
   debugLog,
@@ -5210,10 +5536,11 @@ export {
   LambdaTask,
   JsonTask,
   JavaScriptTask,
+  FileLoaderTask,
   FetchUrlTask,
   FetchUrlJob,
   DelayTask,
   DebugLogTask
 };
 
-//# debugId=5A48AA5322FF43C164756E2164756E21
+//# debugId=0A217C65AC31B63164756E2164756E21