@stubber/virtual-worker 1.0.1 → 1.0.2

package/.env_dev

@@ -1,14 +1 @@
-# when running inside docker and connecting to tag running on host machine
-# WORKER_CONNECT_URL=http://172.17.0.1:7450/orgs/7cf3b2dc-5a49-51dd-8b4d-812a8ba6f233
-# when running using npm run dev and connecting to tag running on host machine
-# WORKER_CONNECT_URL=http://localhost:7450/orgs/7cf3b2dc-5a49-51dd-8b4d-812a8ba6f233
-# when used in docker or with npm run, connect to the devmaster tag
-WORKER_CONNECT_URL="https://virtual-workers.dev.stubber.com/orgs/7cf3b2dc-5a49-51dd-8b4d-812a8ba6f233"
-
-WORKER_SIGNATURE=069c293dc0d71dd42b32b122da62b2deae08c12c67a85224d14a7dd8c4907f21
-
-WORKER_WORKERUUID=b0bf29a8-5ebb-4b79-b464-9d35686ebc7d
-
-WORKER_GROUPS=insurance-worker,b0bf29a8-5ebb-4b79-b464-9d35686ebc7d
-
-NODE_ENV=development
+FILESERVER_URL="https://uploads.secure-link.services.dev.stubber.com/api/v1"

package/.env_devmaster

@@ -1,7 +1 @@
-WORKER_CONNECT_URL="https://virtual-workers.dev.stubber.com/orgs/7cf3b2dc-5a49-51dd-8b4d-812a8ba6f233"
-
-WORKER_SIGNATURE=069c293dc0d71dd42b32b122da62b2deae08c12c67a85224d14a7dd8c4907f21
-
-WORKER_WORKERUUID=b0bf29a8-5ebb-4b79-b464-9d35686ebc7d
-
-WORKER_GROUPS=insurance-worker,b0bf29a8-5ebb-4b79-b464-9d35686ebc7d
+FILESERVER_URL="https://uploads.secure-link.services.dev.stubber.com/api/v1"

package/app/commands/file-server/download_files.js

@@ -5,10 +5,20 @@ import { create_success } from "#app/functions/create_success.js";
 import { writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
+import { config } from "#root/config/main.js";
+
+/**
+ * @typedef FileInfoObject
+ * @property {string} fileuuid - The UUID of the file
+ * @property {string} filename - The name of the file
+ * @property {string} originalname - The original name of the file
+ * @property {string} contentType - The content type of the file
+ * @property {string} f_id - The file ID added by form fields
+ */
 
 /**
  * @typedef DownloadFilesParams
- * @property {Array<string|{fileuuid:string}>} files - The files to download
+ * @property {Array<string|FileInfoObject>} files - The files to download
  */
 
 /**
@@ -23,11 +33,11 @@ import { join } from "node:path";
  * @param {DownloadFilesParams} params
  */
 export const download_files = async (params, _stubber) => {
-  // todo: how will fileserver work now that we dont have sockets
-  const socket = _stubber?.stubber_context.socket;
-  if (!socket) {
-    return create_error_conceptual({ message: "Socket not available", details: {} });
+  const fileserver_url = config?.fileserver_url;
+  if (!fileserver_url) {
+    return create_error_conceptual({ message: "Fileserver URL not configured", details: {} });
   }
+  console.log("Fileserver URL:", fileserver_url);
 
   const { files } = params || {};
   if (!files) {
@@ -37,44 +47,50 @@ export const download_files = async (params, _stubber) => {
     return create_error_conceptual({ message: "Files must be a non-empty array", details: { files } });
   }
 
-  const uuids = [];
+  /**
+   * @type {Array<{fileuuid:string, filename:string}>}
+   */
+  const files_to_download = [];
   for (const file of files) {
     if (typeof file === "string") {
-      uuids.push(file);
+      files_to_download.push({ fileuuid: file, filename: file });
     } else if (file && typeof file === "object" && typeof file.fileuuid === "string") {
-      uuids.push(file.fileuuid);
+      files_to_download.push({
+        fileuuid: file.fileuuid,
+        filename: file.filename || file.originalname || file.fileuuid,
+      });
     } else {
       return create_error_conceptual({ message: "Invalid file entry in files array", details: { entry: file } });
     }
   }
 
-  return new Promise((resolve) => {
-    const downloaded_files = [];
+  const downloaded_files = [];
 
-    socket.emit("download_files", uuids, async (response) => {
-      if (response.success) {
-        /** @type {ResponseFile[]} */
-        const files = response.payload;
+  for (const { fileuuid, filename } of files_to_download) {
+    const file_url = `${fileserver_url}/file/uuid/${encodeURIComponent(fileuuid)}`;
 
-        for (const file of files) {
-          if (file.status == 200) {
-            // todo: support passing download location in params
-            const tmp_dir = tmpdir();
-            const file_path = join(tmp_dir, file.fileuuid);
+    try {
+      const response = await fetch(file_url);
 
-            await writeFile(file_path, file.data);
+      if (!response.ok) {
+        console.log("Error downloading file:", { fileuuid, status: response.status, status_text: response.statusText });
+        continue;
+      }
 
-            downloaded_files.push(file_path);
-          } else {
-            console.log("Error downloading file:", file);
-          }
-        }
+      const array_buffer = await response.arrayBuffer();
+      const data = Buffer.from(array_buffer);
 
-        resolve(create_success({ message: "Files downloaded successfully", payload: { downloaded_files } }));
-      } else {
-        console.log("Error downloading files:", response.error);
-        resolve(create_error_technical(response.error));
-      }
-    });
-  });
+      // todo: support passing download location in params
+      const tmp_dir = tmpdir();
+      const file_path = join(tmp_dir, filename);
+
+      await writeFile(file_path, data);
+
+      downloaded_files.push(file_path);
+    } catch (error) {
+      console.log("Error downloading file:", { fileuuid, error });
+    }
+  }
+
+  return create_success({ message: "Files downloaded successfully", payload: { downloaded_files } });
 };

package/app/commands/file-server/upload_files.js

@@ -1,8 +1,14 @@
 import { create_error_conceptual } from "#app/functions/create_error_conceptual.js";
 import { create_error_technical } from "#app/functions/create_error_technical.js";
 import { create_success } from "#app/functions/create_success.js";
+import { config } from "#root/config/main.js";
 // eslint-disable-next-line id-match
 import { readFile } from "node:fs/promises";
+// eslint-disable-next-line id-match
+import { Blob } from "buffer";
+import mime from "mime-types";
+// eslint-disable-next-line id-match
+import { fileTypeFromBuffer } from "file-type";
 
 /**
  * Upload files to the server
@@ -10,10 +16,15 @@ import { readFile } from "node:fs/promises";
  * @param {Array<string>} params.files - The path to files to upload
  */
 export const upload_files = async (params, _stubber) => {
-  /** @type {import("socket.io-client").Socket} */
-  const socket = _stubber?.stubber_context.socket;
-  if (!socket) {
-    return create_error_conceptual({ message: "Socket not available", details: {} });
+  const fileserver_url = config?.fileserver_url;
+  if (!fileserver_url) {
+    return create_error_conceptual({ message: "Fileserver URL not configured", details: {} });
+  }
+  console.log("Fileserver URL:", fileserver_url);
+
+  const orguuid = _stubber?.orguuid;
+  if (!orguuid) {
+    return create_error_conceptual({ message: "orguuid not available", details: {} });
   }
 
   const { files } = params || {};
@@ -24,34 +35,74 @@ export const upload_files = async (params, _stubber) => {
     });
   }
 
-  const uploaded_files = [];
+  const form_data = new FormData();
+  const endpoint = `${fileserver_url}/upload?orguuid=${encodeURIComponent(orguuid)}`;
 
   for (const filepath of files) {
     const filename = filepath.split("/").pop();
-    const file = await readFile(filepath); // Buffer
 
-    // Wrap socket.emit in a promise so we can await it
-    const file_info = await emit_upload(socket, { filepath, filename }, file);
+    if (!filename) {
+      return create_error_conceptual({
+        message: "Unable to determine filename from path",
+        details: { filepath },
+      });
+    }
+
+    try {
+      let mimetype = mime.lookup(filename) || "application/octet-stream";
 
-    if (file_info) {
-      uploaded_files.push(file_info);
+      const file_buffer = await readFile(filepath);
+
+      const type = await fileTypeFromBuffer(file_buffer);
+      if (type?.mime) {
+        mimetype = type.mime; // trust buffer over extension
+      }
+
+      const blob = new Blob([file_buffer], { type: mimetype });
+      form_data.append("file", blob, filename);
+    } catch (error) {
+      return create_error_technical(error);
     }
   }
 
-  return create_success({ message: "Files uploaded successfully", payload: { uploaded_files } });
-};
+  try {
+    const response = await fetch(endpoint, {
+      method: "POST",
+      body: form_data,
+    });
 
-/**
- * Wraps a socket.emit call in a Promise
- */
-function emit_upload(socket, metadata, buffer) {
-  return new Promise((resolve, reject) => {
-    socket.emit("upload_file", metadata, buffer, (response) => {
-      if (!response.success) {
-        reject(new Error(response));
-      } else {
-        resolve(response.payload);
+    if (!response.ok) {
+      let error_payload = null;
+      try {
+        error_payload = await response.json();
+      } catch (_) {
+        error_payload = { raw_error: await response.text() };
       }
+
+      return create_error_conceptual({
+        message: "Failed to upload files",
+        details: { status: response.status, error_payload },
+      });
+    }
+
+    let payload = null;
+    try {
+      payload = await response.json();
+    } catch (error) {
+      return create_error_technical(error);
+    }
+
+    if (payload?.data?.uploaded_files) {
+      payload = {
+        uploaded_files: payload.data.uploaded_files,
+      };
+    }
+
+    return create_success({
+      message: "Files uploaded successfully",
+      payload,
     });
-  });
-}
+  } catch (error) {
+    return create_error_technical(error);
+  }
+};

package/app/controllers/virtual_worker_task.js

@@ -3,7 +3,7 @@ import { task_schema } from "#lib/interfaces/http/routers/stubber_task_schema.js
 import * as z from "zod";
 import { worker_status } from "./virtual_worker_status.js";
 
-const command_schema = z.object({
+const command_schema = z.looseObject({
   commandtype: z.string().min(1),
   params: z.record(z.string(), z.any()).optional(),
   continue_on_error: z.boolean().optional(),
@@ -11,11 +11,8 @@ const command_schema = z.object({
   conditions: z.array(z.string()).optional(),
 });
 
-const params_schema = z.object({
+const params_schema = z.looseObject({
   commands: z.record(z.string(), command_schema),
-  worker: z.object({
-    group: z.string().min(1),
-  }),
 });
 
 /**

package/app/functions/create_error_technical.js

@@ -11,7 +11,7 @@ export const create_error_technical = (error) => {
       details: {
         name: error.name,
         // stack up to the second \n newline
-        stack: error.stack.split("\n").slice(0, 2).join("\n"),
+        stack: error.stack?.split("\n").slice(0, 2).join("\n"),
       },
     },
   };

package/config/main.js

@@ -0,0 +1,3 @@
+export const config = {
+  fileserver_url: process.env.FILESERVER_URL || "https://uploads.secure-link.services.stubber.com/api/v1",
+};

package/lib/interfaces/http/routers/stubber_task_schema.js

@@ -1,16 +1,16 @@
 import * as z from "zod";
 
-export const stubber_schema = z.object({
+export const stubber_schema = z.looseObject({
   orguuid: z.uuid(),
 });
 
-export const task_schema = z.object({
+export const task_schema = z.looseObject({
   tasktype: z.string().min(1),
   task_name: z.string().min(1),
   params: z.any(),
   _stubber: stubber_schema,
 });
 
-export const task_payload_schema = z.object({
+export const task_payload_schema = z.looseObject({
   task: task_schema,
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stubber/virtual-worker",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "description": "Template to easily create a node app and keep development standards",
   "main": "app.js",
   "directories": {
@@ -33,8 +33,10 @@
     "body-parser": "^1.20.3",
     "dotenv": "^16.0.3",
     "express": "^4.18.2",
+    "file-type": "^21.0.0",
     "jsonata": "^2.1.0",
     "lodash-es": "^4.17.21",
+    "mime-types": "^3.0.1",
     "net": "^1.0.2",
     "playwright": "^1.53.0",
     "socket.io-client": "^4.8.1",