@geode/opengeodeweb-front 10.34.0-rc.20 → 10.34.0-rc.21

package/.build/shared/utils/file.js

@@ -1,5 +1,7 @@
 import _ from "lodash";
 //#region shared/utils/file.ts
+const BYTES_PER_KIBIBYTE = 1024;
+const CHUNK_SIZE_BYTES = 8192 * BYTES_PER_KIBIBYTE;
 function assertFile(file) {
 	console.log("[ASSERT_FILE] Asserting file", { file });
 	if (!(file instanceof File)) throw new Error("file must be an instance of File");
@@ -9,4 +11,4 @@ function hasBody(params) {
 	return !_.isEmpty(params);
 }
 //#endregion
-export { assertFile, hasBody };
+export { BYTES_PER_KIBIBYTE, CHUNK_SIZE_BYTES, assertFile, hasBody };

package/app/stores/back.ts

@@ -133,7 +133,6 @@ export const useBackStore = defineStore("back", {
         {
           schema,
           file,
-          raw: true,
         },
         {
           ...callbacks,

package/internal/utils/upload_file.ts

@@ -1,4 +1,5 @@
 import type { JsonRpcSchema, RequestHandlers } from "@ogw_shared/utils/types.js";
+import { CHUNK_SIZE_BYTES } from "@ogw_shared/utils/file.js";
 import type { Microservice } from "./api_fetch.js";
 import { fetchRaw } from "@ogw_shared/utils/fetch_raw.js";
 import { useFeedbackStore } from "@ogw_front/stores/feedback.js";
@@ -8,11 +9,7 @@ interface UploadFileParams {
   // HTTP-flavored ("front"/"back") schemas.
   schema: JsonRpcSchema & { methods: string[] };
   file: File;
-  params?: Record<string, string | Blob>;
-  // When true, the file is sent as the raw request body instead of multipart
-  // FormData, with `params`/filename moved into the query string. See
-  // OpenGeodeWeb-Back's upload_file route, which supports both request shapes.
-  raw?: boolean;
+  params?: Record<string, string>;
 }
 
 // Loosely-typed shapes for the dynamic error/response values reported by fetchRaw.
@@ -38,7 +35,7 @@ function isFetchErrorResponseLike(value: unknown): value is FetchErrorResponseLi
 
 async function upload_file(
   microservice: Microservice,
-  { schema, file, params = {}, raw = false }: UploadFileParams,
+  { schema, file, params = {} }: UploadFileParams,
   { request_error_function, response_function, response_error_function }: RequestHandlers = {},
 ): Promise<unknown> {
   console.log("[UPLOAD_FILE] Uploading file", { schema, file });
@@ -48,68 +45,70 @@ async function upload_file(
     throw new Error("file must be an instance of File");
   }
 
-  let route = schema.$id;
-  let body: FormData | File = file;
+  const route = schema.$id;
+  const method = schema.methods.find((candidate) => candidate !== "OPTIONS");
+  const totalChunks = Math.max(1, Math.ceil(file.size / CHUNK_SIZE_BYTES));
 
-  if (raw) {
-    const queryEntries = Object.entries(params).filter(
-      (entry): entry is [string, string] => typeof entry[1] === "string",
+  function onRequestError(error: unknown): void {
+    microservice.stop_request();
+    const typedError = isFetchErrorLike(error) ? error : {};
+    feedbackStore.add_error(
+      typedError.code ?? 0,
+      route,
+      typedError.message ?? "",
+      typedError.stack ?? "",
     );
-    const query = new URLSearchParams([...queryEntries, ["filename", file.name]]);
-    route = `${route}?${query.toString()}`;
-  } else {
-    const formData = new FormData();
-    for (const [key, value] of Object.entries(params)) {
-      formData.append(key, value);
+    if (request_error_function) {
+      request_error_function(error);
+    }
+  }
+
+  function onResponseError(response: unknown): void {
+    microservice.stop_request();
+    const typedResponse = isFetchErrorResponseLike(response) ? response : {};
+    feedbackStore.add_error(
+      typedResponse.status ?? 0,
+      route,
+      typedResponse.name ?? "",
+      typedResponse.description ?? "",
+    );
+    if (response_error_function) {
+      response_error_function(response);
     }
-    formData.append("file", file);
-    body = formData;
   }
 
   microservice.start_request();
 
-  const result = await fetchRaw(
-    {
-      route,
-      method: schema.methods.find((method) => method !== "OPTIONS"),
-      params: body,
-      baseURL: microservice.base_url,
-    },
-    {
-      request_error_function(error: unknown) {
-        microservice.stop_request();
-        const typedError = isFetchErrorLike(error) ? error : {};
-        feedbackStore.add_error(
-          typedError.code ?? 0,
-          route,
-          typedError.message ?? "",
-          typedError.stack ?? "",
-        );
-        if (request_error_function) {
-          request_error_function(error);
-        }
-      },
-      response_function(data: unknown) {
-        microservice.stop_request();
-        if (response_function) {
-          response_function(data);
-        }
+  let result: unknown = undefined;
+  for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex += 1) {
+    const start = chunkIndex * CHUNK_SIZE_BYTES;
+    const chunk = file.slice(start, start + CHUNK_SIZE_BYTES);
+    const query = new URLSearchParams({
+      ...params,
+      filename: file.name,
+      chunk_index: String(chunkIndex),
+      total_chunks: String(totalChunks),
+    });
+
+    // oxlint-disable-next-line no-await-in-loop -- chunks must be sent in order; the server appends each one to the file as it arrives.
+    result = await fetchRaw(
+      {
+        route: `${route}?${query.toString()}`,
+        method,
+        params: chunk,
+        baseURL: microservice.base_url,
       },
-      response_error_function(response: unknown) {
-        microservice.stop_request();
-        const typedResponse = isFetchErrorResponseLike(response) ? response : {};
-        feedbackStore.add_error(
-          typedResponse.status ?? 0,
-          route,
-          typedResponse.name ?? "",
-          typedResponse.description ?? "",
-        );
-        if (response_error_function) {
-          response_error_function(response);
-        }
+      {
+        request_error_function: onRequestError,
+        response_error_function: onResponseError,
       },
-    },
-  );
+    );
+  }
+
+  microservice.stop_request();
+  if (response_function) {
+    response_function(result);
+  }
   return result;
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@geode/opengeodeweb-front",
-  "version": "10.34.0-rc.20",
+  "version": "10.34.0-rc.21",
   "description": "OpenSource Vue/Nuxt/Pinia/Vuetify framework for web applications",
   "homepage": "https://github.com/Geode-solutions/OpenGeodeWeb-Front",
   "bugs": {

package/server/api/local/extensions/upload.put.ts

@@ -1,98 +1,71 @@
 // Node imports
-import { finished, pipeline } from "node:stream/promises";
-import type { IncomingHttpHeaders } from "node:http";
 import { Readable } from "node:stream";
 import fs from "node:fs";
+import { pipeline } from "node:stream/promises";
 
 // Third party imports
-import {
-  type H3Event,
-  createError,
-  defineEventHandler,
-  getRequestHeaders,
-  getRequestWebStream,
-} from "h3";
-import busboy from "busboy";
+import { type H3Event, createError, defineEventHandler, getQuery, getRequestWebStream } from "h3";
 
 // Local imports
 import {
   registerExtensionFile,
   targetExtensionFilePath,
 } from "@geode/opengeodeweb-front/server/utils/app_config.ts";
+import { BYTES_PER_KIBIBYTE } from "@ogw_shared/utils/file.js";
 import { toNodeWebStream } from "@geode/opengeodeweb-front/server/utils/stream.ts";
 
+const CODE_200 = 200;
 const CODE_201 = 201;
-const BYTES_PER_KIBIBYTE = 1024;
 const MAX_FILE_MEGABYTES = 500;
 const FILE_SIZE_LIMIT = MAX_FILE_MEGABYTES * BYTES_PER_KIBIBYTE * BYTES_PER_KIBIBYTE;
 
-export default defineEventHandler(async (event: H3Event) => {
-  const writePromises: Promise<void>[] = [];
-  const savedFiles: string[] = [];
-
-  const headers: IncomingHttpHeaders = Object.fromEntries(
-    Object.entries(getRequestHeaders(event)).filter(
-      (entry: readonly [string, string | undefined]) => entry[1] !== undefined,
-    ),
-  );
-  const busboyInstance = busboy({
-    headers,
-    limits: {
-      fileSize: FILE_SIZE_LIMIT,
-      files: 1,
-    },
-  });
-  let projectName = "";
-  busboyInstance.on("field", (name, value) => {
-    console.log(`Field ${name}: ${value}`);
-    if (name === "projectName") {
-      projectName = value;
-    }
-  });
+function requiredQueryString(event: H3Event, key: string): string {
+  const value = getQuery(event)[key];
+  if (typeof value !== "string" || value.length === 0) {
+    throw createError({ statusCode: 400, message: `Missing "${key}" query parameter` });
+  }
+  return value;
+}
 
-  busboyInstance.on("file", (fieldname, fileStream, info) => {
-    if (fieldname !== "file") {
-      // Drain & ignore unwanted fields
-      fileStream.resume();
-      return;
-    }
-    const targetPath = targetExtensionFilePath(projectName, info.filename);
-    const writePromise = (async (): Promise<void> => {
-      const writeStream = fs.createWriteStream(targetPath);
-      await pipeline(fileStream, writeStream);
-      savedFiles.push(targetPath);
-      console.log("File written:", targetPath);
-    })();
-    writePromises.push(writePromise);
-    fileStream.on("limit", () => {
-      busboyInstance.destroy(new Error("File too large"));
-    });
-  });
+function requiredQueryInt(event: H3Event, key: string): number {
+  const parsed = Math.trunc(Number(requiredQueryString(event, key)));
+  if (!Number.isInteger(parsed) || parsed < 0) {
+    throw createError({ statusCode: 400, message: `Invalid "${key}" query parameter` });
+  }
+  return parsed;
+}
 
-  busboyInstance.on("filesLimit", () => {
-    busboyInstance.destroy(new Error("Too many files"));
-  });
-  busboyInstance.on("partsLimit", () => {
-    busboyInstance.destroy(new Error("Too many parts"));
-  });
+// The file is sent as a raw (non-multipart) body, split into sequential chunks the client uploads in order (see OpenGeodeWeb-Front's upload_file.ts and OpenGeodeWeb-Back's upload_file route, which use the same protocol). Each chunk is appended to a `.part` file; the last one triggers the existing zip-metadata registration.
+export default defineEventHandler(async (event: H3Event) => {
+  const projectName = requiredQueryString(event, "projectName");
+  const filename = requiredQueryString(event, "filename");
+  const totalChunks = requiredQueryInt(event, "total_chunks");
+  const chunkIndex = requiredQueryInt(event, "chunk_index");
+  if (chunkIndex >= totalChunks) {
+    throw createError({ statusCode: 400, message: "chunk_index must be less than total_chunks" });
+  }
 
   const webStream = getRequestWebStream(event);
   if (!webStream) {
     throw createError({ statusCode: 400, message: "No request body received" });
   }
-  Readable.fromWeb(toNodeWebStream(webStream)).pipe(busboyInstance);
-  await finished(busboyInstance);
-  if (writePromises.length > 0) {
-    await Promise.all(writePromises);
-    console.log("All disk writes completed");
+
+  const targetPath = targetExtensionFilePath(projectName, filename);
+  const partPath = `${targetPath}.part`;
+  const writeStream = fs.createWriteStream(partPath, { flags: chunkIndex === 0 ? "w" : "a" });
+  await pipeline(Readable.fromWeb(toNodeWebStream(webStream)), writeStream);
+
+  const { size } = await fs.promises.stat(partPath);
+  if (size > FILE_SIZE_LIMIT) {
+    await fs.promises.unlink(partPath);
+    throw createError({ statusCode: 400, message: "File too large" });
   }
-  if (savedFiles.length === 0) {
-    throw createError({ statusCode: 400, message: "No file received" });
+
+  if (chunkIndex < totalChunks - 1) {
+    return { statusCode: CODE_200 };
   }
-  await Promise.all(
-    savedFiles.map(async (file) => {
-      await registerExtensionFile(projectName, file);
-    }),
-  );
+
+  await fs.promises.rename(partPath, targetPath);
+  await registerExtensionFile(projectName, targetPath);
   return { statusCode: CODE_201 };
 });

package/shared/utils/file.ts

@@ -1,5 +1,11 @@
 import _ from "lodash";
 
+const BYTES_PER_KIBIBYTE = 1024;
+const KIBIBYTES_PER_MEBIBYTE = 1024;
+const CHUNK_SIZE_MEBIBYTES = 8;
+// Cloud hosting in front of the upload routes (Cloud Run/Firebase Hosting) enforces a hard ~32MB request body limit that no amount of server-side streaming can bypass, so uploads are always split into chunks comfortably under that cap. The single source of truth here keeps every uploader (OpenGeodeWeb-Front, Vease) and the routes that assemble the chunks in agreement.
+const CHUNK_SIZE_BYTES = CHUNK_SIZE_MEBIBYTES * KIBIBYTES_PER_MEBIBYTE * BYTES_PER_KIBIBYTE;
+
 function assertFile(file: unknown): asserts file is File {
   console.log("[ASSERT_FILE] Asserting file", { file });
   if (!(file instanceof File)) {
@@ -13,4 +19,4 @@ function hasBody(params: unknown): params is Blob | FormData | Record<string, un
   }
   return !_.isEmpty(params);
 }
-export { assertFile, hasBody };
+export { assertFile, BYTES_PER_KIBIBYTE, CHUNK_SIZE_BYTES, hasBody };