geonix 1.20.8 → 1.22.1

package/exports.js

@@ -11,4 +11,6 @@ export { Request, Subscribe, Publish } from "./src/Request.js";
 export { RequestOptions } from "./src/RequestOptions.js";
 export { picoid as randomID } from "./src/Util.js";
 
-export { stats as StreamStats } from "./src/Stream.js";
+export { stats as StreamStats } from "./src/Stream.js";
+
+export { parseMultipart } from "./src/Util.js";

package/index.d.ts

@@ -130,7 +130,11 @@ export class RequestOptionsClass {
 export function RequestOptions(options: any): RequestOptionsClass;
 export class Service {
     static serviceClasses: any[];
-    static start(options?: {}): void;
+    /**
+     *
+     * @param {ServiceOptions} options
+     */
+    static start(options?: ServiceOptions): void;
     connections: Map<any, any>;
     $createConnection(streamId: any): Promise<boolean>;
     $getEnv(): {
@@ -148,17 +152,6 @@ export class Service {
     $getServiceInfo(): {};
     #private;
 }
-export type ServiceOptions = {
-    middleware: {
-        json: boolean;
-        raw: boolean;
-        cookies: boolean;
-    };
-    /**
-     * Enable full beacon
-     */
-    fullBeacon: boolean;
-};
 /**
  * Converts data to stream
  *
@@ -174,6 +167,44 @@ export function streamToString(object: any, encoding: any): Promise<any>;
 export function streamToJSON(object: any): Promise<any>;
 export const stats: {};
 export const activeStreams: {};
+type parseMultipartOptions = {
+    maxFileSize: number;
+    maxFiles: number;
+    useMemory: boolean;
+};
+type ParsedMultiPart = {
+    /**
+     * Field name of the part (extracted from Content-Disposition)
+     */
+    "name-": string;
+    /**
+     * - Filename of the part (extracted from Content-Disposition)
+     */
+    filename: string;
+    /**
+     * - Headers of the part
+     */
+    headers: Object;
+    /**
+     * - Path to the file when useMemory is false
+     */
+    bodyFile: string;
+    /**
+     * - Readable stream of the part
+     */
+    body: Readable;
+};
+type ServiceOptions = {
+    middleware: {
+        json: boolean;
+        raw: boolean;
+        cookies: boolean;
+    };
+    /**
+     * Enable full beacon
+     */
+    fullBeacon: boolean;
+};
 /**
  * Parse nats:// URL
  * @param {string} url
@@ -182,7 +213,22 @@ export const activeStreams: {};
 export function parseURL(url: string): any;
 export function getFirstItemFromAsyncIterable(asyncIterable: any): Promise<any>;
 export function getNetworkAddresses(): any[];
+/**
+ *
+ *
+ * @param {Request} req
+ * @param {parseMultipartOptions} options
+ * @returns ParsedMultiPart[]
+ */
+export function parseMultipart(req: Request, _options: any): Promise<{
+    headers: {};
+    bodyFile: any;
+    body: any;
+}[]>;
+export function tempFilename(): any;
+export function randomSafeId(size?: number): string;
 export function sleep(delay: number): Promise<any>;
+export function nextTick(): Promise<any>;
 export function picoid(size?: number): any;
 export function hash(data: string | Buffer): any;
 export function createServerAtPort(port: number, pkg: Object, handler: Function): Promise<any>;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "geonix",
-    "version": "1.20.8",
+    "version": "1.22.1",
     "type": "module",
     "description": "",
     "bin": {
@@ -16,8 +16,8 @@
     "author": "Davor Tarandek <dtarandek@tria.hr>",
     "license": "MIT",
     "dependencies": {
-        "cookie-parser": "1.4.6",
-        "express": "4.21.0",
+        "cookie-parser": "1.4.7",
+        "express": "4.21.1",
         "express-async-errors": "3.1.1",
         "express-ws": "5.0.2",
         "nats": "2.28.2",

package/src/Gateway.js

@@ -102,8 +102,11 @@ export class Gateway {
         this.#api.use(raw, (req, res, next) => {
             stats.requests++;
 
-            if (this.#router) { this.#router(req, res, next); }
-            else { next(); }
+            if (this.#router) {
+                this.#router(req, res, next);
+            } else {
+                next();
+            }
         });
 
         this.#api.use((req, res, next) => {

package/src/Request.js

@@ -143,7 +143,7 @@ export async function directRequest(identifier, method, args, context, options,
  * @param {string|number} payload 
  */
 export async function Publish(subject, payload) {
-    connection.publishRaw(`gx.sub.${subject}`, Buffer.from(payload));
+    connection.publishRaw(subject, Buffer.from(payload));
 }
 
 /**
@@ -158,7 +158,7 @@ export async function Subscribe(subject, callback) {
         return;
     }
 
-    const subscription = await connection.subscribe(`gx.sub.${subject}`);
+    const subscription = await connection.subscribe(subject);
     for await (const event of subscription) {
         callback(event.data);
     }

package/src/Service.js

@@ -20,12 +20,7 @@ const raw = express.raw({ type: "*/*", limit: "100mb" });
 const cookies = cookieParser();
 
 /**
- * @typedef {Object} ServiceOptions
- * @property {Object} middleware
- * @property {boolean} middleware.json Enable JSON middleware
- * @property {boolean} middleware.raw Enable RAW middleware
- * @property {boolean} middleware.cookies Enable cookies middleware
- * @property {boolean} fullBeacon Enable full beacon
+ * @type ServiceOptions
  */
 const defaultServiceOptions = {
     middleware: {
@@ -40,6 +35,10 @@ export class Service {
 
     static serviceClasses = [];
 
+    /**
+     * 
+     * @param {ServiceOptions} options 
+     */
     static start(options = {}) {
         if (!this.serviceClasses.includes(this.prototype.constructor.name)) {
             this.serviceClasses.push(this.prototype.constructor.name);
@@ -228,7 +227,7 @@ export class Service {
     }
 
     async #sub(subject, handler) {
-        const subscription = await connection.subscribe(`gx.sub.${subject}`);
+        const subscription = await connection.subscribe(subject);
         const processor = async () => {
             for await (const event of subscription) {
                 handler(event.data);

package/src/Types.js

@@ -0,0 +1,24 @@
+/**
+ * @typedef {Object} parseMultipartOptions
+ * @property {number} maxFileSize
+ * @property {number} maxFiles
+ * @property {boolean} useMemory
+ */
+
+/**
+ * @typedef {Object} ParsedMultiPart
+ * @property {string} name- Field name of the part (extracted from Content-Disposition)
+ * @property {string} filename - Filename of the part (extracted from Content-Disposition)
+ * @property {Object} headers - Headers of the part
+ * @property {string} bodyFile - Path to the file when useMemory is false
+ * @property {Readable} body - Readable stream of the part
+ */
+
+/**
+ * @typedef {Object} ServiceOptions
+ * @property {Object} middleware
+ * @property {boolean} middleware.json Enable JSON middleware
+ * @property {boolean} middleware.raw Enable RAW middleware
+ * @property {boolean} middleware.cookies Enable cookies middleware
+ * @property {boolean} fullBeacon Enable full beacon
+ */

package/src/Util.js

@@ -4,9 +4,13 @@ import { readFile } from "fs/promises";
 import { join } from "path";
 import { Transform } from "node:stream";
 import { networkInterfaces } from "os";
+import { Readable } from "stream";
 import * as http from "http";
 import * as https from "https";
 import * as net from "net";
+import { createWriteStream, createReadStream } from "fs";
+import { unlink } from "fs/promises";
+import { tmpdir } from "os";
 
 /**
  * Wait for {delay} ms
@@ -15,6 +19,13 @@ import * as net from "net";
  */
 export const sleep = delay => new Promise(resolve => setTimeout(resolve, delay));
 
+/**
+ * Wait for next tick
+ * 
+ * @returns 
+ */
+export const nextTick = () => sleep(0);
+
 /**
  * Parse nats:// URL
  * @param {string} url 
@@ -228,4 +239,148 @@ export function getNetworkAddresses() {
     }
 
     return list;
+}
+
+/**
+ * 
+ * 
+ * @param {Request} req 
+ * @param {parseMultipartOptions} options
+ * @returns ParsedMultiPart[]
+ */
+export async function parseMultipart(req, _options) {
+    if (!req.headers["content-type"]?.startsWith("multipart/form-data")) {
+        throw new Error("Invalid content type (multipart/form-data expected)");
+    }
+
+    const BUFFER_SIZE = 1024 * 1024;
+    const END_OF_HEADERS = Buffer.from("\r\n\r\n");
+    const options = {
+        useMemory: false,
+        ..._options
+    };
+    const parts = [];
+    let stream = req;
+
+    // if body is present, create a stream from it and use memory
+    if (req.body) {
+        stream = Readable.from(req.body);
+        options.useMemory = true;
+    }
+
+    const boundary = Buffer.from("\r\n--" + req.headers["content-type"].split("boundary=")[1]);
+
+    await nextTick();
+
+    let lastChunk = Buffer.from("\r\n");
+    let activePart;
+    let done = false;
+
+    while (stream.readable) {
+        // next next chunk
+        let chunk = stream.read(BUFFER_SIZE);
+
+        if (!chunk) {
+            await nextTick();
+            continue;
+        }
+
+        let combined = Buffer.concat([lastChunk, chunk]);
+
+        while (combined.length >= boundary.length + 2) {
+            const boundaryIndex = combined.indexOf(boundary);
+            const isLastBoundary = combined[boundaryIndex + boundary.length] === 45 && combined[boundaryIndex + boundary.length + 1] === 45;
+
+            if (boundaryIndex > -1) {
+                if (boundaryIndex > 0) {
+                    if (options.useMemory) {
+                        activePart.body.push(combined.slice(0, boundaryIndex));
+                    } else {
+                        activePart.body.write(combined.slice(0, boundaryIndex));
+                    }
+                }
+
+                if (isLastBoundary) {
+                    combined = combined.slice(boundaryIndex + boundary.length + 2);
+                    done = true;
+                    break;
+                }
+
+                // create new part
+                const bodyFile = tempFilename();
+                activePart = {
+                    headers: {},
+                    bodyFile: options.useMemory ? undefined : bodyFile,
+                    body: options.useMemory ? [] : createWriteStream(bodyFile, { flags: "wx" })
+                };
+                parts.push(activePart);
+
+                const endOfHeaders = combined.indexOf(END_OF_HEADERS, boundaryIndex);
+                activePart.headers = combined
+                    .slice(boundaryIndex + boundary.length + 2, endOfHeaders).toString()
+                    .split("\r\n")
+                    .reduce((acc, val) => {
+                        const [header, value] = val.split(": ");
+                        acc[header.toLowerCase()] = value;
+                        return acc;
+                    }, {});
+
+                combined = combined.slice(endOfHeaders + END_OF_HEADERS.length);
+            }
+
+            lastChunk = combined;
+        }
+
+        // there's no boundary in the chunk, add it to active part
+        if (activePart && lastChunk.length > 0 && !done) {
+            if (options.useMemory) {
+                activePart.body.push(lastChunk);
+            } else {
+                activePart.body.write(lastChunk);
+            }
+            lastChunk = Buffer.alloc(0);
+        }
+    }
+
+    for (const part of parts) {
+        // extract name and filename from content-disposition header
+        if (part.headers["content-disposition"]) {
+            const [, name] = part.headers["content-disposition"].match(/name="([^"]+)"/);
+            const [, filename] = part.headers["content-disposition"].match(/filename="([^"]+)"/) || [];
+            part.name = name ?? null;
+            part.filename = filename ?? null;
+        }
+
+        if (options.useMemory) {
+            part.body = Readable.from(Buffer.concat(part.body));
+            delete part.bodyFile;
+        } else {
+            part.body.end();
+            part.body = createReadStream(part.bodyFile);
+            part.body.on("close", async () => {
+                try {
+                    await unlink(part.bodyFile);
+                } catch {
+                    // ignore errors                        
+                }
+            });
+        }
+    }
+
+    return parts;
+}
+
+export function tempFilename() {
+    return join(tmpdir(), `${randomSafeId(12)}.gxtmp`);
+}
+
+export function randomSafeId(size = 12) {
+    const charset = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+    let result = "";
+
+    for (let i = 0; i < size; i++) {
+        result += charset.charAt(Math.floor(Math.random() * charset.length));
+    }
+
+    return result;
 }

package/test/gateway.js

@@ -1,13 +1,31 @@
-import { Gateway, Service } from "../exports.js";
+import { Gateway, Service, streamToBuffer } from "../exports.js";
+import { parseMultipart } from "../src/Util.js";
 
 class TestService extends Service {
 
     "GET /"(req, res) {
         res.send("Hello World");
     }
+
+    async "POST /upload"(req, res) {
+        const parts = await parseMultipart(req, { useMemory: false });
+
+        for (const part of parts) {
+            console.log(part.body);
+        }
+
+        res.send("OK");
+    }
+
 }
 
-TestService.start();
+TestService.start({
+    middleware: {
+        raw: true,
+        json: false,
+        cookies: false,
+    }
+});
 Gateway.start({
     beforeRequest: (req, res) => {
         res.set("X-Test", "Test");