geonix 1.20.0 → 1.20.2

package/src/Service.js

@@ -1,11 +1,12 @@
 import { codec, connection } from "./Connection.js";
-import { picoid, sleep, hash, getSecondsSinceMidnight, OverlayObject, GeonixVersion, getFirstItemFromAsyncIterable } from "./Util.js";
+import { picoid, sleep, hash, getSecondsSinceMidnight, OverlayObject, GeonixVersion, getFirstItemFromAsyncIterable, getNetworkAddresses } from "./Util.js";
 import { webserver } from "./WebServer.js";
 import { createConnection } from "net";
 import { EOL } from "os";
 import cookieParser from "cookie-parser";
 import express from "express";
 import { isStream, streamToString } from "./Stream.js";
+import { logger } from "./Logger.js";
 
 const protectedMethodNames = ["constructor", "onStart"];
 const endpointMatcher = /^((?<options>.+)\|)?(?<verb>WS|SUB|GET|POST|PATCH|PUT|DELETE|HEAD|OPTIONS|ALL)\s(?<url>.*)/;
@@ -15,14 +16,33 @@ const ERROR_END_DELIMITER = "-".repeat(40);
 
 const json = express.json({ limit: "100mb" });
 const raw = express.raw({ type: "*/*", limit: "100mb" });
+const cookies = cookieParser();
+
+/**
+ * @typedef {Object} ServiceOptions
+ * @property {Object} middleware
+ * @property {boolean} middleware.json Enable JSON middleware
+ * @property {boolean} middleware.raw Enable RAW middleware
+ * @property {boolean} middleware.cookies Enable cookies middleware
+ * @property {boolean} fullBeacon Enable full beacon
+ */
+const defaultServiceOptions = {
+    middleware: {
+        json: true,
+        raw: true,
+        cookies: true
+    },
+    fullBeacon: true
+};
 
 export class Service {
 
     static serviceClasses = [];
 
     static start(options = {}) {
-        if (!this.serviceClasses.includes(this.prototype.constructor.name))
+        if (!this.serviceClasses.includes(this.prototype.constructor.name)) {
             this.serviceClasses.push(this.prototype.constructor.name);
+        }
 
         const instance = new this();
         instance.#start(options);
@@ -31,21 +51,28 @@ export class Service {
     // ---------------------------------------------------------------------------------------------    
 
     #me = {};
-    #options = {};
+    #options = defaultServiceOptions;
 
     async #start(options = {}) {
-        this.#options = options;
+        this.#options = { ...this.#options, ...options };
 
         await webserver.waitUntilReady();
         await connection.waitUntilReady();
 
-        const fields = Object.getOwnPropertyNames(this).filter(isEndpointFilter).concat(Object.getOwnPropertyNames(this.constructor.prototype));
+        const fields = Object.getOwnPropertyNames(this)
+            .filter(isEndpointFilter)
+            .concat(Object.getOwnPropertyNames(this.constructor.prototype));
 
         // preserve order of endpoints as defined in the source
         const serviceSource = this.constructor.toString().split("\n");
         fields.sort((a, b, ia = -1, ib = -1) => {
-            for (let line = 0; line < serviceSource.length; line++) ia = serviceSource[line].includes(a) ? line : ia;
-            for (let line = 0; line < serviceSource.length; line++) ib = serviceSource[line].includes(b) ? line : ib;
+            for (let line = 0; line < serviceSource.length; line++) {
+                ia = serviceSource[line].includes(a) ? line : ia;
+            }
+
+            for (let line = 0; line < serviceSource.length; line++) {
+                ib = serviceSource[line].includes(b) ? line : ib;
+            }
             return ia - ib;
         });
 
@@ -65,7 +92,7 @@ export class Service {
             // geonix version
             gx: GeonixVersion,
             // IP addresses
-            a: webserver.getAddresses().map(address => `${address}:${webserver.getPort()}`)
+            a: getNetworkAddresses().map(address => `${address}:${webserver.getPort()}`)
         };
 
         // check if method takes context as first argument
@@ -79,7 +106,7 @@ export class Service {
         this.#directListener();
         this.#webserver();
 
-        console.log("gx.service.start", this.#me.n, this.#me.v);
+        logger.info("gx.service.start", this.#me.n, this.#me.v);
 
         // execute onStart method, if present
         if (this.onStart) {
@@ -93,7 +120,11 @@ export class Service {
      */
     async #beacon() {
         while (true) {
-            connection.publish("gx2.beacon", { i: this.#me.i });
+            if (this.#options.fullBeacon) {
+                connection.publish("gx2.beacon", this.#me);
+            } else {
+                connection.publish("gx2.beacon", { i: this.#me.i });
+            }
             await sleep(1000);
         }
     }
@@ -108,11 +139,9 @@ export class Service {
         for await (let event of subscription) {
             let call = codec.decode(event.data);
 
-            if (isStream(call))
-                call = JSON.parse(await streamToString(call));
+            if (isStream(call)) { call = JSON.parse(await streamToString(call)); }
 
-            if (call.$r && call.p)
-                this.#onCall(call.p, (json) => connection.publish(call.$r, json));
+            if (call.$r && call.p) { this.#onCall(call.p, (json) => connection.publish(call.$r, json)); }
         }
     }
 
@@ -126,11 +155,13 @@ export class Service {
         for await (let event of subscription) {
             let call = codec.decode(event.data);
 
-            if (isStream(call))
+            if (isStream(call)) {
                 call = JSON.parse(await streamToString(call));
+            }
 
-            if (call.$r && call.p)
+            if (call.$r && call.p) {
                 this.#onCall(call.p, (json) => connection.publish(call.$r, json));
+            }
         }
     }
 
@@ -142,11 +173,24 @@ export class Service {
         const endpoints = this.#me.m
             .filter(isEndpointFilter);
 
-        if (!endpoints || endpoints.length === 0)
+        if (!endpoints || endpoints.length === 0) {
             return;
+        }
 
         const router = webserver.router();
-        router.use(json, raw, cookieParser());
+
+        // setup defualt middlewares
+        if (this.#options.middleware.json) {
+            router.use(json);
+        }
+        if (this.#options.middleware.raw) {
+            router.use(raw);
+        }
+        if (this.#options.middleware.cookies) {
+            router.use(cookies);
+        }
+
+        // register endpoints
         for (let endpoint of endpoints) {
             let { verb, url: uri } = endpointMatcher.exec(endpoint)?.groups || {};
             verb = verb.toLowerCase();
@@ -158,8 +202,9 @@ export class Service {
 
             handlers = [...handlersBefore, ...handlers, ...handlersAfter];
 
-            for (let n = 0; n < handlers.length; n++)
+            for (let n = 0; n < handlers.length; n++) {
                 handlers[n] = handlers[n].bind(this);
+            }
 
             switch (verb) {
                 case "ws":
@@ -180,8 +225,9 @@ export class Service {
     async #sub(subject, handler) {
         const subscription = await connection.subscribe(`gx.sub.${subject}`);
         const processor = async () => {
-            for await (const event of subscription)
+            for await (const event of subscription) {
                 handler(event.data);
+            }
         };
         processor();
     }
@@ -198,13 +244,15 @@ export class Service {
         const method = this[methodName];
         let _args = args;
 
-        if (!method)
+        if (!method) {
             return respond({ e: `unknown method (${this.#me.n}.${methodName})` });
+        }
 
         try {
             // inject context as first argument
-            if (method.takesContext)
+            if (method.takesContext) {
                 _args = [OverlayObject(context, { caller, me: this.#me }), ..._args];
+            }
 
             respond({ r: await method.apply(this, _args) });
         } catch (e) {

package/src/Stream.js

@@ -1,10 +1,13 @@
 import { Readable } from "stream";
 import { connection } from "./Connection.js";
-import { picoid, StreamChunker } from "./Util.js";
+import { getFirstItemFromAsyncIterable, getNetworkAddresses, picoid, StreamChunker } from "./Util.js";
+import { logger } from "./Logger.js";
+import { webserver } from "./WebServer.js";
 
 const CHUNK_SIZE = 1024 * 128;
 
 export const stats = {};
+export const activeStreams = {};
 
 /**
  * Converts data to stream
@@ -14,15 +17,17 @@ export const stats = {};
  * @returns 
  */
 export function Stream(data, tag = "_") {
-    if (isStream(data))
+    if (isStream(data)) {
         return data;
+    }
 
     const id = picoid();
     let readable = data;
 
     // convert Buffer or string to a Readable
-    if (!(readable.pipe && readable.readable))
+    if (!(readable.pipe && readable.readable)) {
         readable = Readable.from(Buffer.from(data));
+    }
 
     // split the stream is smaller chunks
     const transform = StreamChunker(Math.min(connection.getMaxPayloadSize(), CHUNK_SIZE));
@@ -30,23 +35,38 @@ export function Stream(data, tag = "_") {
     readable = transform;
 
     stats[tag] = stats[tag] !== undefined ? stats[tag] + 1 : 1;
+    activeStreams[id] = readable;
 
-    const controlHandler = async () => {
+    // NATS handler
+    (async () => {
         const control = await connection.subscribe(`gx2.stream.${id}.a`, { max: 1 });
 
-        for await (const event of control)
-            if (event.data.length === 0) {
-                readable.on("data", chunk => connection.publishRaw(`gx2.stream.${id}.b`, chunk));
-                readable.on("close", () => {
-                    connection.publishRaw(`gx2.stream.${id}.b`);
-                    stats[tag]--;
-                });
-            }
+        const event = await getFirstItemFromAsyncIterable(control);
+        if (activeStreams[id] !== undefined && event.data.length === 0) {
+            // remove stream from the list
+            delete activeStreams[id];
+
+            // kickstart the stream
+            readable.on("data", chunk => connection.publishRaw(`gx2.stream.${id}.b`, chunk));
+            readable.on("close", () => {
+                connection.publishRaw(`gx2.stream.${id}.b`);
+                stats[tag]--;
+            });
+        }
+    })();
+
+    const result = {
+        $: "stream",
+        id
     };
 
-    controlHandler();
+    // get the port and addresses of the webserver
+    const addresses = webserver.getPort() ? getNetworkAddresses().map(address => `${address}:${webserver.getPort()}`) : undefined;
+    if (addresses) {
+        result.a = addresses;
+    }
 
-    return { $: "stream", id };
+    return result;
 }
 
 export function isStream(object) {
@@ -54,23 +74,43 @@ export function isStream(object) {
 }
 
 export async function getReadable(object) {
-    if (!isStream(object))
+    if (!isStream(object)) {
         return object;
+    }
+
+    // get stream via HTTP
+    if (object.a.length > 0) {
+        for (const address of object.a) {
+            try {
+                const uri = `http://${address}/!!_____stream/${object.id}`;
+                const response = await fetch(uri);
+
+                if (response.status === 200) {
+                    return Readable.fromWeb(response.body);
+                }
+            } catch {
+                // ignore errors
+            }
+        }
+    }
 
+    // get stream via NATS
     const readable = new Readable({ read: () => null });
     const subscription = await connection.subscribe(`gx2.stream.${object.id}.b`);
 
     const dataHandler = async () => {
-        for await (const event of subscription)
+        for await (const event of subscription) {
             try {
                 if (event.data.length === 0) {
                     readable.push(null);
                     subscription.drain();
-                } else
+                } else {
                     readable.push(event.data);
+                }
             } catch (e) {
-                console.error("Stream.getReadable.dataHandler.error:", e);
+                logger.error("Stream.getReadable.dataHandler.error:", e);
             }
+        }
     };
     dataHandler();
 
@@ -82,17 +122,18 @@ export async function getReadable(object) {
 
 export async function streamToBuffer(object) {
     let readable = object;
-    if (isStream(readable))
+    if (isStream(readable)) {
         readable = await getReadable(readable);
+    }
 
     return Buffer.concat(await readable.toArray());
 }
 
 export async function streamToString(object) {
     let readable = object;
-    if (isStream(readable))
+    if (isStream(readable)) {
         readable = await getReadable(readable);
+    }
 
     return Buffer.concat(await readable.toArray()).toString();
-}
-
+}

package/src/Util.js

@@ -3,6 +3,7 @@ import { URL, fileURLToPath } from "url";
 import { readFile } from "fs/promises";
 import { join } from "path";
 import { Transform } from "node:stream";
+import { networkInterfaces } from "os";
 import * as http from "http";
 import * as https from "https";
 import * as net from "net";
@@ -60,7 +61,6 @@ export const createServerAtPort = (port, pkg, handler) =>
     new Promise((resolve) => {
         const server = pkg.createServer(handler);
         server.on("error", (_error) => {
-            // console.log('error', error.message)
             resolve(null);
         });
         server.listen(port, () => {
@@ -78,14 +78,16 @@ export const createServerAtPort = (port, pkg, handler) =>
  * @returns 
  */
 export const createServerAtFreePort = async (pkg, handler, start = 30000, poolSize = 20000) => {
-    for (let port = start; port < start + poolSize; port++)
+    for (let port = start; port < start + poolSize; port++) {
         try {
             const result = await createServerAtPort(port, pkg, handler);
-            // console.log(port, '=', result)
-            if (result) return result;
+            if (result) {
+                return result;
+            }
         } catch {
             // silenty ignore errors
         }
+    }
 };
 
 /**
@@ -121,10 +123,11 @@ export const getFunctionParams = (fn) => {
     const endParenthesisPosition = code.indexOf(")");
     let params;
 
-    if (endParenthesisPosition != -1)
+    if (endParenthesisPosition != -1) {
         params = code.substring(code.indexOf("(") + 1, endParenthesisPosition);
-    else
+    } else {
         params = code.substring(0, code.indexOf("=>"));
+    }
 
     params = params
         // cleanup spaces
@@ -135,8 +138,9 @@ export const getFunctionParams = (fn) => {
     // remove potential default values
     for (let index = 0; index < params.length; index++) {
         const defaultValueAssignmentPosition = params[index].indexOf("=");
-        if (defaultValueAssignmentPosition != -1)
+        if (defaultValueAssignmentPosition != -1) {
             params[index] = params[index].substring(0, defaultValueAssignmentPosition - 1);
+        }
     }
 
     return params;
@@ -153,8 +157,10 @@ export const proxyHttp = (target, req, res) =>
         const protocol = req.protocol === "https" ? https : http;
         const proxyReq = protocol.request(remoteTarget, options, (proxyRes) => {
             res.status(proxyRes.statusCode);
-            for (const header in proxyRes.headers)
+            for (const header in proxyRes.headers) {
                 res.set(header, proxyRes.headers[header]);
+            }
+
             proxyRes.pipe(res);
         });
         proxyReq.on("error", (error) => reject(error));
@@ -207,4 +213,19 @@ export async function getFirstItemFromAsyncIterable(asyncIterable) {
     const iterator = asyncIterable[Symbol.asyncIterator]();
     const result = await iterator.next();
     return result.value;
+}
+
+export function getNetworkAddresses() {
+    const list = [];
+    const interfaces = networkInterfaces();
+
+    for (let interfaceAddresses of Object.values(interfaces)) {
+        for (let addressObject of interfaceAddresses) {
+            if (addressObject.family === "IPv4") {
+                list.push(addressObject.address);
+            }
+        }
+    }
+
+    return list;
 }

package/src/WebServer.js

@@ -1,11 +1,12 @@
 import "express-async-errors";
 import express, { Router } from "express";
 import expressWs from "express-ws";
-import { networkInterfaces } from "os";
 import { createServerAtFreePort, createServerAtPort, sleep } from "./Util.js";
 import * as http from "http";
 import { Service } from "./Service.js";
 import * as path from "path";
+import { logger } from "./Logger.js";
+import { activeStreams } from "./Stream.js";
 
 export const HEALTH_CHECK_ENDPOINT = "/pA4vY7fT9oG5aI8cA4yV3qW5fP9qR1vI";
 
@@ -16,12 +17,14 @@ export const ServeStatic = (root, options = {}) => {
     router.use((req, res, next) => {
         if (options.root) {
             // remove trailing slash
-            if (options.root.endsWith("/"))
+            if (options.root.endsWith("/")) {
                 options.root = options.root.slice(0, -1);
+            }
 
             // replace root prefix
-            if (req.url.startsWith(options.root))
+            if (req.url.startsWith(options.root)) {
                 req.url = req.url.replace(options.root, "");
+            }
         }
 
         next();
@@ -29,11 +32,12 @@ export const ServeStatic = (root, options = {}) => {
 
     router.use(express.static(root, options));
 
-    if (options.indexOn404)
+    if (options.indexOn404) {
         router.get("*", (req, res) => {
-            console.log(path.join(absoluteRoot, "index.html"));
+            logger.info(path.join(absoluteRoot, "index.html"));
             res.sendFile(path.join(absoluteRoot, "index.html"));
         });
+    }
 
     return router;
 };
@@ -47,31 +51,43 @@ class WebServer {
     #started = false;
 
     async start() {
-        if (this.#started)
-            return;
+        if (this.#started) { return; }
 
         this.#started = true;
 
-        let port, server;
+        let srv;
         if (process.env.LOCAL_PORT) {
-            ({ server, port } = await createServerAtPort(process.env.LOCAL_PORT, http, this.#app));
+            srv = await createServerAtPort(process.env.LOCAL_PORT, http, this.#app);
         } else {
-            ({ server, port } = await createServerAtFreePort(http, this.#app));
+            srv = await createServerAtFreePort(http, this.#app);
+        }
+
+        if (!srv) {
+            throw new Error("gx.webserver.start: unable to start");
         }
 
-        this.#server = server;
-        this.#port = port;
+        this.#server = srv.server;
+        this.#port = srv.port;
 
-        expressWs(this.#app, server);
+        expressWs(this.#app, srv.server);
+
+        // stream endpoint
+        this.#app.get("/!!_____stream/:id", (req, res) => {
+            const id = req.params.id;
+
+            if (activeStreams[id]) {
+                res.status(200);
+                activeStreams[id].pipe(res);
+                delete activeStreams[id];
+            } else {
+                res.status(404).send({ error: 404 });
+            }
+        });
 
         this.#app.get(HEALTH_CHECK_ENDPOINT, (req, res) => {
             res.send({ status: "healthy", services: Service.serviceClasses });
         });
 
-        // this.#app.use((req, res, next) => {
-        //     next()
-        // })
-
         // wait for 2 seconds and then set fall-through handler
         // this should provide more than enough time to start all the services
         setTimeout(() => {
@@ -88,23 +104,11 @@ class WebServer {
         this.#app.disable("x-powered-by");
         this.#app.disable("etag");
 
-        console.log(`gx.webserver.start: listening on http://127.0.0.1:${this.#port}`);
+        logger.info(`gx.webserver.start: listening on http://127.0.0.1:${this.#port}`);
 
         this.#ready = true;
     }
 
-    getAddresses() {
-        const list = [];
-        const interfaces = networkInterfaces();
-
-        for (let interfaceAddresses of Object.values(interfaces))
-            for (let addressObject of interfaceAddresses)
-                if (addressObject.family === "IPv4")
-                    list.push(addressObject.address);
-
-        return list;
-    }
-
     getPort() {
         return this.#port;
     }
@@ -118,14 +122,15 @@ class WebServer {
     async waitUntilReady() {
         await this.start();
 
-        while (!this.#ready)
+        while (!this.#ready) {
             await sleep(100);
+        }
     }
 
     stop() {
         if (this.#server) {
             this.#server.close();
-            console.log("gx.webserver.stop");
+            logger.info("gx.webserver.stop");
         }
     }
 

package/test/gateway.js

@@ -2,14 +2,14 @@ import { Gateway, Service } from "../exports.js";
 
 class TestService extends Service {
 
-    'GET /'(req, res) {
-        res.send('Hello World')
+    "GET /"(req, res) {
+        res.send("Hello World");
     }
 }
 
-TestService.start()
+TestService.start();
 Gateway.start({
     beforeRequest: (req, res) => {
-        res.set('X-Test', 'Test')
+        res.set("X-Test", "Test");
     }
-})
+});

package/test/stream.js

@@ -1,38 +1,43 @@
-import { randomBytes } from 'node:crypto'
-import { Stream, getReadable, connection } from 'geonix'
-import { createWriteStream, readFileSync } from 'node:fs'
-import { createHash } from 'node:crypto'
-import { pipeline } from 'node:stream/promises'
+import { randomBytes } from "node:crypto";
+import { Stream, getReadable, connection } from "geonix";
+import { createWriteStream, readFileSync } from "node:fs";
+import { createHash } from "node:crypto";
+import { pipeline } from "node:stream/promises";
+import { webserver } from "../src/WebServer.js";
 
-await connection.waitUntilReady()
+await connection.waitUntilReady();
 
-const hash = data => createHash('sha512').update(data).digest('base64')
+const hash = data => createHash("sha512").update(data).digest("base64");
 
-const PAYLOAD_SIZE = 1024 * 1024 * 1024
-const TEMP_FILE = '/tmp/geonix.stream_test'
+const PAYLOAD_SIZE = 1024 * 1024 * 1024;
+const TEMP_FILE = "/tmp/geonix.stream_test";
 
-console.time('test')
+await webserver.start();
+
+console.time("test");
 try {
-    const payload = randomBytes(PAYLOAD_SIZE)
-    const sourceHash = hash(payload)
+    const payload = randomBytes(PAYLOAD_SIZE);
+    const sourceHash = hash(payload);
+
+    const stream = Stream(payload);
 
-    const source = await getReadable(Stream(payload))
-    const dest = createWriteStream(TEMP_FILE)
+    const source = await getReadable(stream);
+    const dest = createWriteStream(TEMP_FILE);
 
-    await pipeline(source, dest)
+    await pipeline(source, dest);
 
-    const check = readFileSync(TEMP_FILE)
-    const destHash = hash(check)
+    const check = readFileSync(TEMP_FILE);
+    const destHash = hash(check);
 
     if (sourceHash == destHash) {
-        console.log('MATCH')
+        console.log("MATCH");
     } else {
-        console.error('Destination does not match the source!')
-        console.log(payload)
-        console.log(check)
+        console.error("Destination does not match the source!");
+        console.log("P =", payload);
+        console.log("C =", check);
     }
 } catch (e) {
-    console.error(e)
+    console.error(e);
 } finally {
-    console.timeEnd('test')
+    console.timeEnd("test");
 }