geonix 1.23.8 → 1.30.4

package/src/Service.js

@@ -1,5 +1,15 @@
 import { connection } from "./Connection.js";
-import { picoid, sleep, hash, getSecondsSinceMidnight, OverlayObject, GeonixVersion, getFirstItemFromAsyncIterable, getNetworkAddresses, deepMerge } from "./Util.js";
+import {
+    picoid,
+    sleep,
+    hash,
+    getSecondsSinceMidnight,
+    OverlayObject,
+    GeonixVersion,
+    getFirstItemFromAsyncIterable,
+    getNetworkAddresses,
+    deepMerge,
+} from "./Util.js";
 import { webserver } from "./WebServer.js";
 import { createConnection } from "net";
 import { EOL } from "os";
@@ -8,10 +18,17 @@ import express from "express";
 import { isStream, streamToString } from "./Stream.js";
 import { logger } from "./Logger.js";
 import { decode } from "./Codec.js";
+import { _payloadKey, encryptPayload, decryptPayload } from "./Crypto.js";
+import { rpcContext } from "./Request.js";
+
+const BEACON_INTERVAL = 2500;
+const INACTIVITY_TIMEOUT = 90_000;
+
+const getInactivityTimeout = () => parseInt(process.env.GX_INACTIVITY_TIMEOUT) || INACTIVITY_TIMEOUT;
 
 const protectedMethodNames = ["constructor", "onStart"];
 const endpointMatcher = /^((?<options>.+)\|)?(?<verb>WS|SUB|GET|POST|PATCH|PUT|DELETE|HEAD|OPTIONS|ALL)\s(?<url>.*)/;
-const isEndpointFilter = methodName => endpointMatcher.test(methodName);
+const isEndpointFilter = (methodName) => endpointMatcher.test(methodName);
 const ERROR_BEGIN_DELIMITER = "-".repeat(10);
 const ERROR_END_DELIMITER = "-".repeat(40);
 
@@ -26,37 +43,37 @@ const defaultServiceOptions = {
     middleware: {
         json: true,
         raw: true,
-        cookies: true
+        cookies: true,
     },
-    fullBeacon: true
+    fullBeacon: true,
 };
 
+/**
+ * Base class for all Geonix services. Extend this class and call `Service.start()` to register
+ * the service on the NATS bus, expose HTTP endpoints, and begin sending beacons to the registry.
+ */
 export class Service {
-
-    static serviceClasses = [];
-
     /**
-     * 
-     * @param {ServiceOptions} options 
+     * Creates a new instance of the subclass and starts it on the NATS bus.
+     *
+     * @param {ServiceOptions} options - Optional configuration overrides (name, version, middleware, etc.).
+     * @returns {void}
      */
     static start(options = {}) {
-        if (!this.serviceClasses.includes(this.prototype.constructor.name)) {
-            this.serviceClasses.push(this.prototype.constructor.name);
-        }
-
         const instance = new this();
-        instance.#start(options);
+        instance.#start(options).catch((e) => logger.error("gx.service.start:", e));
     }
 
-    // ---------------------------------------------------------------------------------------------    
+    // ---------------------------------------------------------------------------------------------
 
     #isActive = false;
     #me = {};
-    #options = defaultServiceOptions;
+    #options = {};
+    #methodTakesContext = new WeakMap();
 
     async #start(options = {}) {
         this.#isActive = true;
-        this.#options = deepMerge(this.#options, options); // { ...this.#options, ...options };
+        this.#options = deepMerge({}, defaultServiceOptions, options);
 
         await webserver.waitUntilReady();
         await connection.waitUntilReady();
@@ -67,16 +84,15 @@ export class Service {
 
         // preserve order of endpoints as defined in the source
         const serviceSource = this.constructor.toString().split("\n");
-        fields.sort((a, b, ia = -1, ib = -1) => {
-            for (let line = 0; line < serviceSource.length; line++) {
-                ia = serviceSource[line].includes(a) ? line : ia;
-            }
-
-            for (let line = 0; line < serviceSource.length; line++) {
-                ib = serviceSource[line].includes(b) ? line : ib;
-            }
-            return ia - ib;
-        });
+        const lineMap = new Map();
+        for (const name of fields) {
+            const quoted = JSON.stringify(name);
+            lineMap.set(
+                name,
+                serviceSource.findIndex((line) => line.includes(quoted)),
+            );
+        }
+        fields.sort((a, b) => lineMap.get(a) - lineMap.get(b));
 
         this.#me = {
             id: options?.id,
@@ -86,33 +102,52 @@ export class Service {
             // name
             n: options.name ?? this.constructor.name,
             // version
-            v: process.env.VERSION || process.env.version || options?.version || this.version || `999.999.${getSecondsSinceMidnight()}`,
+            v:
+                process.env.GX_VERSION ||
+                process.env.VERSION ||
+                process.env.version ||
+                options?.version ||
+                this.version ||
+                `999.999.${getSecondsSinceMidnight()}`,
             // methods
             m: fields
-                .filter(methodName => !protectedMethodNames.includes(methodName))
-                .filter(methodName => !methodName.startsWith("$")),
+                .filter((methodName) => !protectedMethodNames.includes(methodName))
+                .filter((methodName) => !methodName.startsWith("$")),
             // geonix version
             gx: GeonixVersion,
             // IP addresses
-            a: getNetworkAddresses().map(address => `${address}:${webserver.getPort()}`)
+            a: getNetworkAddresses().map((address) => `${address}:${webserver.getPort()}`),
         };
 
         // check if method takes context as first argument
         for (let methodName of this.#me.m) {
             const method = this[methodName];
-            method.takesContext = method.toString()?.match(/\((?<args>.*)\)/)?.groups?.args.startsWith("$");
+            this.#methodTakesContext.set(
+                method,
+                method
+                    .toString()
+                    ?.match(/\((?<args>.*)\)/)
+                    ?.groups?.args.startsWith("$"),
+            );
         }
 
-        this.#beacon();
-        this.#queueListener();
-        this.#directListener();
-        this.#webserver();
+        this.#beacon().catch((e) => logger.error("gx.beacon:", e));
+
+        this.#callListener(`${this.#me.n}@${this.#me.v}`).catch((e) => logger.error("gx.queueListener:", e));
+
+        this.#callListener(this.#me.i).catch((e) => logger.error("gx.directListener:", e));
+
+        this.#webserver().catch((e) => logger.error("gx.webserver:", e));
 
         logger.info("gx.service.start", this.#me.n, this.#me.v);
 
         // execute onStart method, if present
         if (this.onStart) {
-            this.onStart();
+            try {
+                await this.onStart();
+            } catch (e) {
+                logger.error("onStart:", e);
+            }
         }
     }
 
@@ -121,69 +156,67 @@ export class Service {
      * on the transport
      */
     async #beacon() {
-        while (true) {
-            if (this.#options.fullBeacon) {
-                connection.publish("gx2.beacon", this.#me);
-            } else {
-                connection.publish("gx2.beacon", { i: this.#me.i });
-            }
-            await sleep(1000);
+        while (this.#isActive) {
+            const payload = this.#options.fullBeacon ? this.#me : { i: this.#me.i };
+            connection.publish("gx2.beacon", payload).catch((e) => logger.warn("beacon.publish:", e));
+            await sleep(BEACON_INTERVAL);
         }
     }
 
-    /**
-     * Wait and respond to remote call events (queue)
-     */
-    async #queueListener() {
-        const identifier = `${this.#me.n}@${this.#me.v}`;
+    async #callListener(identifier) {
         const subscription = await connection.subscribe(`gx2.service.${hash(identifier)}`, { queue: identifier });
 
         for await (let event of subscription) {
-            let call = decode(event.data);
-
-            if (isStream(call)) {
-                call = JSON.parse(await streamToString(call));
-            }
-
-            if (call.$r && call.p) { this.#onCall(call.p, (json) => connection.publish(call.$r, json)); }
-        }
-    }
-
-    /**
-     * Wait and respond to remote call events (queue)
-     */
-    async #directListener() {
-        const identifier = this.#me.i;
-        const subscription = await connection.subscribe(`gx2.service.${hash(identifier)}`, { queue: identifier });
-
-        for await (let event of subscription) {
-            let call = decode(event.data);
-
-            if (isStream(call)) {
-                call = JSON.parse(await streamToString(call));
-            }
-
-            if (call.$r && call.p) {
-                this.#onCall(call.p, (json) => connection.publish(call.$r, json));
+            try {
+                let call = decode(event.data);
+
+                if (isStream(call)) {
+                    call = JSON.parse(await streamToString(call));
+                }
+
+                if (call.$r && call.p) {
+                    this.#onCall(call.p, (json) => connection.publish(call.$r, json));
+                }
+            } catch (e) {
+                logger.warn("gx.callListener.message:", e);
             }
         }
     }
 
     /**
      * Register local endpoints with express instance
-     * @returns 
+     * @returns
      */
     async #webserver() {
-        const endpoints = this.#me.m
-            .filter(isEndpointFilter);
+        const endpoints = this.#me.m.filter(isEndpointFilter);
+
+        const router = webserver.router();
+
+        // HTTP RPC endpoint — keyed by instance ID so multiple versions on the same
+        // host each get a unique path
+        router.post(`/!!_gx/rpc/${hash(this.#me.i)}`, raw, async (req, res) => {
+            const body = _payloadKey
+                ? JSON.parse(decryptPayload(req.body))
+                : Buffer.isBuffer(req.body)
+                  ? JSON.parse(req.body.toString())
+                  : req.body;
+            await this.#onCall(body, (result) => {
+                const payload = Buffer.from(JSON.stringify(result));
+                if (_payloadKey) {
+                    res.set("Content-Type", "application/octet-stream");
+                    res.send(encryptPayload(payload));
+                } else {
+                    res.set("Content-Type", "application/json");
+                    res.send(payload);
+                }
+            });
+        });
 
         if (!endpoints || endpoints.length === 0) {
             return;
         }
 
-        const router = webserver.router();
-
-        // setup defualt middlewares
+        // setup default middlewares
         if (this.#options.middleware?.json) {
             router.use(json);
         }
@@ -199,7 +232,7 @@ export class Service {
             let { verb, url: uri } = endpointMatcher.exec(endpoint)?.groups || {};
             verb = verb.toLowerCase();
 
-            let handlers = (Array.isArray(this[endpoint]) ? this[endpoint] : [this[endpoint]]);
+            let handlers = Array.isArray(this[endpoint]) ? this[endpoint] : [this[endpoint]];
 
             const handlersBefore = this.#options?.handlers?.before ?? [];
             const handlersAfter = this.#options?.handlers?.after ?? [];
@@ -214,10 +247,23 @@ export class Service {
 
             switch (verb) {
                 case "ws":
+                    // handlersBefore run as route-scoped middleware before the upgrade;
+                    // handlersAfter does not apply to WebSocket connections.
+                    if (handlersBefore.length > 0) {
+                        router.use(
+                            uri,
+                            ...handlersBefore.map(
+                                (h) =>
+                                    (...args) =>
+                                        h.apply(this, args),
+                            ),
+                        );
+                    }
                     router.ws(uri, this[endpoint].bind(this));
                     break;
 
                 case "sub":
+                    // SUB operates at the NATS transport level — HTTP middleware does not apply.
                     this.#sub(uri, this[endpoint].bind(this));
                     break;
 
@@ -237,14 +283,14 @@ export class Service {
                 handler(event.data);
             }
         };
-        processor();
+        processor().catch((e) => logger.error("$sub.processor:", e));
     }
 
     /**
      * Handle individual call
-     * @param {Object} call 
-     * @param {Function} respond 
-     * @returns 
+     * @param {Object} call
+     * @param {Function} respond
+     * @returns
      */
     async #onCall(call, respond) {
         const { m: methodName, a: args, c: context, o: caller } = call;
@@ -252,23 +298,24 @@ export class Service {
         const method = this[methodName];
         let _args = args;
 
-        if (!method) {
+        if (!method || methodName.startsWith("$$")) {
             return respond({ e: `unknown method (${this.#me.n}.${methodName})` });
         }
 
         try {
             // inject context as first argument
-            if (method.takesContext) {
-                _args = [OverlayObject(context, { caller, me: this.#me }), ..._args];
+            if (this.#methodTakesContext.get(method)) {
+                _args = [OverlayObject(context ?? {}, { caller, me: this.#me }), ..._args];
             }
 
-            respond({ r: await method.apply(this, _args) });
+            const originator = `${this.#me.n}.${methodName}`;
+            respond({ r: await rpcContext.run({ originator }, () => method.apply(this, _args)) });
         } catch (e) {
             respond({ e: `${EOL}${ERROR_BEGIN_DELIMITER} ${this.#me.n}${EOL}${e.stack}${EOL}${ERROR_END_DELIMITER}` });
         }
     }
 
-    connections = new Map();
+    #connections = new Map();
     async $createConnection(streamId) {
         const ingress = await connection.subscribe(`gx2.stream.${streamId}.b`);
         const control = await connection.subscribe(`gx2.stream.${streamId}.c`);
@@ -276,14 +323,35 @@ export class Service {
         const client = createConnection({ port: webserver.getPort() });
         client.on("data", (chunk) => connection.publishRaw(`gx2.stream.${streamId}.a`, chunk));
         client.on("close", () => connection.unsubscribe(ingress));
+        client.on("error", (e) => logger.error("$createConnection.client.error:", e));
 
-        this.connections.set(streamId, {
+        this.#connections.set(streamId, {
             client,
-            sub: ingress
+            sub: ingress,
         });
 
+        const cleanup = () => {
+            const _connection = this.#connections.get(streamId);
+            if (!_connection) {
+                return;
+            }
+
+            connection.unsubscribe(ingress);
+            connection.unsubscribe(control);
+            _connection.client.destroy();
+            this.#connections.delete(streamId);
+        };
+
+        const timeout = getInactivityTimeout();
+        let inactivityTimer = setTimeout(cleanup, timeout);
+        const resetTimer = () => {
+            clearTimeout(inactivityTimer);
+            inactivityTimer = setTimeout(cleanup, timeout);
+        };
+
         const incomingLoop = async () => {
             for await (const event of ingress) {
+                resetTimer();
                 client.write(Buffer.from(event.data));
             }
         };
@@ -291,22 +359,15 @@ export class Service {
         const controlLoop = async () => {
             // wait for first control message to arrive
             await getFirstItemFromAsyncIterable(control);
-
-            const _connection = this.connections.get(streamId);
-            if (!_connection) {
-                return;
-            }
-
-            connection.unsubscribe(ingress);
-            connection.unsubscribe(control);
-
-            _connection.client.destroy();
-
-            this.connections.delete(streamId);
+            clearTimeout(inactivityTimer);
+            cleanup();
         };
 
-        incomingLoop();
-        controlLoop();
+        incomingLoop().catch((e) => {
+            logger.error("$createConnection.incomingLoop:", e);
+            cleanup();
+        });
+        controlLoop().catch((e) => logger.error("$createConnection.controlLoop:", e));
 
         return true;
     }
@@ -317,12 +378,11 @@ export class Service {
             node: {
                 version: process.version,
                 platform: process.platform,
-                arch: process.arch
+                arch: process.arch,
             },
-            env: process.env,
             mem: process.memoryUsage(),
             rss: process.memoryUsage.rss(),
-            cpu: process.cpuUsage()
+            cpu: process.cpuUsage(),
         };
     }
 
@@ -330,8 +390,11 @@ export class Service {
         return this.#me;
     }
 
-    async $stop() {
-        this.#isActive = false;
+    $connectionCount() {
+        return this.#connections.size;
     }
 
-}
+    $stop() {
+        this.#isActive = false;
+    }
+}

package/src/Stream.js

@@ -1,22 +1,32 @@
 import { Readable } from "stream";
 import { connection } from "./Connection.js";
-import { getFirstItemFromAsyncIterable, getNetworkAddresses, picoid, StreamChunker } from "./Util.js";
+import { fetchWithTimeout, getFirstItemFromAsyncIterable, getNetworkAddresses, picoid, StreamChunker } from "./Util.js";
 import { logger } from "./Logger.js";
 import { webserver } from "./WebServer.js";
 
 const CHUNK_SIZE = 1024 * 128;
+const STREAM_TIMEOUT = 90_000;
 
-export const stats = {};
+const getStreamTimeout = () => parseInt(process.env.GX_STREAM_TIMEOUT) || STREAM_TIMEOUT;
+
+/**
+ * Map of stream IDs to their in-process {@link Readable} instances. Entries are added when a
+ * stream is created and removed once the stream has been consumed or the timeout expires.
+ *
+ * @type {Object.<string, import('stream').Readable>}
+ */
 export const activeStreams = {};
 
 /**
- * Converts data to stream
- * 
- * @param {*} data 
- * @param {*} automated 
- * @returns 
+ * Wraps data in a Geonix stream descriptor that can be sent over the transport.
+ * If `data` is already a stream descriptor it is returned unchanged. Buffers and strings are
+ * converted to a {@link Readable} automatically. The underlying data is chunked and made
+ * available to consumers either via HTTP (preferred) or NATS.
+ *
+ * @param {Buffer|string|import('stream').Readable|object} data - The data to stream.
+ * @returns {{ $: 'stream', id: string, a?: string[] }} Stream descriptor object.
  */
-export function Stream(data, tag = "_") {
+export function Stream(data) {
     if (isStream(data)) {
         return data;
     }
@@ -34,34 +44,46 @@ export function Stream(data, tag = "_") {
     readable.pipe(transform);
     readable = transform;
 
-    stats[tag] = stats[tag] !== undefined ? stats[tag] + 1 : 1;
     activeStreams[id] = readable;
 
     // NATS handler
     (async () => {
         const control = await connection.subscribe(`gx2.stream.${id}.a`, { max: 1 });
 
-        const event = await getFirstItemFromAsyncIterable(control);
+        const event = await Promise.race([
+            getFirstItemFromAsyncIterable(control),
+            new Promise((resolve) => setTimeout(() => resolve(null), getStreamTimeout())),
+        ]);
+
+        if (!event) {
+            // timed out — clean up orphaned stream
+            connection.unsubscribe(control);
+            delete activeStreams[id];
+            readable.destroy();
+            return;
+        }
+
         if (activeStreams[id] !== undefined && event.data.length === 0) {
             // remove stream from the list
             delete activeStreams[id];
 
             // kickstart the stream
-            readable.on("data", chunk => connection.publishRaw(`gx2.stream.${id}.b`, chunk));
+            readable.on("data", (chunk) => connection.publishRaw(`gx2.stream.${id}.b`, chunk));
             readable.on("close", () => {
                 connection.publishRaw(`gx2.stream.${id}.b`);
-                stats[tag]--;
             });
         }
-    })();
+    })().catch((e) => logger.error("stream.nats.handler:", e));
 
     const result = {
         $: "stream",
-        id
+        id,
     };
 
     // get the port and addresses of the webserver
-    const addresses = webserver.getPort() ? getNetworkAddresses().map(address => `${address}:${webserver.getPort()}`) : undefined;
+    const addresses = webserver.getPort()
+        ? getNetworkAddresses().map((address) => `${address}:${webserver.getPort()}`)
+        : undefined;
     if (addresses) {
         result.a = addresses;
     }
@@ -69,10 +91,25 @@ export function Stream(data, tag = "_") {
     return result;
 }
 
+/**
+ * Returns `true` if `object` is a Geonix stream descriptor.
+ *
+ * @param {*} object - Value to test.
+ * @returns {boolean}
+ */
 export function isStream(object) {
     return object && typeof object === "object" && object.$ === "stream";
 }
 
+/**
+ * Resolves a Geonix stream descriptor to a Node.js {@link import('stream').Readable}. If the
+ * descriptor advertises HTTP addresses the stream is fetched over HTTP; otherwise it falls back
+ * to NATS. Non-descriptor values are returned as-is.
+ *
+ * @param {{ $: 'stream', id: string, a?: string[] }|any} object - Stream descriptor or plain value.
+ * @returns {Promise<import('stream').Readable>}
+ * @throws {Error} If `object` is falsy.
+ */
 export async function getReadable(object) {
     if (!object) {
         throw Error("Stream.getReadable: invalid object");
@@ -86,8 +123,8 @@ export async function getReadable(object) {
     if (object.a && object.a.length > 0) {
         for (const address of object.a) {
             try {
-                const uri = `http://${address}/!!_____stream/${object.id}`;
-                const response = await fetch(uri);
+                const uri = `http://${address}/!!_gx/stream/${object.id}`;
+                const response = await fetchWithTimeout(uri);
 
                 if (response.status === 200) {
                     return Readable.fromWeb(response.body);
@@ -116,7 +153,11 @@ export async function getReadable(object) {
             }
         }
     };
-    dataHandler();
+    dataHandler().catch((e) => {
+        logger.error("stream.dataHandler:", e);
+        subscription.unsubscribe();
+        readable.destroy();
+    });
 
     // kickstart remote stream with a blank message
     await connection.publishRaw(`gx2.stream.${object.id}.a`);
@@ -124,6 +165,12 @@ export async function getReadable(object) {
     return readable;
 }
 
+/**
+ * Reads a stream (or stream descriptor) fully into memory and returns it as a {@link Buffer}.
+ *
+ * @param {import('stream').Readable|{ $: 'stream', id: string, a?: string[] }} object - Readable or stream descriptor.
+ * @returns {Promise<Buffer>}
+ */
 export async function streamToBuffer(object) {
     let readable = object;
     if (isStream(readable)) {
@@ -133,10 +180,23 @@ export async function streamToBuffer(object) {
     return Buffer.concat(await readable.toArray());
 }
 
+/**
+ * Reads a stream (or stream descriptor) fully into memory and returns it as a string.
+ *
+ * @param {import('stream').Readable|{ $: 'stream', id: string, a?: string[] }} object - Readable or stream descriptor.
+ * @param {BufferEncoding} [encoding] - Character encoding passed to `Buffer.toString()`. Defaults to UTF-8.
+ * @returns {Promise<string>}
+ */
 export async function streamToString(object, encoding) {
     return (await streamToBuffer(object)).toString(encoding);
 }
 
+/**
+ * Reads a stream (or stream descriptor) fully into memory and parses its contents as JSON.
+ *
+ * @param {import('stream').Readable|{ $: 'stream', id: string, a?: string[] }} object - Readable or stream descriptor.
+ * @returns {Promise<any>}
+ */
 export async function streamToJSON(object) {
     return JSON.parse(await streamToBuffer(object));
 }