geonix 1.12.3 → 1.20.0

package/src/Service.js

@@ -1,57 +1,53 @@
-import { codec, connection } from './Connection.js'
-import { picoid, sleep, hash, getSecondsSinceMidnight, OverlayObject, GeonixVersion, getNetworkAddresses } from './Util.js'
-import { webserver } from './WebServer.js'
-import { createConnection } from 'net'
-import { EOL } from 'os'
-import cookieParser from 'cookie-parser'
-import express from 'express'
-import { isStream, streamToString } from './Stream.js'
-
-const protectedMethodNames = ['constructor', 'onStart']
-const endpointMatcher = /^((?<options>.+)\|)?(?<verb>WS|SUB|GET|POST|PATCH|PUT|DELETE|HEAD|OPTIONS|ALL)\s(?<url>.*)/
-const isEndpointFilter = methodName => endpointMatcher.test(methodName)
-const ERROR_BEGIN_DELIMITER = '-'.repeat(10)
-const ERROR_END_DELIMITER = '-'.repeat(40)
-
-const json = express.json({ limit: '100mb' })
-const raw = express.raw({ type: '*/*', limit: '100mb' })
+import { codec, connection } from "./Connection.js";
+import { picoid, sleep, hash, getSecondsSinceMidnight, OverlayObject, GeonixVersion, getFirstItemFromAsyncIterable } from "./Util.js";
+import { webserver } from "./WebServer.js";
+import { createConnection } from "net";
+import { EOL } from "os";
+import cookieParser from "cookie-parser";
+import express from "express";
+import { isStream, streamToString } from "./Stream.js";
+
+const protectedMethodNames = ["constructor", "onStart"];
+const endpointMatcher = /^((?<options>.+)\|)?(?<verb>WS|SUB|GET|POST|PATCH|PUT|DELETE|HEAD|OPTIONS|ALL)\s(?<url>.*)/;
+const isEndpointFilter = methodName => endpointMatcher.test(methodName);
+const ERROR_BEGIN_DELIMITER = "-".repeat(10);
+const ERROR_END_DELIMITER = "-".repeat(40);
+
+const json = express.json({ limit: "100mb" });
+const raw = express.raw({ type: "*/*", limit: "100mb" });
 
 export class Service {
 
-    static serviceClasses = []
+    static serviceClasses = [];
 
     static start(options = {}) {
         if (!this.serviceClasses.includes(this.prototype.constructor.name))
-            this.serviceClasses.push(this.prototype.constructor.name)
-
-        const instance = new this()
-        instance.#start(options)
-    }
-
-    static static() {
+            this.serviceClasses.push(this.prototype.constructor.name);
 
+        const instance = new this();
+        instance.#start(options);
     }
 
     // ---------------------------------------------------------------------------------------------    
 
-    #me = {}
-    #options = {}
+    #me = {};
+    #options = {};
 
     async #start(options = {}) {
-        this.#options = options
+        this.#options = options;
 
-        await webserver.waitUntilReady()
-        await connection.waitUntilReady()
+        await webserver.waitUntilReady();
+        await connection.waitUntilReady();
 
-        const fields = Object.getOwnPropertyNames(this).filter(isEndpointFilter).concat(Object.getOwnPropertyNames(this.constructor.prototype))
+        const fields = Object.getOwnPropertyNames(this).filter(isEndpointFilter).concat(Object.getOwnPropertyNames(this.constructor.prototype));
 
         // preserve order of endpoints as defined in the source
-        const serviceSource = this.constructor.toString().split('\n')
+        const serviceSource = this.constructor.toString().split("\n");
         fields.sort((a, b, ia = -1, ib = -1) => {
-            for (let line = 0; line < serviceSource.length; line++) ia = serviceSource[line].includes(a) ? line : ia
-            for (let line = 0; line < serviceSource.length; line++) ib = serviceSource[line].includes(b) ? line : ib
-            return ia - ib
-        })
+            for (let line = 0; line < serviceSource.length; line++) ia = serviceSource[line].includes(a) ? line : ia;
+            for (let line = 0; line < serviceSource.length; line++) ib = serviceSource[line].includes(b) ? line : ib;
+            return ia - ib;
+        });
 
         this.#me = {
             id: options?.id,
@@ -59,33 +55,36 @@ export class Service {
             // instance
             i: picoid(),
             // name
-            n: this.constructor.name,
+            n: options.name ?? this.constructor.name,
             // version
             v: process.env.VERSION || process.env.version || options?.version || this.version || `999.999.${getSecondsSinceMidnight()}`,
             // methods
             m: fields
                 .filter(methodName => !protectedMethodNames.includes(methodName))
-                .filter(methodName => !methodName.startsWith('$')),
+                .filter(methodName => !methodName.startsWith("$")),
+            // geonix version
+            gx: GeonixVersion,
             // IP addresses
-            a: getNetworkAddresses().map(address => `${address}:${webserver.getPort()}`)
-        }
+            a: webserver.getAddresses().map(address => `${address}:${webserver.getPort()}`)
+        };
 
         // check if method takes context as first argument
         for (let methodName of this.#me.m) {
-            const method = this[methodName]
-            method.takesContext = method.toString()?.match(/\((?<args>.*)\)/)?.groups?.args.startsWith('$')
+            const method = this[methodName];
+            method.takesContext = method.toString()?.match(/\((?<args>.*)\)/)?.groups?.args.startsWith("$");
         }
 
-        this.#beacon()
-        this.#queueListener()
-        this.#directListener()
-        this.#webserver()
+        this.#beacon();
+        this.#queueListener();
+        this.#directListener();
+        this.#webserver();
 
-        console.log('gx.service.start', this.#me.n, this.#me.v)
+        console.log("gx.service.start", this.#me.n, this.#me.v);
 
         // execute onStart method, if present
-        if (this.onStart)
-            this.onStart()
+        if (this.onStart) {
+            this.onStart();
+        }
     }
 
     /**
@@ -94,8 +93,8 @@ export class Service {
      */
     async #beacon() {
         while (true) {
-            connection.publish('gx2.beacon', this.#me)
-            await sleep(1000)
+            connection.publish("gx2.beacon", { i: this.#me.i });
+            await sleep(1000);
         }
     }
 
@@ -103,17 +102,17 @@ export class Service {
      * Wait and respond to remote call events (queue)
      */
     async #queueListener() {
-        const identifier = `${this.#me.n}@${this.#me.v}`
-        const subscription = await connection.subscribe(`gx2.service.${hash(identifier)}`, { queue: identifier })
+        const identifier = `${this.#me.n}@${this.#me.v}`;
+        const subscription = await connection.subscribe(`gx2.service.${hash(identifier)}`, { queue: identifier });
 
         for await (let event of subscription) {
-            let call = codec.decode(event.data)
+            let call = codec.decode(event.data);
 
             if (isStream(call))
-                call = JSON.parse(await streamToString(call))
+                call = JSON.parse(await streamToString(call));
 
             if (call.$r && call.p)
-                this.#onCall(call.p, (json) => connection.publish(call.$r, json))
+                this.#onCall(call.p, (json) => connection.publish(call.$r, json));
         }
     }
 
@@ -121,17 +120,17 @@ export class Service {
      * Wait and respond to remote call events (queue)
      */
     async #directListener() {
-        const identifier = this.#me.i
-        const subscription = await connection.subscribe(`gx2.service.${hash(identifier)}`, { queue: identifier })
+        const identifier = this.#me.i;
+        const subscription = await connection.subscribe(`gx2.service.${hash(identifier)}`, { queue: identifier });
 
         for await (let event of subscription) {
-            let call = codec.decode(event.data)
+            let call = codec.decode(event.data);
 
             if (isStream(call))
-                call = JSON.parse(await streamToString(call))
+                call = JSON.parse(await streamToString(call));
 
             if (call.$r && call.p)
-                this.#onCall(call.p, (json) => connection.publish(call.$r, json))
+                this.#onCall(call.p, (json) => connection.publish(call.$r, json));
         }
     }
 
@@ -141,48 +140,52 @@ export class Service {
      */
     async #webserver() {
         const endpoints = this.#me.m
-            .filter(isEndpointFilter)
+            .filter(isEndpointFilter);
 
         if (!endpoints || endpoints.length === 0)
-            return
+            return;
 
-        const router = webserver.router()
-        router.use(json, raw, cookieParser())
+        const router = webserver.router();
+        router.use(json, raw, cookieParser());
         for (let endpoint of endpoints) {
-            let { verb, url: uri } = endpointMatcher.exec(endpoint)?.groups || {}
-            verb = verb.toLowerCase()
+            let { verb, url: uri } = endpointMatcher.exec(endpoint)?.groups || {};
+            verb = verb.toLowerCase();
 
-            let handlers = (Array.isArray(this[endpoint]) ? this[endpoint] : [this[endpoint]])
+            let handlers = (Array.isArray(this[endpoint]) ? this[endpoint] : [this[endpoint]]);
 
-            if (Array.isArray(this.#options?.handlers?.before))
-                handlers = [...this.#options?.handlers?.before, ...handlers]
-            if (Array.isArray(this.#options?.handlers?.after))
-                handlers = [...handlers, ...this.#options?.handlers?.after]
+            const handlersBefore = this.#options?.handlers?.before ?? [];
+            const handlersAfter = this.#options?.handlers?.after ?? [];
+
+            handlers = [...handlersBefore, ...handlers, ...handlersAfter];
 
             for (let n = 0; n < handlers.length; n++)
-                handlers[n] = handlers[n].bind(this)
+                handlers[n] = handlers[n].bind(this);
 
             switch (verb) {
-                case 'ws':
-                    router.ws(uri, this[endpoint].bind(this))
-                    break
-
-                case 'sub':
-                    const subscription = await connection.subscribe(`gx.sub.${uri}`)
-                    const processor = async () => {
-                        for await (const event of subscription)
-                            this[endpoint](event.data)
-                    }
-                    processor()
-                    break
+                case "ws":
+                    router.ws(uri, this[endpoint].bind(this));
+                    break;
+
+                case "sub":
+                    this.#sub(uri, this[endpoint]);
+                    break;
 
                 default:
-                    router[verb](uri, ...handlers)
-                    break
+                    router[verb](uri, ...handlers);
+                    break;
             }
         }
     }
 
+    async #sub(subject, handler) {
+        const subscription = await connection.subscribe(`gx.sub.${subject}`);
+        const processor = async () => {
+            for await (const event of subscription)
+                handler(event.data);
+        };
+        processor();
+    }
+
     /**
      * Handle individual call
      * @param {Object} call 
@@ -190,69 +193,69 @@ export class Service {
      * @returns 
      */
     async #onCall(call, respond) {
-        const { m: methodName, a: args, c: context, o: caller } = call
+        const { m: methodName, a: args, c: context, o: caller } = call;
 
-        const method = this[methodName]
-        let _args = args
+        const method = this[methodName];
+        let _args = args;
 
         if (!method)
-            return respond({ e: `unknown method (${this.#me.n}.${methodName})` })
+            return respond({ e: `unknown method (${this.#me.n}.${methodName})` });
 
         try {
             // inject context as first argument
             if (method.takesContext)
-                _args = [OverlayObject(context, { caller, me: this.#me }), ..._args]
+                _args = [OverlayObject(context, { caller, me: this.#me }), ..._args];
 
-            respond({ r: await method.apply(this, _args) })
+            respond({ r: await method.apply(this, _args) });
         } catch (e) {
-            respond({ e: `${EOL}${ERROR_BEGIN_DELIMITER} ${this.#me.n}${EOL}${e.stack}${EOL}${ERROR_END_DELIMITER}` })
+            respond({ e: `${EOL}${ERROR_BEGIN_DELIMITER} ${this.#me.n}${EOL}${e.stack}${EOL}${ERROR_END_DELIMITER}` });
         }
     }
 
-    connections = new Map()
-    async SYS_createConnection(streamId) {
-        const ingress = await connection.subscribe(`gx2.stream.${streamId}.b`)
-        const control = await connection.subscribe(`gx2.stream.${streamId}.c`)
+    connections = new Map();
+    async $createConnection(streamId) {
+        const ingress = await connection.subscribe(`gx2.stream.${streamId}.b`);
+        const control = await connection.subscribe(`gx2.stream.${streamId}.c`);
 
-        const client = createConnection({ port: webserver.getPort() })
-        client.on('data', (chunk) => connection.publishRaw(`gx2.stream.${streamId}.a`, chunk))
-        client.on('close', () => connection.unsubscribe(ingress))
+        const client = createConnection({ port: webserver.getPort() });
+        client.on("data", (chunk) => connection.publishRaw(`gx2.stream.${streamId}.a`, chunk));
+        client.on("close", () => connection.unsubscribe(ingress));
 
         this.connections.set(streamId, {
             client,
             sub: ingress
-        })
+        });
 
         const incomingLoop = async () => {
             for await (const event of ingress) {
-                client.write(Buffer.from(event.data))
+                client.write(Buffer.from(event.data));
             }
-        }
+        };
 
         const controlLoop = async () => {
-            // eslint-disable-next-line
-            for await (const event of control) {
-                const _connection = this.connections.get(streamId)
-                if (!_connection) {
-                    return
-                }
+            // wait for first control message to arrive
+            await getFirstItemFromAsyncIterable(control);
 
-                connection.unsubscribe(ingress)
-                connection.unsubscribe(control)
+            const _connection = this.connections.get(streamId);
+            if (!_connection) {
+                return;
+            }
 
-                _connection.client.destroy()
+            connection.unsubscribe(ingress);
+            connection.unsubscribe(control);
 
-                this.connections.delete(streamId)
-            }
-        }
+            _connection.client.destroy();
 
-        incomingLoop()
-        controlLoop()
+            this.connections.delete(streamId);
+        };
 
-        return true
+        incomingLoop();
+        controlLoop();
+
+        return true;
     }
 
-    async SYS_getEnv() {
+    $getEnv() {
         return {
             geonix: GeonixVersion,
             node: {
@@ -264,7 +267,11 @@ export class Service {
             mem: process.memoryUsage(),
             rss: process.memoryUsage.rss(),
             cpu: process.cpuUsage()
-        }
+        };
+    }
+
+    $getServiceInfo() {
+        return this.#me;
     }
 
 }

package/src/Stream.js

@@ -1,189 +1,98 @@
-import { Readable } from 'stream'
-import { connection } from './Connection.js'
-import { abortableAsyncGenerator, createServerAtFreePort, getNetworkAddresses, picoid, StreamChunker } from './Util.js'
-import http from 'http'
-
-const CHUNK_SIZE = 1024 * 128
-const STREAM_TIMEOUT = 120000
-
-export const stats = {}
-
-// HTTP stream server
-const streams = {}
-const { port: streamServerPort } = await createServerAtFreePort(http, (req, res) => {
-    const stream = streams[decodeURIComponent(req.url.substring(1))]
-    if (stream) {
-        stream.pipe(res)
-    } else {
-        res.statusCode = 404
-        res.end()
-        return
-    }
-}, 40000)
+import { Readable } from "stream";
+import { connection } from "./Connection.js";
+import { picoid, StreamChunker } from "./Util.js";
+
+const CHUNK_SIZE = 1024 * 128;
+
+export const stats = {};
 
 /**
- * Converts input data to stream
+ * Converts data to stream
  * 
  * @param {*} data 
  * @param {*} automated 
  * @returns 
  */
-export function Stream(data, tag = '_') {
+export function Stream(data, tag = "_") {
     if (isStream(data))
-        return data
+        return data;
 
-    const id = picoid()
-    const result = { $: 'stream', id }
-    let readable = data
+    const id = picoid();
+    let readable = data;
 
     // convert Buffer or string to a Readable
     if (!(readable.pipe && readable.readable))
-        readable = Readable.from(Buffer.from(data))
+        readable = Readable.from(Buffer.from(data));
 
     // split the stream is smaller chunks
-    const transform = StreamChunker(Math.min(connection.getMaxPayloadSize(), CHUNK_SIZE))
-    readable.pipe(transform)
-    readable = transform
-
-    stats[tag] = stats[tag] !== undefined ? stats[tag] + 1 : 1
-
-    const acDeliveryOverNATS = new AbortController();
+    const transform = StreamChunker(Math.min(connection.getMaxPayloadSize(), CHUNK_SIZE));
+    readable.pipe(transform);
+    readable = transform;
 
-    const deliverOverNATS = async () => {
-        const control = await connection.subscribe(`gx2.stream.${id}.a`, { max: 1 })
+    stats[tag] = stats[tag] !== undefined ? stats[tag] + 1 : 1;
 
-        // abort if no request to start streaming
-        const timeout = setTimeout(() => {
-            acDeliveryOverNATS.abort()
-        }, STREAM_TIMEOUT)
+    const controlHandler = async () => {
+        const control = await connection.subscribe(`gx2.stream.${id}.a`, { max: 1 });
 
-        // deliver the stream
-        for await (const event of abortableAsyncGenerator(control, acDeliveryOverNATS.signal)) {
+        for await (const event of control)
             if (event.data.length === 0) {
-                readable.on('data', chunk => connection.publishRaw(`gx2.stream.${id}.b`, chunk))
-                readable.on('close', () => {
-                    connection.publishRaw(`gx2.stream.${id}.b`)
-                    stats[tag]--
-                })
+                readable.on("data", chunk => connection.publishRaw(`gx2.stream.${id}.b`, chunk));
+                readable.on("close", () => {
+                    connection.publishRaw(`gx2.stream.${id}.b`);
+                    stats[tag]--;
+                });
             }
-        }
-
-        // cleanup
-        clearTimeout(timeout)
-        await control.drain()
-        await connection.unsubscribe(control)
-    }
-
-    const deliverOverHTTP = async () => {
-        // stop listening for NATS request
-        acDeliveryOverNATS.abort()
-
-        // register stream as available over HTTP
-        streams[id] = readable
+    };
 
-        // cleanup
-        readable.on('finish', () => {
-            delete streams[id]
-            stats[tag]--
-        })
-    }
+    controlHandler();
 
-    deliverOverNATS()
-
-    if (streamServerPort != -1) {
-        result.a = getNetworkAddresses()
-        result.p = streamServerPort;
-        deliverOverHTTP()
-    }
-
-    return result
+    return { $: "stream", id };
 }
 
 export function isStream(object) {
-    return object && typeof object === 'object' && object.$ === 'stream'
+    return object && typeof object === "object" && object.$ === "stream";
 }
 
-export async function getReadable(object, forceNats = false) {
+export async function getReadable(object) {
     if (!isStream(object))
-        return object
-
-    if (forceNats) {
-        return getReadableOverNATS(object);
-    }
-
-    try {
-        return await getReadableOverHTTP(object);
-    } catch (e) {
-        return getReadableOverNATS(object);
-    }
-}
-
-export async function getReadableOverHTTP(object) {
-    if (!object.a) {
-        throw new Error('Stream is not available over HTTP')
-    }
-
-    for (const address of object.a) {
-        try {
-            const response = await new Promise((resolve, reject) => {
-                const request = http.request(`http://${address}:${object.p}/${object.id}`, { method: 'GET', timeout: 5000 })
-                request.on('response', (response) => {
-                    resolve(response)
-                })
-
-                request.on('error', (e) => {
-                    reject(e)
-                })
-
-                request.end()
-            })
-
-            return response
-        } catch (e) {
-            console.error(`Stream.getReadableOverHTTP.error:`, e)
-        }
-    }
-
-    throw new Error('No data')
-}
+        return object;
 
-export async function getReadableOverNATS(object) {
-    const readable = new Readable({ read: () => null })
-    const subscription = await connection.subscribe(`gx2.stream.${object.id}.b`)
+    const readable = new Readable({ read: () => null });
+    const subscription = await connection.subscribe(`gx2.stream.${object.id}.b`);
 
     const dataHandler = async () => {
         for await (const event of subscription)
             try {
                 if (event.data.length === 0) {
-                    readable.push(null)
-                    subscription.drain()
+                    readable.push(null);
+                    subscription.drain();
                 } else
-                    readable.push(event.data)
+                    readable.push(event.data);
             } catch (e) {
-                console.error(`Stream.getReadable.dataHandler.error:`, e)
+                console.error("Stream.getReadable.dataHandler.error:", e);
             }
-    }
-    dataHandler()
+    };
+    dataHandler();
 
     // kickstart remote stream with a blank message
-    await connection.publishRaw(`gx2.stream.${object.id}.a`)
+    await connection.publishRaw(`gx2.stream.${object.id}.a`);
 
-    return readable
+    return readable;
 }
 
-export async function streamToBuffer(object, forceNats = false) {
-    let readable = object
+export async function streamToBuffer(object) {
+    let readable = object;
     if (isStream(readable))
-        readable = await getReadable(readable, forceNats)
+        readable = await getReadable(readable);
 
-    return Buffer.concat(await readable.toArray())
+    return Buffer.concat(await readable.toArray());
 }
 
 export async function streamToString(object) {
-    let readable = object
+    let readable = object;
     if (isStream(readable))
-        readable = await getReadable(readable)
+        readable = await getReadable(readable);
 
-    return Buffer.concat(await readable.toArray()).toString()
+    return Buffer.concat(await readable.toArray()).toString();
 }