geonix 1.10.8 → 1.11.0-beta.1

package/index.d.ts

@@ -124,7 +124,7 @@ export class Service {
     #private;
 }
 /**
- * Converts data to stream
+ * Converts input data to stream
  *
  * @param {*} data
  * @param {*} automated
@@ -133,6 +133,8 @@ export class Service {
 export function Stream(data: any, tag?: string): any;
 export function isStream(object: any): any;
 export function getReadable(object: any): Promise<any>;
+export function getReadableOverHTTP(object: any): Promise<any>;
+export function getReadableOverNATS(object: any): Promise<any>;
 export function streamToBuffer(object: any): Promise<any>;
 export function streamToString(object: any): Promise<any>;
 export const stats: {};
@@ -147,6 +149,10 @@ export function parseURL(url: string): {
     pass: any;
     token: any;
 };
+export function timeoutAsyncGenerator(target: any, ms: any): {};
+export function waitForAbort(abortSignal: any, result: any): Promise<any>;
+export function abortableAsyncGenerator(target: any, abortSignal: any): {};
+export function getNetworkAddresses(): any[];
 export function sleep(delay: number): Promise<any>;
 export function picoid(size?: number): any;
 export function hash(data: string | Buffer): any;
@@ -165,7 +171,6 @@ export function ServeStatic(root: any, options?: {}): any;
 export const webserver: WebServer;
 declare class WebServer {
     start(): Promise<void>;
-    getAddresses(): any[];
     getPort(): any;
     router(): any;
     waitUntilReady(): Promise<void>;

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "geonix",
-    "version": "1.10.8",
+    "version": "1.11.0-beta.1",
     "type": "module",
     "description": "",
     "bin": {
@@ -20,9 +20,9 @@
         "express-async-errors": "^3.1.1",
         "express-ws": "^5.0.2",
         "multer": "^1.4.5-lts.1",
-        "nats": "^2.16.0",
+        "nats": "^2.19.0",
         "semver": "^7.5.4",
-        "ws": "^8.13.0"
+        "ws": "^8.16.0"
     },
     "publishConfig": {
         "registry": "https://registry.npmjs.org/"

package/src/Service.js

@@ -1,5 +1,5 @@
 import { codec, connection } from './Connection.js'
-import { picoid, sleep, hash, getSecondsSinceMidnight, OverlayObject, GeonixVersion } from './Util.js'
+import { picoid, sleep, hash, getSecondsSinceMidnight, OverlayObject, GeonixVersion, getNetworkAddresses } from './Util.js'
 import { webserver } from './WebServer.js'
 import { createConnection } from 'net'
 import { EOL } from 'os'
@@ -67,7 +67,7 @@ export class Service {
                 .filter(methodName => !protectedMethodNames.includes(methodName))
                 .filter(methodName => !methodName.startsWith('$')),
             // IP addresses
-            a: webserver.getAddresses().map(address => `${address}:${webserver.getPort()}`)
+            a: getNetworkAddresses().map(address => `${address}:${webserver.getPort()}`)
         }
 
         // check if method takes context as first argument

package/src/Stream.js

@@ -1,13 +1,28 @@
 import { Readable } from 'stream'
 import { connection } from './Connection.js'
-import { picoid, StreamChunker } from './Util.js'
+import { abortableAsyncGenerator, createServerAtFreePort, getNetworkAddresses, picoid, StreamChunker } from './Util.js'
+import http from 'http'
 
 const CHUNK_SIZE = 1024 * 128
+const STREAM_TIMEOUT = 120000
 
 export const stats = {}
 
+// HTTP stream server
+const streams = {}
+const { port: streamServerPort } = await createServerAtFreePort(http, (req, res) => {
+    const stream = streams[decodeURIComponent(req.url.substring(1))]
+    if (stream) {
+        stream.pipe(res)
+    } else {
+        res.statusCode = 404
+        res.end()
+        return
+    }
+}, 40000)
+
 /**
- * Converts data to stream
+ * Converts input data to stream
  * 
  * @param {*} data 
  * @param {*} automated 
@@ -18,6 +33,7 @@ export function Stream(data, tag = '_') {
         return data
 
     const id = picoid()
+    const result = { $: 'stream', id }
     let readable = data
 
     // convert Buffer or string to a Readable
@@ -31,10 +47,18 @@ export function Stream(data, tag = '_') {
 
     stats[tag] = stats[tag] !== undefined ? stats[tag] + 1 : 1
 
-    const controlHandler = async () => {
+    const acDeliveryOverNATS = new AbortController();
+
+    const deliverOverNATS = async () => {
         const control = await connection.subscribe(`gx2.stream.${id}.a`, { max: 1 })
 
-        for await (const event of control)
+        // abort if no request to start streaming
+        const timeout = setTimeout(() => {
+            acDeliveryOverNATS.abort()
+        }, STREAM_TIMEOUT)
+
+        // deliver the stream
+        for await (const event of abortableAsyncGenerator(control, acDeliveryOverNATS.signal)) {
             if (event.data.length === 0) {
                 readable.on('data', chunk => connection.publishRaw(`gx2.stream.${id}.b`, chunk))
                 readable.on('close', () => {
@@ -42,11 +66,37 @@ export function Stream(data, tag = '_') {
                     stats[tag]--
                 })
             }
+        }
+
+        // cleanup
+        clearTimeout(timeout)
+        await control.drain()
+        await connection.unsubscribe(control)
     }
 
-    controlHandler()
+    const deliverOverHTTP = async () => {
+        // stop listening for NATS request
+        acDeliveryOverNATS.abort()
+
+        // register stream as available over HTTP
+        streams[id] = readable
 
-    return { $: 'stream', id }
+        // cleanup
+        readable.on('finish', () => {
+            delete streams[id]
+            stats[tag]--
+        })
+    }
+
+    deliverOverNATS()
+
+    if (streamServerPort != -1) {
+        result.a = getNetworkAddresses()
+        result.p = streamServerPort;
+        deliverOverHTTP()
+    }
+
+    return result
 }
 
 export function isStream(object) {
@@ -57,6 +107,43 @@ export async function getReadable(object) {
     if (!isStream(object))
         return object
 
+    try {
+        return await getReadableOverHTTP(object);
+    } catch (e) {
+        return getReadableOverNATS(object);
+    }
+}
+
+export async function getReadableOverHTTP(object) {
+    if (!object.a) {
+        throw new Error('Stream is not available over HTTP')
+    }
+
+    for (const address of object.a) {
+        try {
+            const response = await new Promise((resolve, reject) => {
+                const request = http.request(`http://${address}:${object.p}/${object.id}`, { method: 'GET', timeout: 5000 })
+                request.on('response', (response) => {
+                    resolve(response)
+                })
+
+                request.on('error', (e) => {
+                    reject(e)
+                })
+
+                request.end()
+            })
+
+            return response
+        } catch (e) {
+            console.error(`Stream.getReadableOverHTTP.error:`, e)
+        }
+    }
+
+    throw new Error('No data')
+}
+
+export async function getReadableOverNATS(object) {
     const readable = new Readable({ read: () => null })
     const subscription = await connection.subscribe(`gx2.stream.${object.id}.b`)
 

package/src/Util.js

@@ -3,11 +3,9 @@ import { URL, fileURLToPath } from 'url'
 import { readFile } from 'fs/promises'
 import { join } from 'path'
 import { Transform } from 'node:stream'
-import WebSocket from 'ws'
-import * as http from 'http'
-import * as https from 'http'
-import * as net from 'net'
-
+import { networkInterfaces } from 'os'
+import http from 'http'
+import https from 'http'
 /**
  * Wait for {delay} ms
  * @param {number} delay 
@@ -196,3 +194,54 @@ export const StreamChunker = (chunkSize = 65536) => {
 
     return chunker;
 }
+
+export async function* timeoutAsyncGenerator(target, ms) {
+    const iterator = target[Symbol.asyncIterator]()
+
+    while (true) {
+        const { value, done } = await Promise.race([iterator.next(), sleep(ms)]) || {};
+        if (done || (value === undefined && done === undefined)) {
+            break;
+        }
+
+        yield value;
+    }
+}
+
+export async function waitForAbort(abortSignal, result) {
+    return new Promise(resolve => {
+        if (abortSignal.aborted) {
+            resolve(result)
+        }
+
+        abortSignal.addEventListener('abort', () => {
+            console.log('aborted')
+            resolve(result)
+        })
+    })
+}
+
+export async function* abortableAsyncGenerator(target, abortSignal) {
+    const iterator = target[Symbol.asyncIterator]()
+
+    while (true) {
+        const { value, done } = await Promise.race([iterator.next(), waitForAbort(abortSignal)]) || {};
+        if (done || (value === undefined && done === undefined)) {
+            break;
+        }
+
+        yield value;
+    }
+}
+
+export function getNetworkAddresses() {
+    const list = []
+    const interfaces = networkInterfaces()
+
+    for (let interfaceAddresses of Object.values(interfaces))
+        for (let addressObject of interfaceAddresses)
+            if (addressObject.family === 'IPv4')
+                list.push(addressObject.address)
+
+    return list
+}

package/src/WebServer.js

@@ -102,18 +102,6 @@ class WebServer {
         this.#ready = true
     }
 
-    getAddresses() {
-        const list = []
-        const interfaces = networkInterfaces()
-
-        for (let interfaceAddresses of Object.values(interfaces))
-            for (let addressObject of interfaceAddresses)
-                if (addressObject.family === 'IPv4')
-                    list.push(addressObject.address)
-
-        return list
-    }
-
     getPort() {
         return this.#port
     }