@toa.io/extensions.realtime 1.0.0-alpha.31 → 1.0.0-alpha.310

package/components/realtime.streams/manifest.toa.yaml

@@ -0,0 +1,52 @@
+namespace: realtime
+name: streams
+
+operations:
+  create:
+    input:
+      properties:
+        key:
+          type: string
+        token:
+          type: string
+      type: object
+  push:
+    output: { type: 'null' }
+    bindings: ~ # only for internal use
+    input:
+      properties:
+        data: {}
+        key:
+          type: string
+        event:
+          type: string
+      type: object
+      required:
+        - key
+        - event
+
+configuration:
+  schema:
+    properties:
+      maxlen:
+        type: number
+      expire:
+        type: number
+    type: object
+    # both have defaults, so the component reads them without checking
+    required: [maxlen, expire]
+
+  defaults:
+    maxlen: 1000
+    expire: 300
+
+state: ~
+stash: ~
+
+exposition:
+  /:key:
+    auth:id: key
+    GET:
+      endpoint: create
+      # an effect: opening a stream is not a read, and `GET` is the method a client opens one with
+      io:readonly: false

package/components/realtime.streams/operations/create.js

@@ -0,0 +1,77 @@
+import { Stream, Stash } from './lib/index.js';
+/**
+ * A stream is opened per call, and a key has as many of them as it has consumers: one stream read
+ * by several would hand each of its values to one of them only, and a consumer leaving would
+ * destroy it under the others. What arrives under a key is pushed to every stream of it; what
+ * belongs to a connection — the welcome, the token, the events missed since a token — to that
+ * connection's stream alone.
+ */
+export class Effect {
+    streams = new Map();
+    stash;
+    logs;
+    mount(context) {
+        context.state.streams = this.streams;
+        context.state.stash = new Stash(context.stash, context.configuration, context.logs);
+        this.logs = context.logs;
+        this.stash = context.state.stash;
+    }
+    unmount() {
+        const streams = [...this.streams.values()].flatMap((set) => [...set]);
+        this.logs.info('Destroying streams', { count: streams.length });
+        // closed, not destroyed: destroying a stream that is still piped to a response
+        // makes end-of-stream report a premature close, which reaches no one and takes
+        // the process down.
+        for (const stream of streams)
+            stream.close();
+    }
+    async execute(input) {
+        const key = input.key;
+        const stream = this.createStream(key);
+        // welcome
+        setTimeout(() => stream.heartbeat(), 1000);
+        if (input.token === undefined)
+            void this.stash.connect(key).then((token) => {
+                if (token instanceof Error)
+                    this.logs.error('Failed to connect to stash', { key, error: token });
+                else
+                    stream.push({ event: 'token', data: token });
+            });
+        else
+            void this.stash.pop(key, input.token).then((result) => {
+                if (result === null)
+                    return;
+                if ('code' in result && result.code === 'NO_RESULTS')
+                    return;
+                if (result instanceof Error) {
+                    this.logs.error('Failed to pop from stash', { key, error: result });
+                    return;
+                }
+                if (stream.destroyed)
+                    return;
+                const [token, events] = result;
+                for (const event of events)
+                    stream.push({ event: event.event, data: event.data });
+                stream.push({ event: 'token', data: token });
+            });
+        return stream;
+    }
+    createStream(key) {
+        const stream = new Stream();
+        let set = this.streams.get(key);
+        if (set === undefined) {
+            set = new Set();
+            this.streams.set(key, set);
+        }
+        set.add(stream);
+        this.logs.debug('Stream created', { key, count: set.size });
+        stream.events.once('destroy', () => {
+            set.delete(stream);
+            if (set.size === 0)
+                this.streams.delete(key);
+            this.logs.debug('Stream destroyed', { key, count: set.size });
+        });
+        return stream;
+    }
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY3JlYXRlLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc291cmNlL2NyZWF0ZS50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFHQSxPQUFPLEVBQUUsTUFBTSxFQUFFLEtBQUssRUFBRSxNQUFNLGdCQUFnQixDQUFBO0FBRTlDOzs7Ozs7R0FNRztBQUNILE1BQU0sT0FBTyxNQUFNO0lBQ0EsT0FBTyxHQUFHLElBQUksR0FBRyxFQUF1QixDQUFBO0lBQ2pELEtBQUssQ0FBUTtJQUNiLElBQUksQ0FBSztJQUVWLEtBQUssQ0FBQyxPQUFnQjtRQUMzQixPQUFPLENBQUMsS0FBSyxDQUFDLE9BQU8sR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFBO1FBQ3BDLE9BQU8sQ0FBQyxLQUFLLENBQUMsS0FBSyxHQUFHLElBQUksS0FBSyxDQUFDLE9BQU8sQ0FBQyxLQUFLLEVBQUUsT0FBTyxDQUFDLGFBQWEsRUFBRSxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUE7UUFFbkYsSUFBSSxDQUFDLElBQUksR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFBO1FBQ3hCLElBQUksQ0FBQyxLQUFLLEdBQUcsT0FBTyxDQUFDLEtBQUssQ0FBQyxLQUFLLENBQUE7SUFDbEMsQ0FBQztJQUVNLE9BQU87UUFDWixNQUFNLE9BQU8sR0FBRyxDQUFDLEdBQUcsSUFBSSxDQUFDLE9BQU8sQ0FBQyxNQUFNLEVBQUUsQ0FBQyxDQUFDLE9BQU8sQ0FBQyxDQUFDLEdBQUcsRUFBRSxFQUFFLENBQUMsQ0FBQyxHQUFHLEdBQUcsQ0FBQyxDQUFDLENBQUE7UUFFckUsSUFBSSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsb0JBQW9CLEVBQUUsRUFBRSxLQUFLLEVBQUUsT0FBTyxDQUFDLE1BQU0sRUFBRSxDQUFDLENBQUE7UUFFL0QsK0VBQStFO1FBQy9FLCtFQUErRTtRQUMvRSxvQkFBb0I7UUFDcEIsS0FBSyxNQUFNLE1BQU0sSUFBSSxPQUFPO1lBQUUsTUFBTSxDQUFDLEtBQUssRUFBRSxDQUFBO0lBQzlDLENBQUM7SUFFTSxLQUFLLENBQUMsT0FBTyxDQUFDLEtBQVk7UUFDL0IsTUFBTSxHQUFHLEdBQUcsS0FBSyxDQUFDLEdBQUcsQ0FBQTtRQUNyQixNQUFNLE1BQU0sR0FBRyxJQUFJLENBQUMsWUFBWSxDQUFDLEdBQUcsQ0FBQyxDQUFBO1FBRXJDLFVBQVU7UUFDVixVQUFVLENBQUMsR0FBRyxFQUFFLENBQUMsTUFBTSxDQUFDLFNBQVMsRUFBRSxFQUFFLElBQUksQ0FBQyxDQUFBO1FBRTFDLElBQUksS0FBSyxDQUFDLEtBQUssS0FBSyxTQUFTO1lBQzNCLEtBQUssSUFBSSxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsR0FBRyxDQUFDLENBQUMsSUFBSSxDQUFDLENBQUMsS0FBSyxFQUFFLEVBQUU7Z0JBQzFDLElBQUksS0FBSyxZQUFZLEtBQUs7b0JBQ3hCLElBQUksQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLDRCQUE0QixFQUFFLEVBQUUsR0FBRyxFQUFFLEtBQUssRUFBRSxLQUFLLEVBQUUsQ0FBQyxDQUFBOztvQkFDakUsTUFBTSxDQUFDLElBQUksQ0FBQyxFQUFFLEtBQUssRUFBRSxPQUFPLEVBQUUsSUFBSSxFQUFFLEtBQUssRUFBRSxDQUFDLENBQUE7WUFDbkQsQ0FBQyxDQUFDLENBQUE7O1lBRUYsS0FBSyxJQUFJLENBQUMsS0FBSyxDQUFDLEdBQUcsQ0FBQyxHQUFHLEVBQUUsS0FBSyxDQUFDLEtBQUssQ0FBQyxDQUFDLElBQUksQ0FBQyxDQUFDLE1BQU0sRUFBRSxFQUFFO2dCQUNwRCxJQUFJLE1BQU0sS0FBSyxJQUFJO29CQUFFLE9BQU07Z0JBRTNCLElBQUksTUFBTSxJQUFJLE1BQU0sSUFBSSxNQUFNLENBQUMsSUFBSSxLQUFLLFlBQVk7b0JBQUUsT0FBTTtnQkFFNUQsSUFBSSxNQUFNLFlBQVksS0FBSyxFQUFFLENBQUM7b0JBQzVCLElBQUksQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLDBCQUEwQixFQUFFLEVBQUUsR0FBRyxFQUFFLEtBQUssRUFBRSxNQUFNLEVBQUUsQ0FBQyxDQUFBO29CQUVuRSxPQUFNO2dCQUNSLENBQUM7Z0JBRUQsSUFBSSxNQUFNLENBQUMsU0FBUztvQkFBRSxPQUFNO2dCQUU1QixNQUFNLENBQUMsS0FBSyxFQUFFLE1BQU0sQ0FBQyxHQUFHLE1BQU0sQ0FBQTtnQkFFOUIsS0FBSyxNQUFNLEtBQUssSUFBSSxNQUFpQjtvQkFDbkMsTUFBTSxDQUFDLElBQUksQ0FBQyxFQUFFLEtBQUssRUFBRSxLQUFLLENBQUMsS0FBSyxFQUFFLElBQUksRUFBRSxLQUFLLENBQUMsSUFBSSxFQUFFLENBQUMsQ0FBQTtnQkFFdkQsTUFBTSxDQUFDLElBQUksQ0FBQyxFQUFFLEtBQUssRUFBRSxPQUFPLEVBQUUsSUFBSSxFQUFFLEtBQUssRUFBRSxDQUFDLENBQUE7WUFDOUMsQ0FBQyxDQUFDLENBQUE7UUFFSixPQUFPLE1BQU0sQ0FBQTtJQUNmLENBQUM7SUFFTyxZQUFZLENBQUMsR0FBVztRQUM5QixNQUFNLE1BQU0sR0FBRyxJQUFJLE1BQU0sRUFBRSxDQUFBO1FBQzNCLElBQUksR0FBRyxHQUFHLElBQUksQ0FBQyxPQUFPLENBQUMsR0FBRyxDQUFDLEdBQUcsQ0FBQyxDQUFBO1FBRS9CLElBQUksR0FBRyxLQUFLLFNBQVMsRUFBRSxDQUFDO1lBQ3RCLEdBQUcsR0FBRyxJQUFJLEdBQUcsRUFBRSxDQUFBO1lBQ2YsSUFBSSxDQUFDLE9BQU8sQ0FBQyxHQUFHLENBQUMsR0FBRyxFQUFFLEdBQUcsQ0FBQyxDQUFBO1FBQzVCLENBQUM7UUFFRCxHQUFHLENBQUMsR0FBRyxDQUFDLE1BQU0sQ0FBQyxDQUFBO1FBQ2YsSUFBSSxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMsZ0JBQWdCLEVBQUUsRUFBRSxHQUFHLEVBQUUsS0FBSyxFQUFFLEdBQUcsQ0FBQyxJQUFJLEVBQUUsQ0FBQyxDQUFBO1FBRTNELE1BQU0sQ0FBQyxNQUFNLENBQUMsSUFBSSxDQUFDLFNBQVMsRUFBRSxHQUFHLEVBQUU7WUFDakMsR0FBRyxDQUFDLE1BQU0sQ0FBQyxNQUFNLENBQUMsQ0FBQTtZQUVsQixJQUFJLEdBQUcsQ0FBQyxJQUFJLEtBQUssQ0FBQztnQkFBRSxJQUFJLENBQUMsT0FBTyxDQUFDLE1BQU0sQ0FBQyxHQUFHLENBQUMsQ0FBQTtZQUU1QyxJQUFJLENBQUMsSUFBSSxDQUFDLEtBQUssQ0FBQyxrQkFBa0IsRUFBRSxFQUFFLEdBQUcsRUFBRSxLQUFLLEVBQUUsR0FBRyxDQUFDLElBQUksRUFBRSxDQUFDLENBQUE7UUFDL0QsQ0FBQyxDQUFDLENBQUE7UUFFRixPQUFPLE1BQU0sQ0FBQTtJQUNmLENBQUM7Q0FDRiJ9

package/components/realtime.streams/operations/lib/Stash.js

@@ -0,0 +1,86 @@
+export class Stash {
+    stash;
+    configuration;
+    logs;
+    constructor(stash, configuration, logs) {
+        this.stash = stash;
+        this.configuration = configuration;
+        this.logs = logs;
+    }
+    async connect(key) {
+        return await this.xadd(key, 'connect');
+    }
+    async push(key, event, data) {
+        return await this.xadd(key, event, data);
+    }
+    async pop(key, token) {
+        const stamp = this.decode(token);
+        const results = await this.stash
+            .xread('STREAMS', key, stamp)
+            .catch((error) => error);
+        if (results === null)
+            return ERR_NO_RESULTS;
+        if (results instanceof Error)
+            return results;
+        const items = entries(results);
+        if (items === null)
+            return null;
+        const events = [];
+        let lastStamp = null;
+        for (const item of items) {
+            const [, event, , json] = item[1];
+            lastStamp = item[0];
+            const data = json === undefined ? undefined : JSON.parse(json);
+            if (data === undefined)
+                this.logs.debug('Undefined event payload', { key, event });
+            events.push({ event, data });
+        }
+        if (lastStamp === null)
+            return null;
+        return [this.encode(lastStamp), events];
+    }
+    async xadd(key, event, data) {
+        const args = ['MAXLEN', '~', this.configuration.maxlen, '*', 'type', event];
+        if (data !== undefined)
+            args.push('data', JSON.stringify(data));
+        const results = await this.stash
+            .multi()
+            .xadd(key, ...args)
+            .expire(key, this.configuration.expire)
+            .exec()
+            .catch((error) => error);
+        if (results === null)
+            return ERR_NO_RESULTS;
+        if (results instanceof Error)
+            return results;
+        const [[error, stamp]] = results;
+        if (error !== null)
+            return error;
+        return this.encode(stamp);
+    }
+    encode(token) {
+        return Buffer.from(token).toString('base64url');
+    }
+    decode(token) {
+        return Buffer.from(token, 'base64url').toString();
+    }
+}
+/**
+ * `xread` answers with an array of `[stream, entries]` pairs over RESP2 and with an
+ * object keyed by stream name over RESP3. One stream is read, so either way it is the
+ * first, and an empty answer means nothing has been added since the token.
+ */
+function entries(results) {
+    if (Array.isArray(results))
+        return results.length === 0 ? null : results[0][1];
+    if (typeof results === 'object' && results !== null) {
+        const streams = Object.values(results);
+        return streams.length === 0 ? null : streams[0];
+    }
+    return null;
+}
+class NoResultsError extends Error {
+    code = 'NO_RESULTS';
+}
+const ERR_NO_RESULTS = new NoResultsError();
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiU3Rhc2guanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zb3VyY2UvbGliL1N0YXNoLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUVBLE1BQU0sT0FBTyxLQUFLO0lBQ0MsS0FBSyxDQUFPO0lBQ1osYUFBYSxDQUFlO0lBQzVCLElBQUksQ0FBSztJQUUxQixZQUFtQixLQUFVLEVBQUUsYUFBNEIsRUFBRSxJQUFTO1FBQ3BFLElBQUksQ0FBQyxLQUFLLEdBQUcsS0FBSyxDQUFBO1FBQ2xCLElBQUksQ0FBQyxhQUFhLEdBQUcsYUFBYSxDQUFBO1FBQ2xDLElBQUksQ0FBQyxJQUFJLEdBQUcsSUFBSSxDQUFBO0lBQ2xCLENBQUM7SUFFTSxLQUFLLENBQUMsT0FBTyxDQUFDLEdBQVc7UUFDOUIsT0FBTyxNQUFNLElBQUksQ0FBQyxJQUFJLENBQUMsR0FBRyxFQUFFLFNBQVMsQ0FBQyxDQUFBO0lBQ3hDLENBQUM7SUFFTSxLQUFLLENBQUMsSUFBSSxDQUFDLEdBQVcsRUFBRSxLQUFhLEVBQUUsSUFBYTtRQUN6RCxPQUFPLE1BQU0sSUFBSSxDQUFDLElBQUksQ0FBQyxHQUFHLEVBQUUsS0FBSyxFQUFFLElBQUksQ0FBQyxDQUFBO0lBQzFDLENBQUM7SUFFTSxLQUFLLENBQUMsR0FBRyxDQUNkLEdBQVcsRUFDWCxLQUFhO1FBRWIsTUFBTSxLQUFLLEdBQUcsSUFBSSxDQUFDLE1BQU0sQ0FBQyxLQUFLLENBQUMsQ0FBQTtRQUVoQyxNQUFNLE9BQU8sR0FBRyxNQUFNLElBQUksQ0FBQyxLQUFLO2FBQzdCLEtBQUssQ0FBQyxTQUFTLEVBQUUsR0FBRyxFQUFFLEtBQUssQ0FBQzthQUM1QixLQUFLLENBQUMsQ0FBQyxLQUFZLEVBQUUsRUFBRSxDQUFDLEtBQUssQ0FBQyxDQUFBO1FBRWpDLElBQUksT0FBTyxLQUFLLElBQUk7WUFBRSxPQUFPLGNBQWMsQ0FBQTtRQUUzQyxJQUFJLE9BQU8sWUFBWSxLQUFLO1lBQUUsT0FBTyxPQUFPLENBQUE7UUFFNUMsTUFBTSxLQUFLLEdBQUcsT0FBTyxDQUFDLE9BQU8sQ0FBQyxDQUFBO1FBRTlCLElBQUksS0FBSyxLQUFLLElBQUk7WUFBRSxPQUFPLElBQUksQ0FBQTtRQUUvQixNQUFNLE1BQU0sR0FBWSxFQUFFLENBQUE7UUFFMUIsSUFBSSxTQUFTLEdBQWtCLElBQUksQ0FBQTtRQUVuQyxLQUFLLE1BQU0sSUFBSSxJQUFJLEtBQUssRUFBRSxDQUFDO1lBQ3pCLE1BQU0sQ0FBQyxFQUFFLEtBQUssRUFBRSxBQUFELEVBQUcsSUFBSSxDQUFDLEdBQUcsSUFBSSxDQUFDLENBQUMsQ0FBQyxDQUFBO1lBRWpDLFNBQVMsR0FBRyxJQUFJLENBQUMsQ0FBQyxDQUFDLENBQUE7WUFFbkIsTUFBTSxJQUFJLEdBQUcsSUFBSSxLQUFLLFNBQVMsQ0FBQyxDQUFDLENBQUMsU0FBUyxDQUFDLENBQUMsQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLElBQUksQ0FBQyxDQUFBO1lBRTlELElBQUksSUFBSSxLQUFLLFNBQVM7Z0JBQUUsSUFBSSxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMseUJBQXlCLEVBQUUsRUFBRSxHQUFHLEVBQUUsS0FBSyxFQUFFLENBQUMsQ0FBQTtZQUVsRixNQUFNLENBQUMsSUFBSSxDQUFDLEVBQUUsS0FBSyxFQUFFLElBQUksRUFBRSxDQUFDLENBQUE7UUFDOUIsQ0FBQztRQUVELElBQUksU0FBUyxLQUFLLElBQUk7WUFBRSxPQUFPLElBQUksQ0FBQTtRQUVuQyxPQUFPLENBQUMsSUFBSSxDQUFDLE1BQU0sQ0FBQyxTQUFTLENBQUMsRUFBRSxNQUFNLENBQUMsQ0FBQTtJQUN6QyxDQUFDO0lBRU8sS0FBSyxDQUFDLElBQUksQ0FDaEIsR0FBVyxFQUNYLEtBQWEsRUFDYixJQUFjO1FBRWQsTUFBTSxJQUFJLEdBQUcsQ0FBQyxRQUFRLEVBQUUsR0FBRyxFQUFFLElBQUksQ0FBQyxhQUFhLENBQUMsTUFBTSxFQUFFLEdBQUcsRUFBRSxNQUFNLEVBQUUsS0FBSyxDQUFDLENBQUE7UUFFM0UsSUFBSSxJQUFJLEtBQUssU0FBUztZQUFFLElBQUksQ0FBQyxJQUFJLENBQUMsTUFBTSxFQUFFLElBQUksQ0FBQyxTQUFTLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQTtRQUUvRCxNQUFNLE9BQU8sR0FBRyxNQUFNLElBQUksQ0FBQyxLQUFLO2FBQzdCLEtBQUssRUFBRTthQUNQLElBQUksQ0FBQyxHQUFHLEVBQUUsR0FBRyxJQUFJLENBQUM7YUFDbEIsTUFBTSxDQUFDLEdBQUcsRUFBRSxJQUFJLENBQUMsYUFBYSxDQUFDLE1BQU0sQ0FBQzthQUN0QyxJQUFJLEVBQUU7YUFDTixLQUFLLENBQUMsQ0FBQyxLQUFZLEVBQUUsRUFBRSxDQUFDLEtBQUssQ0FBQyxDQUFBO1FBRWpDLElBQUksT0FBTyxLQUFLLElBQUk7WUFBRSxPQUFPLGNBQWMsQ0FBQTtRQUUzQyxJQUFJLE9BQU8sWUFBWSxLQUFLO1lBQUUsT0FBTyxPQUFPLENBQUE7UUFFNUMsTUFBTSxDQUFDLENBQUMsS0FBSyxFQUFFLEtBQUssQ0FBQyxDQUFDLEdBQUcsT0FBTyxDQUFBO1FBRWhDLElBQUksS0FBSyxLQUFLLElBQUk7WUFBRSxPQUFPLEtBQUssQ0FBQTtRQUVoQyxPQUFPLElBQUksQ0FBQyxNQUFNLENBQUMsS0FBZSxDQUFDLENBQUE7SUFDckMsQ0FBQztJQUVPLE1BQU0sQ0FBQyxLQUFhO1FBQzFCLE9BQU8sTUFBTSxDQUFDLElBQUksQ0FBQyxLQUFLLENBQUMsQ0FBQyxRQUFRLENBQUMsV0FBVyxDQUFDLENBQUE7SUFDakQsQ0FBQztJQUVPLE1BQU0sQ0FBQyxLQUFhO1FBQzFCLE9BQU8sTUFBTSxDQUFDLElBQUksQ0FBQyxLQUFLLEVBQUUsV0FBVyxDQUFDLENBQUMsUUFBUSxFQUFFLENBQUE7SUFDbkQsQ0FBQztDQUNGO0FBRUQ7Ozs7R0FJRztBQUNILFNBQVMsT0FBTyxDQUFDLE9BQWdCO0lBQy9CLElBQUksS0FBSyxDQUFDLE9BQU8sQ0FBQyxPQUFPLENBQUM7UUFDeEIsT0FBTyxPQUFPLENBQUMsTUFBTSxLQUFLLENBQUMsQ0FBQyxDQUFDLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBRSxPQUFPLENBQUMsQ0FBQyxDQUF1QixDQUFDLENBQUMsQ0FBQyxDQUFBO0lBRTNFLElBQUksT0FBTyxPQUFPLEtBQUssUUFBUSxJQUFJLE9BQU8sS0FBSyxJQUFJLEVBQUUsQ0FBQztRQUNwRCxNQUFNLE9BQU8sR0FBRyxNQUFNLENBQUMsTUFBTSxDQUFDLE9BQWtDLENBQUMsQ0FBQTtRQUVqRSxPQUFPLE9BQU8sQ0FBQyxNQUFNLEtBQUssQ0FBQyxDQUFDLENBQUMsQ0FBQyxJQUFJLENBQUMsQ0FBQyxDQUFDLE9BQU8sQ0FBQyxDQUFDLENBQUMsQ0FBQTtJQUNqRCxDQUFDO0lBRUQsT0FBTyxJQUFJLENBQUE7QUFDYixDQUFDO0FBY0QsTUFBTSxjQUFlLFNBQVEsS0FBSztJQUNoQixJQUFJLEdBQUcsWUFBWSxDQUFBO0NBQ3BDO0FBRUQsTUFBTSxjQUFjLEdBQUcsSUFBSSxjQUFjLEVBQUUsQ0FBQSJ9

package/components/realtime.streams/operations/lib/Stream.js

@@ -0,0 +1,43 @@
+import { EventEmitter } from 'node:events';
+import { Readable } from 'node:stream';
+export class Stream extends Readable {
+    events = new EventEmitter();
+    interval = null;
+    ended = false;
+    constructor() {
+        super(objectMode);
+    }
+    // has to be here
+    _read() {
+        if (this.interval === null)
+            this.interval = setInterval(() => this.heartbeat(), HEARTBEAT_INTERVAL);
+    }
+    _destroy(error, callback) {
+        if (this.interval !== null)
+            clearInterval(this.interval);
+        this.events.emit('destroy');
+        super._destroy(error, callback);
+    }
+    /** Ends the stream: whoever reads it gets EOF rather than a broken pipe. */
+    close() {
+        this.ended = true;
+        if (this.interval !== null) {
+            clearInterval(this.interval);
+            this.interval = null;
+        }
+        this.push(null);
+    }
+    heartbeat(stream = this) {
+        if (this.ended)
+            return false;
+        const resume = stream.push('heartbeat ' + Date.now());
+        if (!resume && this.interval !== null) {
+            clearInterval(this.interval);
+            this.interval = null;
+        }
+        return resume;
+    }
+}
+const HEARTBEAT_INTERVAL = 16_000; // why?
+const objectMode = { objectMode: true };
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiU3RyZWFtLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc291cmNlL2xpYi9TdHJlYW0udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLFlBQVksRUFBRSxNQUFNLGFBQWEsQ0FBQTtBQUMxQyxPQUFPLEVBQUUsUUFBUSxFQUFFLE1BQU0sYUFBYSxDQUFBO0FBRXRDLE1BQU0sT0FBTyxNQUFPLFNBQVEsUUFBUTtJQUMzQixNQUFNLEdBQUcsSUFBSSxZQUFZLEVBQUUsQ0FBQTtJQUUxQixRQUFRLEdBQTBCLElBQUksQ0FBQTtJQUN0QyxLQUFLLEdBQUcsS0FBSyxDQUFBO0lBRXJCO1FBQ0UsS0FBSyxDQUFDLFVBQVUsQ0FBQyxDQUFBO0lBQ25CLENBQUM7SUFFRCxpQkFBaUI7SUFDRCxLQUFLO1FBQ25CLElBQUksSUFBSSxDQUFDLFFBQVEsS0FBSyxJQUFJO1lBQ3hCLElBQUksQ0FBQyxRQUFRLEdBQUcsV0FBVyxDQUFDLEdBQUcsRUFBRSxDQUFDLElBQUksQ0FBQyxTQUFTLEVBQUUsRUFBRSxrQkFBa0IsQ0FBQyxDQUFBO0lBQzNFLENBQUM7SUFFZSxRQUFRLENBQ3RCLEtBQW1CLEVBQ25CLFFBQXdDO1FBRXhDLElBQUksSUFBSSxDQUFDLFFBQVEsS0FBSyxJQUFJO1lBQUUsYUFBYSxDQUFDLElBQUksQ0FBQyxRQUFRLENBQUMsQ0FBQTtRQUV4RCxJQUFJLENBQUMsTUFBTSxDQUFDLElBQUksQ0FBQyxTQUFTLENBQUMsQ0FBQTtRQUUzQixLQUFLLENBQUMsUUFBUSxDQUFDLEtBQUssRUFBRSxRQUFRLENBQUMsQ0FBQTtJQUNqQyxDQUFDO0lBRUQsNEVBQTRFO0lBQ3JFLEtBQUs7UUFDVixJQUFJLENBQUMsS0FBSyxHQUFHLElBQUksQ0FBQTtRQUVqQixJQUFJLElBQUksQ0FBQyxRQUFRLEtBQUssSUFBSSxFQUFFLENBQUM7WUFDM0IsYUFBYSxDQUFDLElBQUksQ0FBQyxRQUFRLENBQUMsQ0FBQTtZQUM1QixJQUFJLENBQUMsUUFBUSxHQUFHLElBQUksQ0FBQTtRQUN0QixDQUFDO1FBRUQsSUFBSSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQTtJQUNqQixDQUFDO0lBRU0sU0FBUyxDQUFDLE1BQU0sR0FBYSxJQUFJO1FBQ3RDLElBQUksSUFBSSxDQUFDLEtBQUs7WUFBRSxPQUFPLEtBQUssQ0FBQTtRQUU1QixNQUFNLE1BQU0sR0FBRyxNQUFNLENBQUMsSUFBSSxDQUFDLFlBQVksR0FBRyxJQUFJLENBQUMsR0FBRyxFQUFFLENBQUMsQ0FBQTtRQUVyRCxJQUFJLENBQUMsTUFBTSxJQUFJLElBQUksQ0FBQyxRQUFRLEtBQUssSUFBSSxFQUFFLENBQUM7WUFDdEMsYUFBYSxDQUFDLElBQUksQ0FBQyxRQUFRLENBQUMsQ0FBQTtZQUM1QixJQUFJLENBQUMsUUFBUSxHQUFHLElBQUksQ0FBQTtRQUN0QixDQUFDO1FBRUQsT0FBTyxNQUFNLENBQUE7SUFDZixDQUFDO0NBQ0Y7QUFFRCxNQUFNLGtCQUFrQixHQUFHLE1BQU0sQ0FBQSxDQUFDLE9BQU87QUFDekMsTUFBTSxVQUFVLEdBQUcsRUFBRSxVQUFVLEVBQUUsSUFBSSxFQUFFLENBQUEifQ==

package/components/realtime.streams/operations/lib/index.js

@@ -0,0 +1,3 @@
+export { Stash } from './Stash.js';
+export { Stream } from './Stream.js';
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiaW5kZXguanMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zb3VyY2UvbGliL2luZGV4LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxLQUFLLEVBQUUsTUFBTSxZQUFZLENBQUE7QUFDbEMsT0FBTyxFQUFFLE1BQU0sRUFBRSxNQUFNLGFBQWEsQ0FBQSJ9

package/components/realtime.streams/operations/push.js

@@ -0,0 +1,18 @@
+export async function effect({ key, event, data }, context) {
+    broadcast(context, key, { event, data });
+    void context.state.stash.push(key, event, data).then((token) => {
+        if (token instanceof Error)
+            context.logs.error('Failed to push to stash', { key, error: token });
+        else
+            broadcast(context, key, { event: 'token', data: token });
+    });
+}
+/** To every stream the key has: each is a consumer of its own. */
+function broadcast(context, key, message) {
+    const streams = context.state.streams.get(key);
+    if (streams === undefined)
+        return;
+    for (const stream of streams)
+        stream.push(message);
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoicHVzaC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3NvdXJjZS9wdXNoLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUVBLE1BQU0sQ0FBQyxLQUFLLFVBQVUsTUFBTSxDQUMxQixFQUFFLEdBQUcsRUFBRSxLQUFLLEVBQUUsSUFBSSxFQUFhLEVBQy9CLE9BQWdCO0lBRWhCLFNBQVMsQ0FBQyxPQUFPLEVBQUUsR0FBRyxFQUFFLEVBQUUsS0FBSyxFQUFFLElBQUksRUFBRSxDQUFDLENBQUE7SUFFeEMsS0FBSyxPQUFPLENBQUMsS0FBSyxDQUFDLEtBQUssQ0FBQyxJQUFJLENBQUMsR0FBRyxFQUFFLEtBQUssRUFBRSxJQUFJLENBQUMsQ0FBQyxJQUFJLENBQUMsQ0FBQyxLQUFLLEVBQUUsRUFBRTtRQUM3RCxJQUFJLEtBQUssWUFBWSxLQUFLO1lBQ3hCLE9BQU8sQ0FBQyxJQUFJLENBQUMsS0FBSyxDQUFDLHlCQUF5QixFQUFFLEVBQUUsR0FBRyxFQUFFLEtBQUssRUFBRSxLQUFLLEVBQUUsQ0FBQyxDQUFBOztZQUNqRSxTQUFTLENBQUMsT0FBTyxFQUFFLEdBQUcsRUFBRSxFQUFFLEtBQUssRUFBRSxPQUFPLEVBQUUsSUFBSSxFQUFFLEtBQUssRUFBRSxDQUFDLENBQUE7SUFDL0QsQ0FBQyxDQUFDLENBQUE7QUFDSixDQUFDO0FBRUQsa0VBQWtFO0FBQ2xFLFNBQVMsU0FBUyxDQUFDLE9BQWdCLEVBQUUsR0FBVyxFQUFFLE9BQWU7SUFDL0QsTUFBTSxPQUFPLEdBQUcsT0FBTyxDQUFDLEtBQUssQ0FBQyxPQUFPLENBQUMsR0FBRyxDQUFDLEdBQUcsQ0FBQyxDQUFBO0lBRTlDLElBQUksT0FBTyxLQUFLLFNBQVM7UUFBRSxPQUFNO0lBRWpDLEtBQUssTUFBTSxNQUFNLElBQUksT0FBTztRQUFFLE1BQU0sQ0FBQyxJQUFJLENBQUMsT0FBTyxDQUFDLENBQUE7QUFDcEQsQ0FBQyJ9

package/components/realtime.streams/source/create.ts

@@ -0,0 +1,107 @@
+import { type Readable } from 'node:stream'
+import { type Operation } from '@toa.io/bridges.node'
+import type { Context } from '../types/index.d.ts'
+import { Stream, Stash } from './lib/index.ts'
+
+/**
+ * A stream is opened per call, and a key has as many of them as it has consumers: one stream read
+ * by several would hand each of its values to one of them only, and a consumer leaving would
+ * destroy it under the others. What arrives under a key is pushed to every stream of it; what
+ * belongs to a connection — the welcome, the token, the events missed since a token — to that
+ * connection's stream alone.
+ */
+export class Effect implements Operation {
+  private readonly streams = new Map<string, Set<Stream>>()
+  private stash!: Stash
+  private logs: any
+
+  public mount(context: Context): void {
+    context.state.streams = this.streams
+    context.state.stash = new Stash(context.stash, context.configuration, context.logs)
+
+    this.logs = context.logs
+    this.stash = context.state.stash
+  }
+
+  public unmount(): void {
+    const streams = [...this.streams.values()].flatMap((set) => [...set])
+
+    this.logs.info('Destroying streams', { count: streams.length })
+
+    // closed, not destroyed: destroying a stream that is still piped to a response
+    // makes end-of-stream report a premature close, which reaches no one and takes
+    // the process down.
+    for (const stream of streams) stream.close()
+  }
+
+  public async execute(input: Input): Promise<Readable> {
+    const key = input.key
+    const stream = this.createStream(key)
+
+    // welcome
+    setTimeout(() => stream.heartbeat(), 1000)
+
+    if (input.token === undefined)
+      void this.stash.connect(key).then((token) => {
+        if (token instanceof Error)
+          this.logs.error('Failed to connect to stash', { key, error: token })
+        else stream.push({ event: 'token', data: token })
+      })
+    else
+      void this.stash.pop(key, input.token).then((result) => {
+        if (result === null) return
+
+        if ('code' in result && result.code === 'NO_RESULTS') return
+
+        if (result instanceof Error) {
+          this.logs.error('Failed to pop from stash', { key, error: result })
+
+          return
+        }
+
+        if (stream.destroyed) return
+
+        const [token, events] = result
+
+        for (const event of events as Event[])
+          stream.push({ event: event.event, data: event.data })
+
+        stream.push({ event: 'token', data: token })
+      })
+
+    return stream
+  }
+
+  private createStream(key: string): Stream {
+    const stream = new Stream()
+    let set = this.streams.get(key)
+
+    if (set === undefined) {
+      set = new Set()
+      this.streams.set(key, set)
+    }
+
+    set.add(stream)
+    this.logs.debug('Stream created', { key, count: set.size })
+
+    stream.events.once('destroy', () => {
+      set.delete(stream)
+
+      if (set.size === 0) this.streams.delete(key)
+
+      this.logs.debug('Stream destroyed', { key, count: set.size })
+    })
+
+    return stream
+  }
+}
+
+interface Input {
+  key: string
+  token?: string
+}
+
+interface Event {
+  event: string
+  data: unknown
+}

package/components/realtime.streams/source/lib/Stash.ts

@@ -0,0 +1,131 @@
+import type { Redis } from 'ioredis'
+
+export class Stash {
+  private readonly stash: Redis
+  private readonly configuration: Configuration
+  private readonly logs: any
+
+  public constructor(stash: any, configuration: Configuration, logs: any) {
+    this.stash = stash
+    this.configuration = configuration
+    this.logs = logs
+  }
+
+  public async connect(key: string): Promise<string | Error> {
+    return await this.xadd(key, 'connect')
+  }
+
+  public async push(key: string, event: string, data: unknown): Promise<string | Error> {
+    return await this.xadd(key, event, data)
+  }
+
+  public async pop(
+    key: string,
+    token: string
+  ): Promise<[string, Event[]] | null | Error> {
+    const stamp = this.decode(token)
+
+    const results = await this.stash
+      .xread('STREAMS', key, stamp)
+      .catch((error: Error) => error)
+
+    if (results === null) return ERR_NO_RESULTS
+
+    if (results instanceof Error) return results
+
+    const items = entries(results)
+
+    if (items === null) return null
+
+    const events: Event[] = []
+
+    let lastStamp: string | null = null
+
+    for (const item of items) {
+      const [, event, , json] = item[1]
+
+      lastStamp = item[0]
+
+      const data = json === undefined ? undefined : JSON.parse(json)
+
+      if (data === undefined) this.logs.debug('Undefined event payload', { key, event })
+
+      events.push({ event, data })
+    }
+
+    if (lastStamp === null) return null
+
+    return [this.encode(lastStamp), events]
+  }
+
+  private async xadd(
+    key: string,
+    event: string,
+    data?: unknown
+  ): Promise<string | Error> {
+    const args = ['MAXLEN', '~', this.configuration.maxlen, '*', 'type', event]
+
+    if (data !== undefined) args.push('data', JSON.stringify(data))
+
+    const results = await this.stash
+      .multi()
+      .xadd(key, ...args)
+      .expire(key, this.configuration.expire)
+      .exec()
+      .catch((error: Error) => error)
+
+    if (results === null) return ERR_NO_RESULTS
+
+    if (results instanceof Error) return results
+
+    const [[error, stamp]] = results
+
+    if (error !== null) return error
+
+    return this.encode(stamp as string)
+  }
+
+  private encode(token: string): string {
+    return Buffer.from(token).toString('base64url')
+  }
+
+  private decode(token: string): string {
+    return Buffer.from(token, 'base64url').toString()
+  }
+}
+
+/**
+ * `xread` answers with an array of `[stream, entries]` pairs over RESP2 and with an
+ * object keyed by stream name over RESP3. One stream is read, so either way it is the
+ * first, and an empty answer means nothing has been added since the token.
+ */
+function entries(results: unknown): Entry[] | null {
+  if (Array.isArray(results))
+    return results.length === 0 ? null : (results[0] as [string, Entry[]])[1]
+
+  if (typeof results === 'object' && results !== null) {
+    const streams = Object.values(results as Record<string, Entry[]>)
+
+    return streams.length === 0 ? null : streams[0]
+  }
+
+  return null
+}
+
+type Entry = [stamp: string, fields: string[]]
+
+interface Configuration {
+  maxlen: number
+  expire: number
+}
+
+interface Event {
+  event: string
+  data: unknown
+}
+
+class NoResultsError extends Error {
+  public readonly code = 'NO_RESULTS'
+}
+
+const ERR_NO_RESULTS = new NoResultsError()

package/components/realtime.streams/source/lib/Stream.ts

@@ -0,0 +1,58 @@
+import { EventEmitter } from 'node:events'
+import { Readable } from 'node:stream'
+
+export class Stream extends Readable {
+  public events = new EventEmitter()
+
+  private interval: NodeJS.Timeout | null = null
+  private ended = false
+
+  public constructor() {
+    super(objectMode)
+  }
+
+  // has to be here
+  public override _read(): void {
+    if (this.interval === null)
+      this.interval = setInterval(() => this.heartbeat(), HEARTBEAT_INTERVAL)
+  }
+
+  public override _destroy(
+    error: Error | null,
+    callback: (error?: Error | null) => void
+  ): void {
+    if (this.interval !== null) clearInterval(this.interval)
+
+    this.events.emit('destroy')
+
+    super._destroy(error, callback)
+  }
+
+  /** Ends the stream: whoever reads it gets EOF rather than a broken pipe. */
+  public close(): void {
+    this.ended = true
+
+    if (this.interval !== null) {
+      clearInterval(this.interval)
+      this.interval = null
+    }
+
+    this.push(null)
+  }
+
+  public heartbeat(stream: Readable = this): boolean {
+    if (this.ended) return false
+
+    const resume = stream.push('heartbeat ' + Date.now())
+
+    if (!resume && this.interval !== null) {
+      clearInterval(this.interval)
+      this.interval = null
+    }
+
+    return resume
+  }
+}
+
+const HEARTBEAT_INTERVAL = 16_000 // why?
+const objectMode = { objectMode: true }

package/components/realtime.streams/source/lib/index.ts

@@ -0,0 +1,2 @@
+export { Stash } from './Stash.ts'
+export { Stream } from './Stream.ts'

package/components/realtime.streams/source/push.ts

@@ -0,0 +1,23 @@
+import type { Context, PushInput } from '../types/index.d.ts'
+
+export async function effect(
+  { key, event, data }: PushInput,
+  context: Context
+): Promise<void> {
+  broadcast(context, key, { event, data })
+
+  void context.state.stash.push(key, event, data).then((token) => {
+    if (token instanceof Error)
+      context.logs.error('Failed to push to stash', { key, error: token })
+    else broadcast(context, key, { event: 'token', data: token })
+  })
+}
+
+/** To every stream the key has: each is a consumer of its own. */
+function broadcast(context: Context, key: string, message: object): void {
+  const streams = context.state.streams.get(key)
+
+  if (streams === undefined) return
+
+  for (const stream of streams) stream.push(message)
+}

package/components/{streams → realtime.streams}/tsconfig.json

@@ -1,7 +1,8 @@
 {
-  "extends": "../../tsconfig.json",
+  "extends": "../../../../tsconfig.component.json",
   "compilerOptions": {
     "outDir": "./operations",
+    "rootDir": "./source"
   },
   "include": [
     "source"

package/components/realtime.streams/types/index.d.ts

@@ -0,0 +1,20 @@
+export * from './toa.d.ts'
+
+// What a manifest does not state belongs here, and every run keeps it.
+
+import type { Logs } from '@toa.io/extensions.telemetry'
+import type { Stash as Redis } from '@toa.io/extensions.stash'
+import type { Configuration } from './toa.d.ts'
+import type { Stream } from '../source/lib/Stream.ts'
+import type { Stash } from '../source/lib/Stash.ts'
+
+/** What this component keeps between calls: the streams it serves, and what backs them. */
+export interface Context {
+  stash: Redis
+  state: {
+    streams: Map<string, Set<Stream>>
+    stash: Stash
+  }
+  logs: Logs
+  configuration: Configuration
+}