@hotmeshio/long-tail 0.26.0 → 0.27.0

package/build/lib/messaging/memory.js

@@ -0,0 +1,147 @@
+"use strict";
+/**
+ * Messaging in memory, for tests and single-process development. Pub/sub fans
+ * out to every matching subscriber. Streams keep each message until it ages
+ * out, and deliver to each consumer one at a time in order; a nak'd message
+ * comes back after its delay while later messages go on, as JetStream does.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MemoryMessagingProvider = void 0;
+const constants_1 = require("./constants");
+const subjects_1 = require("./subjects");
+const DEFAULT_NAK_DELAY_MS = 1_000;
+class MemoryPubSub {
+    constructor() {
+        this.subs = new Set();
+    }
+    async publish(subject, payload) {
+        for (const sub of [...this.subs])
+            if ((0, subjects_1.subjectMatches)(sub.filter, subject))
+                sub.handler({ subject, payload });
+    }
+    subscribe(filter, handler) {
+        const sub = { filter, handler };
+        this.subs.add(sub);
+        return { close: async () => { this.subs.delete(sub); } };
+    }
+}
+class MemoryStreams {
+    constructor() {
+        this.streams = new Map();
+    }
+    async ensureStream(spec) {
+        const existing = this.streams.get(spec.name);
+        if (existing)
+            existing.spec = spec;
+        else
+            this.streams.set(spec.name, { spec, messages: [], seq: 0, consumers: new Map(), wake: new Set() });
+    }
+    async ensureConsumer(stream, spec) {
+        const s = this.stream(stream);
+        const existing = s.consumers.get(spec.name);
+        if (existing) {
+            existing.spec = { ...existing.spec, filterSubjects: spec.filterSubjects };
+            return;
+        }
+        const floor = (spec.startAt ?? constants_1.START_AT.NEW) === constants_1.START_AT.ALL ? 0 : s.seq;
+        s.consumers.set(spec.name, { spec, floor, settled: new Set(), deliveries: new Map(), retryAt: new Map() });
+    }
+    async append(subject, payload) {
+        const now = Date.now();
+        for (const s of this.streams.values()) {
+            if (!s.spec.subjects.some((f) => (0, subjects_1.subjectMatches)(f, subject)))
+                continue;
+            s.messages.push({ seq: ++s.seq, subject, payload, at: now });
+            s.messages = s.messages.filter((m) => now - m.at <= s.spec.maxAgeMs);
+            for (const wake of [...s.wake])
+                wake();
+        }
+    }
+    async consume(stream, consumer, handler, opts = {}) {
+        const s = this.stream(stream);
+        const c = s.consumers.get(consumer);
+        if (!c)
+            throw new Error(`stream ${stream} has no consumer ${consumer}; ensure it first`);
+        const nakDelayMs = opts.nakDelayMs ?? DEFAULT_NAK_DELAY_MS;
+        let open = true;
+        let running = false;
+        let timer;
+        const next = () => {
+            const now = Date.now();
+            let waitMs;
+            for (const m of s.messages) {
+                if (m.seq <= c.floor || c.settled.has(m.seq))
+                    continue;
+                if (!c.spec.filterSubjects.some((f) => (0, subjects_1.subjectMatches)(f, m.subject)))
+                    continue;
+                const at = c.retryAt.get(m.seq);
+                if (at === undefined || at <= now)
+                    return { message: m };
+                waitMs = Math.min(waitMs ?? Infinity, at - now);
+            }
+            return { waitMs };
+        };
+        const pump = async () => {
+            if (running || !open)
+                return;
+            running = true;
+            try {
+                for (let step = next(); open && step.message; step = next()) {
+                    const m = step.message;
+                    const deliveries = (c.deliveries.get(m.seq) ?? 0) + 1;
+                    c.deliveries.set(m.seq, deliveries);
+                    const outcome = await handler({ subject: m.subject, payload: m.payload, deliveries }).catch(() => constants_1.DELIVERY.NAK);
+                    if (outcome === constants_1.DELIVERY.NAK) {
+                        c.retryAt.set(m.seq, Date.now() + nakDelayMs);
+                    }
+                    else {
+                        c.settled.add(m.seq);
+                        c.retryAt.delete(m.seq);
+                    }
+                }
+                const { waitMs } = next();
+                if (open && waitMs !== undefined) {
+                    clearTimeout(timer);
+                    timer = setTimeout(() => void pump(), Math.max(0, waitMs));
+                }
+            }
+            finally {
+                running = false;
+            }
+        };
+        const wake = () => void pump();
+        s.wake.add(wake);
+        void pump();
+        return {
+            close: async () => {
+                open = false;
+                clearTimeout(timer);
+                s.wake.delete(wake);
+            },
+        };
+    }
+    stream(name) {
+        const s = this.streams.get(name);
+        if (!s)
+            throw new Error(`no stream ${name}; ensure it first`);
+        return s;
+    }
+}
+class MemoryMessagingProvider {
+    constructor() {
+        this.name = 'memory';
+        this.pubsub = new MemoryPubSub();
+        this.streams = new MemoryStreams();
+        this.live = false;
+    }
+    get connected() {
+        return this.live;
+    }
+    async connect() {
+        this.live = true;
+    }
+    async disconnect() {
+        this.live = false;
+    }
+}
+exports.MemoryMessagingProvider = MemoryMessagingProvider;

package/build/lib/messaging/nats/permissions.d.ts

@@ -0,0 +1,13 @@
+/**
+ * JetStream rights scoped to named streams and one durable consumer: look up,
+ * create and update those streams, create, read and acknowledge through that
+ * consumer, and nothing else. Deleting or purging a stream is not granted.
+ */
+export interface JsScope {
+    streams: string[];
+    durable: string;
+}
+export declare function jsPermissions({ streams, durable }: JsScope): {
+    publish: string[];
+    subscribe: string[];
+};

package/build/lib/messaging/nats/permissions.js

@@ -0,0 +1,22 @@
+"use strict";
+/**
+ * JetStream rights scoped to named streams and one durable consumer: look up,
+ * create and update those streams, create, read and acknowledge through that
+ * consumer, and nothing else. Deleting or purging a stream is not granted.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.jsPermissions = jsPermissions;
+function jsPermissions({ streams, durable }) {
+    const perStream = streams.flatMap((s) => [
+        `$JS.API.STREAM.INFO.${s}`,
+        `$JS.API.STREAM.CREATE.${s}`,
+        `$JS.API.STREAM.UPDATE.${s}`,
+        `$JS.API.CONSUMER.INFO.${s}.${durable}`,
+        `$JS.API.CONSUMER.CREATE.${s}.${durable}`,
+        `$JS.API.CONSUMER.CREATE.${s}.${durable}.>`,
+        `$JS.API.CONSUMER.DURABLE.CREATE.${s}.${durable}`,
+        `$JS.API.CONSUMER.MSG.NEXT.${s}.${durable}`,
+        `$JS.ACK.${s}.${durable}.>`,
+    ]);
+    return { publish: ['$JS.API.INFO', ...perStream], subscribe: ['_INBOX.>'] };
+}

package/build/lib/messaging/nats/provider.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Messaging on NATS. One connection carries pub/sub (core NATS subjects) and,
+ * when the server runs JetStream for this account, durable streams. Without
+ * JetStream the provider still serves pub/sub and reports no streams.
+ */
+import type { LTMessagingProvider, LTPubSub, LTStreams } from '../../../types/messaging';
+export interface NatsMessagingOptions {
+    url: string;
+    /** Token auth, or user and password; neither for an open server. */
+    token?: string;
+    user?: string;
+    password?: string;
+    /** The connection's name on the server. */
+    name?: string;
+}
+export declare class NatsMessagingProvider implements LTMessagingProvider {
+    private readonly opts;
+    readonly name = "nats";
+    readonly pubsub: LTPubSub;
+    private nc;
+    private durable;
+    constructor(opts: NatsMessagingOptions);
+    get connected(): boolean;
+    get streams(): LTStreams | null;
+    connect(): Promise<void>;
+    disconnect(): Promise<void>;
+    /** Streams when the account has JetStream; null otherwise, and pub/sub still serves. */
+    private jetStream;
+    private connection;
+}

package/build/lib/messaging/nats/provider.js

@@ -0,0 +1,77 @@
+"use strict";
+/**
+ * Messaging on NATS. One connection carries pub/sub (core NATS subjects) and,
+ * when the server runs JetStream for this account, durable streams. Without
+ * JetStream the provider still serves pub/sub and reports no streams.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.NatsMessagingProvider = void 0;
+const nats_1 = require("nats");
+const logger_1 = require("../../logger");
+const constants_1 = require("../constants");
+const streams_1 = require("./streams");
+class NatsPubSub {
+    constructor(nc) {
+        this.nc = nc;
+    }
+    async publish(subject, payload) {
+        this.nc().publish(subject, payload);
+    }
+    subscribe(filter, handler) {
+        const sub = this.nc().subscribe(filter, {
+            callback: (err, m) => { if (!err)
+                handler({ subject: m.subject, payload: (0, streams_1.payloadOf)(m) }); },
+        });
+        return { close: async () => { sub.unsubscribe(); } };
+    }
+}
+class NatsMessagingProvider {
+    constructor(opts) {
+        this.opts = opts;
+        this.name = 'nats';
+        this.pubsub = new NatsPubSub(() => this.connection());
+        this.nc = null;
+        this.durable = null;
+    }
+    get connected() {
+        return !!this.nc && !this.nc.isClosed();
+    }
+    get streams() {
+        return this.durable;
+    }
+    async connect() {
+        if (this.connected)
+            return;
+        const { url, token, user, password, name } = this.opts;
+        this.nc = await (0, nats_1.connect)({ servers: url, token, user, pass: password, name });
+        this.durable = await this.jetStream(this.nc);
+    }
+    async disconnect() {
+        if (!this.nc)
+            return;
+        await this.nc.drain();
+        this.nc = null;
+        this.durable = null;
+    }
+    /** Streams when the account has JetStream; null otherwise, and pub/sub still serves. */
+    async jetStream(nc) {
+        try {
+            const jsm = await nc.jetstreamManager();
+            await jsm.getAccountInfo();
+            return new streams_1.NatsStreams(jsm, nc.jetstream());
+        }
+        catch (err) {
+            const reason = String(err?.message ?? err?.code ?? err);
+            if (!/jetstream not enabled|503|no responders/i.test(reason)) {
+                logger_1.loggerRegistry.warn(`${constants_1.MESSAGING_LOG.STREAMS_UNAVAILABLE} provider=${this.name} reason=${reason}`);
+            }
+            return null;
+        }
+    }
+    connection() {
+        if (!this.nc)
+            throw new Error('NATS messaging is not connected; call connect() first');
+        return this.nc;
+    }
+}
+exports.NatsMessagingProvider = NatsMessagingProvider;

package/build/lib/messaging/nats/render-conf.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Long Tail's NATS server, rendered. Accounts keep traffic apart:
+ *
+ * - LT: Long Tail itself. It owns the durable streams and imports each edge's
+ *   subjects, which its streams can then keep.
+ * - one edge account per line: the site broker's bridge logs in here (MQTT
+ *   only) and may publish its line's subjects and nothing else. The account
+ *   exports exactly those subjects, so nothing of Long Tail's is reachable.
+ * - SYS: the server's own monitoring.
+ *
+ * NATS configuration accepts JSON, so accounts are written as JSON.
+ */
+export interface NatsPermissions {
+    publish?: string[] | {
+        allow?: string[];
+        deny?: string[];
+    };
+    subscribe?: string[] | {
+        allow?: string[];
+        deny?: string[];
+    };
+}
+export interface NatsUser {
+    user: string;
+    password?: string;
+    permissions?: NatsPermissions;
+    allowed_connection_types?: string[];
+}
+/** A line's edge: who logs in for it, and the subjects it may publish (e.g. `line.v1.site-1.line-1.>`). */
+export interface EdgeAccount {
+    name: string;
+    subjects: string;
+    users: NatsUser[];
+}
+export interface NatsConfOptions {
+    serverName: string;
+    port?: number;
+    httpPort?: number;
+    jetstream?: {
+        storeDir: string;
+    };
+    /** WebSocket listener for browsers; `noAuthUser` maps a connection without credentials to that LT user. */
+    websocket?: {
+        port: number;
+        noTls?: boolean;
+        noAuthUser?: string;
+    };
+    /** MQTT listener for edge bridges; with `tls`, certificates map to users by their subject. */
+    mqtt?: {
+        port: number;
+        tls?: {
+            certFile: string;
+            keyFile: string;
+            caFile: string;
+        };
+    };
+    lt: {
+        users: NatsUser[];
+    };
+    edges: EdgeAccount[];
+    system?: {
+        user: string;
+        password: string;
+    };
+}
+export declare const LT_ACCOUNT = "LT";
+export declare const SYS_ACCOUNT = "SYS";
+export declare function renderNatsConf(opts: NatsConfOptions): string;

package/build/lib/messaging/nats/render-conf.js

@@ -0,0 +1,58 @@
+"use strict";
+/**
+ * Long Tail's NATS server, rendered. Accounts keep traffic apart:
+ *
+ * - LT: Long Tail itself. It owns the durable streams and imports each edge's
+ *   subjects, which its streams can then keep.
+ * - one edge account per line: the site broker's bridge logs in here (MQTT
+ *   only) and may publish its line's subjects and nothing else. The account
+ *   exports exactly those subjects, so nothing of Long Tail's is reachable.
+ * - SYS: the server's own monitoring.
+ *
+ * NATS configuration accepts JSON, so accounts are written as JSON.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SYS_ACCOUNT = exports.LT_ACCOUNT = void 0;
+exports.renderNatsConf = renderNatsConf;
+exports.LT_ACCOUNT = 'LT';
+exports.SYS_ACCOUNT = 'SYS';
+function accounts(opts) {
+    const out = {
+        [exports.LT_ACCOUNT]: {
+            jetstream: opts.jetstream ? 'enabled' : 'disabled',
+            users: opts.lt.users,
+            imports: opts.edges.map((e) => ({ stream: { account: e.name, subject: e.subjects } })),
+        },
+    };
+    for (const edge of opts.edges) {
+        // The edge account has JetStream because NATS keeps MQTT session state there.
+        out[edge.name] = { jetstream: opts.jetstream ? 'enabled' : 'disabled', users: edge.users, exports: [{ stream: edge.subjects }] };
+    }
+    if (opts.system)
+        out[exports.SYS_ACCOUNT] = { users: [opts.system] };
+    return out;
+}
+function renderNatsConf(opts) {
+    const lines = [
+        '# Rendered by Long Tail (lib/messaging/nats/render-conf.ts). Do not edit by hand.',
+        `server_name: ${JSON.stringify(opts.serverName)}`,
+        `port: ${opts.port ?? 4222}`,
+        `http: ${opts.httpPort ?? 8222}`,
+    ];
+    if (opts.jetstream)
+        lines.push(`jetstream { store_dir: ${JSON.stringify(opts.jetstream.storeDir)} }`);
+    if (opts.websocket) {
+        const ws = opts.websocket;
+        lines.push(`websocket { port: ${ws.port}${ws.noTls ? ', no_tls: true, same_origin: false' : ''}${ws.noAuthUser ? `, no_auth_user: ${JSON.stringify(ws.noAuthUser)}` : ''} }`);
+    }
+    if (opts.mqtt) {
+        const tls = opts.mqtt.tls;
+        lines.push(tls
+            ? `mqtt { port: ${opts.mqtt.port}, tls { cert_file: ${JSON.stringify(tls.certFile)}, key_file: ${JSON.stringify(tls.keyFile)}, ca_file: ${JSON.stringify(tls.caFile)}, verify_and_map: true, timeout: 5 } }`
+            : `mqtt { port: ${opts.mqtt.port} }`);
+    }
+    lines.push(`accounts: ${JSON.stringify(accounts(opts), null, 2)}`);
+    if (opts.system)
+        lines.push(`system_account: ${exports.SYS_ACCOUNT}`);
+    return `${lines.join('\n')}\n`;
+}

package/build/lib/messaging/nats/streams.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Durable streams on JetStream. A spec becomes a file-backed stream or an
+ * explicit-ack durable consumer; ensuring either is safe on every boot, and an
+ * existing consumer keeps its position while its subjects follow the spec.
+ */
+import { type ConsumerConfig, type JetStreamClient, type JetStreamManager, type StreamConfig } from 'nats';
+import type { ConsumerSpec, Delivery, LTConsumeOptions, LTStreamMessage, LTStreams, LTSubscription, StreamSpec, Subject } from '../../../types/messaging';
+export declare function streamConfig(spec: StreamSpec): Partial<StreamConfig>;
+export declare function consumerConfig(spec: ConsumerSpec): Partial<ConsumerConfig>;
+/** Create the stream, or update an existing one to the spec. */
+export declare function ensureStream(jsm: JetStreamManager, spec: StreamSpec): Promise<void>;
+/** Create the consumer, or move an existing one to the spec's subjects without losing its place. */
+export declare function ensureConsumer(jsm: JetStreamManager, stream: string, spec: ConsumerSpec): Promise<void>;
+export declare function payloadOf(m: {
+    data: Uint8Array;
+}): string;
+export declare class NatsStreams implements LTStreams {
+    private readonly jsm;
+    private readonly js;
+    constructor(jsm: JetStreamManager, js: JetStreamClient);
+    ensureStream(spec: StreamSpec): Promise<void>;
+    ensureConsumer(stream: string, spec: ConsumerSpec): Promise<void>;
+    append(subject: Subject, payload: string): Promise<void>;
+    /** One message at a time: the next is not taken until the handler settles this one. */
+    consume(stream: string, consumer: string, handler: (m: LTStreamMessage) => Promise<Delivery>, opts?: LTConsumeOptions): Promise<LTSubscription>;
+}

package/build/lib/messaging/nats/streams.js

@@ -0,0 +1,97 @@
+"use strict";
+/**
+ * Durable streams on JetStream. A spec becomes a file-backed stream or an
+ * explicit-ack durable consumer; ensuring either is safe on every boot, and an
+ * existing consumer keeps its position while its subjects follow the spec.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.NatsStreams = void 0;
+exports.streamConfig = streamConfig;
+exports.consumerConfig = consumerConfig;
+exports.ensureStream = ensureStream;
+exports.ensureConsumer = ensureConsumer;
+exports.payloadOf = payloadOf;
+const nats_1 = require("nats");
+const constants_1 = require("../constants");
+const DEFAULT_ACK_WAIT_MS = 30_000;
+const DEFAULT_NAK_DELAY_MS = 1_000;
+function streamConfig(spec) {
+    return { name: spec.name, subjects: spec.subjects, max_age: (0, nats_1.nanos)(spec.maxAgeMs), storage: nats_1.StorageType.File };
+}
+function consumerConfig(spec) {
+    return {
+        durable_name: spec.name,
+        ack_policy: nats_1.AckPolicy.Explicit,
+        deliver_policy: (spec.startAt ?? constants_1.START_AT.NEW) === constants_1.START_AT.ALL ? nats_1.DeliverPolicy.All : nats_1.DeliverPolicy.New,
+        ack_wait: (0, nats_1.nanos)(spec.ackWaitMs ?? DEFAULT_ACK_WAIT_MS),
+        filter_subjects: spec.filterSubjects,
+    };
+}
+/** Create the stream, or update an existing one to the spec. */
+async function ensureStream(jsm, spec) {
+    const config = streamConfig(spec);
+    try {
+        await jsm.streams.info(spec.name);
+    }
+    catch (err) {
+        if (!/stream not found/i.test(err?.message ?? ''))
+            throw err;
+        await jsm.streams.add(config);
+        return;
+    }
+    await jsm.streams.update(spec.name, config);
+}
+/** Create the consumer, or move an existing one to the spec's subjects without losing its place. */
+async function ensureConsumer(jsm, stream, spec) {
+    let existing;
+    try {
+        existing = await jsm.consumers.info(stream, spec.name);
+    }
+    catch (err) {
+        if (!/consumer not found/i.test(err?.message ?? ''))
+            throw err;
+        await jsm.consumers.add(stream, consumerConfig(spec));
+        return;
+    }
+    const want = [...spec.filterSubjects].sort();
+    const have = [...(existing.config.filter_subjects ?? [])].sort();
+    if (want.join('|') !== have.join('|'))
+        await jsm.consumers.update(stream, spec.name, { filter_subjects: spec.filterSubjects });
+}
+function payloadOf(m) {
+    return Buffer.from(m.data).toString('utf8');
+}
+class NatsStreams {
+    constructor(jsm, js) {
+        this.jsm = jsm;
+        this.js = js;
+    }
+    ensureStream(spec) {
+        return ensureStream(this.jsm, spec);
+    }
+    ensureConsumer(stream, spec) {
+        return ensureConsumer(this.jsm, stream, spec);
+    }
+    async append(subject, payload) {
+        await this.js.publish(subject, payload);
+    }
+    /** One message at a time: the next is not taken until the handler settles this one. */
+    async consume(stream, consumer, handler, opts = {}) {
+        const nakDelayMs = opts.nakDelayMs ?? DEFAULT_NAK_DELAY_MS;
+        const messages = await (await this.js.consumers.get(stream, consumer)).consume();
+        void (async () => {
+            for await (const m of messages) {
+                const delivery = await handler({ subject: m.subject, payload: payloadOf(m), deliveries: m.info.deliveryCount })
+                    .catch(() => constants_1.DELIVERY.NAK);
+                if (delivery === constants_1.DELIVERY.ACK)
+                    m.ack();
+                else if (delivery === constants_1.DELIVERY.NAK)
+                    m.nak(nakDelayMs);
+                else
+                    m.term();
+            }
+        })();
+        return { close: async () => { await messages.close(); } };
+    }
+}
+exports.NatsStreams = NatsStreams;

package/build/lib/messaging/subjects.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Subject filters. Subjects are dot-separated tokens; in a filter `*` matches
+ * exactly one token and `>` matches one or more trailing tokens.
+ */
+import type { Subject } from '../../types/messaging';
+export declare function subjectMatches(filter: Subject, subject: Subject): boolean;
+/** Whether `subject` is a valid concrete subject: nonempty tokens, no wildcards or whitespace. */
+export declare function isSubject(subject: string): boolean;

package/build/lib/messaging/subjects.js

@@ -0,0 +1,25 @@
+"use strict";
+/**
+ * Subject filters. Subjects are dot-separated tokens; in a filter `*` matches
+ * exactly one token and `>` matches one or more trailing tokens.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.subjectMatches = subjectMatches;
+exports.isSubject = isSubject;
+function subjectMatches(filter, subject) {
+    const f = filter.split('.');
+    const s = subject.split('.');
+    for (let i = 0; i < f.length; i++) {
+        if (f[i] === '>')
+            return i === f.length - 1 && s.length > i;
+        if (i >= s.length)
+            return false;
+        if (f[i] !== '*' && f[i] !== s[i])
+            return false;
+    }
+    return f.length === s.length;
+}
+/** Whether `subject` is a valid concrete subject: nonempty tokens, no wildcards or whitespace. */
+function isSubject(subject) {
+    return !!subject && subject.split('.').every((t) => !!t && !/[\s*>]/.test(t));
+}

package/build/scripts/bambu-boat.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Make the example's first print: a small 3DBenchy (by Creative Tools), sliced
+ * for the printer with OrcaSlicer, written to examples/workflows/bambu-printer/files/boat.gcode.3mf.
+ * Run it on the host, where OrcaSlicer is installed:
+ *
+ *   npm run bambu:boat
+ *
+ * 1. Download the model once into files/.
+ * 2. Scale it (BAMBU_BOAT_SCALE, 0.5 by default: about ten minutes on a P1S).
+ * 3. Slice it with the printer, process and filament presets. Each is an
+ *    OrcaSlicer system preset name or a path to a preset JSON, which may
+ *    inherit a system preset by name.
+ *
+ * Env: BAMBU_BOAT_PRINTER, BAMBU_BOAT_PROCESS, BAMBU_BOAT_FILAMENT, BAMBU_BOAT_SCALE,
+ * ORCA_SLICER (the OrcaSlicer executable) and ORCA_PROFILES (its BBL profiles directory).
+ */
+export {};