nesoi 3.1.8 → 3.2.2

package/lib/engine/apps/distributed/distributed.app.d.ts

@@ -0,0 +1,30 @@
+import { $Space } from "../../../schema";
+import { Space } from '../../space';
+import { Daemon } from "../../daemon";
+import { DistributedAppConfig } from './distributed.app.config';
+import { App } from '../app';
+import { DistributedAppNode, DistributedAppNodeDef, DistributedNodeDaemon } from './distributed_node.app';
+/**
+ * @category App
+ * @subcategory Distributed
+ */
+export declare class DistributedApp<S extends $Space, Name extends string, Nodes extends Record<string, DistributedAppNode<any, any, any, any>> = {}> extends App<S, never, never> {
+    nodes: Nodes;
+    constructor(name: Name, space?: Space<S>);
+    boot(): DistributedApp<S, Name, Nodes>;
+    daemon(): Promise<DistributedDaemon<S, { [N in keyof Nodes as `${Name}-${N & string}`]: Nodes[N]; }>>;
+    node<N extends string, ModuleNames extends string, Services extends Record<string, any>, Def extends DistributedAppNodeDef<S, Nodes, ModuleNames, Services>>(name: N, def: Def): DistributedApp<S, Name, Nodes & {
+        [K in N]: ReturnType<Def>;
+    }>;
+}
+/**
+ * @category App
+ * @subcategory Distributed
+ */
+export declare class DistributedDaemon<S extends $Space, Nodes extends Record<string, DistributedAppNode<any, any, any, any>>> extends Daemon<S, never> {
+    nodes: {
+        [K in keyof Nodes]: DistributedNodeDaemon<S, Nodes[K] extends DistributedAppNode<any, any, infer X, any> ? X : never>;
+    };
+    constructor(name: string, config: DistributedAppConfig<any, any, any, any>);
+    boot(nodes: Nodes): Promise<void>;
+}

package/lib/engine/apps/distributed/distributed.app.js

@@ -0,0 +1,52 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DistributedDaemon = exports.DistributedApp = void 0;
+const daemon_1 = require("../../daemon");
+const app_1 = require("../app");
+const distributed_node_app_1 = require("./distributed_node.app");
+/**
+ * @category App
+ * @subcategory Distributed
+ */
+class DistributedApp extends app_1.App {
+    constructor(name, space) {
+        super(name, { builders: [] });
+        this.nodes = {};
+        this.builders = undefined;
+        this.space = space;
+    }
+    // Override App abstract methods
+    boot() {
+        return this;
+    }
+    async daemon() {
+        const config = this._config;
+        const daemon = new DistributedDaemon(this.name, config);
+        await daemon.boot(this.nodes);
+        return daemon;
+    }
+    // Type Builder Overrides
+    node(name, def) {
+        let node = new distributed_node_app_1.DistributedAppNode(`${this.name}-${name}`, this.space);
+        node = def(node);
+        this.nodes[node.name] = node;
+        return this;
+    }
+}
+exports.DistributedApp = DistributedApp;
+/**
+ * @category App
+ * @subcategory Distributed
+ */
+class DistributedDaemon extends daemon_1.Daemon {
+    constructor(name, config) {
+        super(name, {}, {}, config);
+        this.nodes = {};
+    }
+    async boot(nodes) {
+        for (const name in nodes) {
+            this.nodes[name] = await nodes[name].daemon();
+        }
+    }
+}
+exports.DistributedDaemon = DistributedDaemon;

package/lib/engine/apps/distributed/distributed_node.app.d.ts

@@ -0,0 +1,67 @@
+import { $Module, $Space, ModuleName } from "../../../schema";
+import { IService } from '../service';
+import { DistributedAppConfig, DistributedAppConfigBuilder } from './distributed.app.config';
+import { App } from '../app';
+import { MonolythDaemon } from '../monolyth/monolyth.app';
+import { AnyTrxEngine } from "../../transaction/trx_engine";
+import { AnyModule, Module } from "../../module";
+import { Space } from "../../space";
+export declare class DistributedAppNode<S extends $Space, Nodes extends Record<string, DistributedAppNode<any, any, any, any>>, ModuleNames extends string, Services extends Record<string, any>> extends App<S, ModuleNames, Services> {
+    _host: {
+        host: string;
+        port: number;
+    };
+    protected _daemon?: DistributedNodeDaemon<S, ModuleNames>;
+    protected _modules: Record<string, AnyModule>;
+    protected packageJson?: Record<string, any>;
+    protected bootPromise?: Promise<void>;
+    constructor(name: string, space?: Space<S>);
+    /**
+     * Treeshake and build modules declared for this application.
+     */
+    protected build(): Promise<void>;
+    boot(): DistributedAppNode<S, Nodes, ModuleNames, Services>;
+    /**
+     * Build the application, start services and trx engines.
+     * Returns references to start a daemon.
+     */
+    protected make(): Promise<{
+        modules: Record<string, AnyModule>;
+        services: Record<string, any>;
+        trxEngines: Record<ModuleNames, AnyTrxEngine>;
+    }>;
+    /**
+     * This method injects elements flagged as externals by referencing them
+     * from the other module directly, given this is a single-threaded App.
+     *
+     * TODO: allow overriding this behavior with adapters
+     */
+    protected linkExternals(modules: Record<string, Module<S, $Module>>): void;
+    /**
+     * This method injects values from environment variables into each module's
+     * app constants.
+     */
+    protected linkAppValues(modules: Record<string, Module<S, $Module>>): void;
+    daemon(): Promise<DistributedNodeDaemon<S, ModuleNames>>;
+    host(config: {
+        host: string;
+        port: number;
+    }): this;
+    modules<M extends ModuleName<S>>(modules: M[]): DistributedAppNode<S, Nodes, M & ModuleNames, Services>;
+    service<T extends IService>($: T): DistributedAppNode<S, Nodes, ModuleNames, Services & { [K in T["name"]]: T; }>;
+    package(_package: Record<string, any>): this;
+    get config(): DistributedAppConfigBuilder<S, Nodes, ModuleNames, Services>;
+}
+export type DistributedAppNodeDef<S extends $Space, Nodes extends Record<string, DistributedAppNode<any, any, any, any>>, ModuleNames extends string, Services extends Record<string, any>> = (builder: DistributedAppNode<S, Nodes, keyof S['modules'] & string, {}>) => DistributedAppNode<S, Nodes, ModuleNames, Services>;
+/**
+ * @category App
+ * @subcategory Distributed
+ */
+export declare class DistributedNodeDaemon<S extends $Space, Modules extends ModuleName<S>> extends MonolythDaemon<S, Modules> {
+    private inc;
+    constructor(name: string, host: {
+        host: string;
+        port: number;
+    }, trxEngines: Record<Modules, AnyTrxEngine>, services: Record<string, IService>, config?: DistributedAppConfig<any, any, any, any> | undefined);
+    boot(): Promise<void>;
+}

package/lib/engine/apps/distributed/distributed_node.app.js

@@ -0,0 +1,191 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DistributedNodeDaemon = exports.DistributedAppNode = void 0;
+const distributed_app_config_1 = require("./distributed.app.config");
+const app_1 = require("../app");
+const monolyth_app_1 = require("../monolyth/monolyth.app");
+const inc_server_1 = require("./inc/inc.server");
+const trx_engine_1 = require("../../transaction/trx_engine");
+const tree_1 = require("../../tree");
+const log_1 = require("../../util/log");
+const promise_1 = __importDefault(require("../../util/promise"));
+const distributed_job_1 = require("./elements/distributed_job");
+class DistributedAppNode extends app_1.App {
+    constructor(name, space) {
+        super(name, { builders: [] });
+        this._modules = {};
+        this.space = space;
+    }
+    /**
+     * Treeshake and build modules declared for this application.
+     */
+    async build() {
+        log_1.Log.info('app', this.name, 'Booting');
+        this._modules = await this.makeModules();
+        log_1.Log.debug('app', this.name, 'Building');
+        const tree = new tree_1.ModuleTree(this._modules, {
+            exclude: ['*.test.ts']
+        });
+        await tree.resolve();
+        await tree.traverse('Building', async (node) => {
+            // Inline nodes are built by their root builder
+            if (node.isInline) {
+                return;
+            }
+            const module = this._modules[node.module];
+            await module.buildNode(node, tree);
+        });
+    }
+    boot() {
+        if (!this.bootPromise) {
+            this.bootPromise = this.build();
+        }
+        return this;
+    }
+    /**
+     * Build the application, start services and trx engines.
+     * Returns references to start a daemon.
+     */
+    async make() {
+        if (this.space || this.builders) {
+            await this.boot().bootPromise;
+        }
+        const modules = this._modules;
+        for (const key in this._injectedModules) {
+            const mod = this._injectedModules[key];
+            modules[mod.name] = mod;
+        }
+        const services = {};
+        for (const key in this._services) {
+            const service = this._services[key];
+            await promise_1.default.solve(service.up({
+                modules
+            }));
+            services[key] = service;
+        }
+        log_1.Log.debug('app', this.name, 'Starting transaction engines');
+        const trxEngines = {};
+        for (const m in modules) {
+            const module = modules[m];
+            module.start(this, services);
+            const trxConfig = this._config.modules?.[m]?.trx;
+            const authn = {};
+            for (const a in this._config?.authn || {}) {
+                const prov = this._config.authn?.[a]?.();
+                if (prov) {
+                    authn[a] = prov;
+                }
+            }
+            trxEngines[m] = new trx_engine_1.TrxEngine(`app:${this.name}`, module, authn, trxConfig, services);
+        }
+        log_1.Log.debug('app', this.name, 'Linking externals');
+        this.linkExternals(modules);
+        log_1.Log.debug('app', this.name, 'Linking app values');
+        this.linkAppValues(modules);
+        return {
+            modules,
+            services,
+            trxEngines
+        };
+    }
+    /**
+     * This method injects elements flagged as externals by referencing them
+     * from the other module directly, given this is a single-threaded App.
+     *
+     * TODO: allow overriding this behavior with adapters
+     */
+    linkExternals(modules) {
+        Object.values(modules).forEach(module => {
+            const jobs = {};
+            for (const name in module.schema.externals.jobs) {
+                const dep = module.schema.externals.jobs[name];
+                const config = this._config;
+                const node = config?.modules?.[dep.module]?.jobs?.[dep.name];
+                if (!node) {
+                    throw new Error(`External job '${dep.tag}' not configured on module '${this.name}'`);
+                }
+                jobs[name] = new distributed_job_1.DistributedJob(module, node);
+            }
+            module.injectRunners({
+                // buckets: Object.values(module.schema.externals.buckets),
+                // messages: Object.values(module.schema.externals.messages),
+                jobs
+                // machines: Object.values(module.schema.externals.machines),
+            });
+            // const buckets = module.schema.externals.buckets;
+            // Object.values(buckets).forEach(bucket => {
+            //     module.nql.linkExternal(modules[bucket.module].buckets[bucket.name]);
+            // })
+        });
+    }
+    /**
+     * This method injects values from environment variables into each module's
+     * app constants.
+     */
+    linkAppValues(modules) {
+        Object.values(modules).forEach(module => {
+            const values = module.schema.constants.values;
+            Object.values(values).forEach(value => {
+                if (value.scope !== 'app')
+                    return;
+                value.value = process.env[value.key];
+            });
+        });
+    }
+    // Override App abstract methods
+    async daemon() {
+        if (this._daemon) {
+            return this._daemon;
+        }
+        const app = await this.make();
+        log_1.Log.debug('app', this.name, 'Spawning daemon');
+        this._daemon = new DistributedNodeDaemon(this.name, this._host, app.trxEngines, app.services, this._config);
+        await this._daemon.boot();
+        // Link daemon to modules
+        for (const m in app.modules) {
+            const module = app.modules[m];
+            module.daemon = this._daemon;
+        }
+        return this._daemon;
+    }
+    host(config) {
+        this._host = config;
+        return this;
+    }
+    modules(modules) {
+        super.modules(modules);
+        return this;
+    }
+    service($) {
+        super.service($);
+        return this;
+    }
+    package(_package) {
+        this.packageJson = _package;
+        return this;
+    }
+    get config() {
+        return new distributed_app_config_1.DistributedAppConfigBuilder(this);
+    }
+}
+exports.DistributedAppNode = DistributedAppNode;
+/**
+ * @category App
+ * @subcategory Distributed
+ */
+class DistributedNodeDaemon extends monolyth_app_1.MonolythDaemon {
+    constructor(name, host, trxEngines, services, config) {
+        super(name, trxEngines, services, config);
+        this.inc = new inc_server_1.INCServer({
+            name,
+            ...host
+        });
+    }
+    async boot() {
+        await this.inc.start();
+    }
+}
+exports.DistributedNodeDaemon = DistributedNodeDaemon;

package/lib/engine/apps/distributed/elements/distributed_job.d.ts

@@ -0,0 +1,7 @@
+import { $Job, $Module, $Space } from "../../../../elements";
+import { Job } from "../../../../elements/blocks/job/job";
+import { Module } from "../../../module";
+export declare class DistributedJob<S extends $Space, M extends $Module, $ extends $Job> extends Job<S, M, $> {
+    node: string;
+    constructor(module: Module<S, M>, node: string);
+}

package/lib/engine/apps/distributed/elements/distributed_job.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DistributedJob = void 0;
+const job_1 = require("../../../../elements/blocks/job/job");
+class DistributedJob extends job_1.Job {
+    constructor(module, node) {
+        super(module, {});
+        this.node = node;
+    }
+}
+exports.DistributedJob = DistributedJob;

package/lib/engine/apps/distributed/inc/inc.client.d.ts

@@ -0,0 +1,17 @@
+export type INCClientConfig = {
+    name: string;
+    server: {
+        host: string;
+        port: number;
+    };
+};
+export declare class INCClient {
+    config: INCClientConfig;
+    private client;
+    constructor(config: INCClientConfig);
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    send(data: string | Buffer): Promise<void>;
+    private onError;
+    private onData;
+}

package/lib/engine/apps/distributed/inc/inc.client.js

@@ -0,0 +1,105 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.INCClient = void 0;
+const net = __importStar(require("net"));
+const log_1 = require("../../../util/log");
+class INCClient {
+    constructor(config) {
+        this.config = config;
+        this.client = new net.Socket();
+        this.client.setEncoding('utf8');
+        this.client.on('data', this.onData.bind(this));
+    }
+    start() {
+        log_1.Log.info('inc.client', this.config.name, 'Starting...');
+        return new Promise((resolve, reject) => {
+            this.client.connect({
+                host: this.config.server.host,
+                port: this.config.server.port
+            });
+            const errorFn = (error) => {
+                log_1.Log.error('inc.server', this.config.name, error.toString(), error);
+                reject(error);
+            };
+            this.client.on('error', errorFn);
+            this.client.on('connect', () => {
+                const address = this.client.address();
+                log_1.Log.debug('inc.client', this.config.name, 'Connected', {
+                    server: `${this.client.remoteAddress}:${this.client.remotePort}`,
+                    client: `${address.address}:${address.port}`
+                });
+                this.client.off('error', errorFn);
+                this.client.on('error', this.onError.bind(this));
+                resolve();
+            });
+        });
+    }
+    stop() {
+        log_1.Log.info('inc.client', this.config.name, 'Stop');
+        return new Promise((resolve) => {
+            this.client.end('Bye bye server', () => {
+                resolve();
+            });
+        });
+    }
+    send(data) {
+        log_1.Log.debug('inc.client', this.config.name, 'Sending data', {
+            data
+        });
+        return new Promise((resolve, reject) => {
+            this.client.write(data, error => {
+                if (error) {
+                    reject(error);
+                }
+                else {
+                    resolve();
+                }
+            });
+        });
+    }
+    // Callbacks
+    onError(error) {
+        log_1.Log.error('inc.client', this.config.name, error.toString(), {
+            error
+        });
+    }
+    onData(data) {
+        log_1.Log.debug('inc.client', this.config.name, 'Data received', {
+            data
+        });
+    }
+}
+exports.INCClient = INCClient;

package/lib/engine/apps/distributed/inc/inc.server.d.ts

@@ -0,0 +1,19 @@
+import * as net from 'net';
+export type INCServerConfig = {
+    name: string;
+    port: number;
+    maxConnections?: number;
+};
+export declare class INCServer {
+    config: INCServerConfig;
+    private server;
+    clients: Record<string, net.Socket>;
+    constructor(config: INCServerConfig);
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    send(client: string, data: string | Buffer): Promise<void>;
+    private getClientCount;
+    private onError;
+    private onClose;
+    private onConnection;
+}

package/lib/engine/apps/distributed/inc/inc.server.js

@@ -0,0 +1,174 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.INCServer = void 0;
+const crypto_1 = require("crypto");
+const net = __importStar(require("net"));
+const log_1 = require("../../../util/log");
+class INCServer {
+    constructor(config) {
+        this.config = config;
+        this.clients = {};
+        this.server = net.createServer();
+        this.server.on('close', this.onClose.bind(this));
+        this.server.on('connection', this.onConnection.bind(this));
+        this.server.maxConnections = this.config.maxConnections || 1;
+    }
+    start() {
+        log_1.Log.info('inc.server', this.config.name, 'Starting...');
+        return new Promise((resolve, reject) => {
+            this.server.listen(this.config.port);
+            const errorFn = (error) => {
+                log_1.Log.error('inc.server', this.config.name, error.toString(), error);
+                reject(error);
+            };
+            this.server.on('error', errorFn);
+            this.server.on('listening', () => {
+                log_1.Log.info('inc.server', this.config.name, `Listening on port ${this.config.port}`);
+                this.server.off('error', errorFn);
+                this.server.on('error', this.onError.bind(this));
+                resolve();
+            });
+            if (!this.server.listening) {
+                throw new Error('Failed to start INC server');
+            }
+        });
+    }
+    stop() {
+        log_1.Log.info('inc.server', this.config.name, 'Stop');
+        return new Promise((resolve) => {
+            this.server.close(() => {
+                resolve();
+            });
+        });
+    }
+    send(client, data) {
+        log_1.Log.debug('inc.server', this.config.name, 'Sending data', {
+            data
+        });
+        return new Promise((resolve, reject) => {
+            this.clients[client].write(data, error => {
+                if (error) {
+                    reject(error);
+                }
+                else {
+                    resolve();
+                }
+            });
+        });
+    }
+    // Utils
+    getClientCount() {
+        return new Promise((resolve, reject) => {
+            this.server.getConnections(function (error, count) {
+                if (error) {
+                    reject(error);
+                }
+                resolve(count);
+            });
+        });
+    }
+    // Callbacks
+    onError(error) {
+        log_1.Log.error('inc.server', this.config.name, error.toString(), error);
+    }
+    onClose() {
+        log_1.Log.warn('inc.server', this.config.name, 'Closed');
+    }
+    async onConnection(socket) {
+        const id = (0, crypto_1.randomUUID)();
+        socket.id = id;
+        this.clients[id] = socket;
+        const serverAddress = this.server.address();
+        log_1.Log.debug('inc.server', this.config.name, 'New client connected', {
+            id,
+            server: `${serverAddress.address}:${serverAddress.port}`,
+            client: `${socket.remoteAddress}:${socket.remotePort}`,
+            clientCount: await this.getClientCount()
+        });
+        socket.setEncoding('utf8');
+        socket.setTimeout(1000 * 60 * 15); // 15 mins
+        socket.on('data', (data) => {
+            log_1.Log.debug('inc.server', this.config.name, 'Data received', {
+                client: socket.id,
+                data
+            });
+        });
+        socket.on('drain', function () {
+            socket.resume();
+        });
+        socket.on('error', (error) => {
+            delete this.clients[id];
+            log_1.Log.error('inc.server', this.config.name, 'Client error', {
+                client: socket.id,
+                error
+            });
+        });
+        socket.on('timeout', () => {
+            delete this.clients[id];
+            log_1.Log.warn('inc.server', this.config.name, 'Client timed-out', {
+                client: socket.id
+            });
+            socket.end('{"$":"__inc.timed_out"}');
+        });
+        socket.on('end', (data) => {
+            delete this.clients[id];
+            log_1.Log.debug('inc.server', this.config.name, 'Client disconnected', {
+                client: socket.id,
+                bytes_read: socket.bytesRead,
+                bytes_written: socket.bytesWritten,
+                data
+            });
+        });
+        socket.on('close', (error) => {
+            delete this.clients[id];
+            if (error) {
+                log_1.Log.error('inc.server', this.config.name, 'Client disconnected with error', {
+                    client: socket.id,
+                    bytes_read: socket.bytesRead,
+                    bytes_written: socket.bytesWritten,
+                    error
+                });
+                return;
+            }
+            log_1.Log.debug('inc.server', this.config.name, 'Client disconnected', {
+                client: socket.id,
+                bytes_read: socket.bytesRead,
+                bytes_written: socket.bytesWritten
+            });
+        });
+    }
+}
+exports.INCServer = INCServer;

package/lib/engine/apps/distributed/inc/sandbox.d.ts

@@ -0,0 +1 @@
+export {};