@hotmeshio/hotmesh 0.3.5 → 0.3.6

package/build/index.d.ts

@@ -1,7 +1,8 @@
-import { MeshCall } from './services/meshcall';
-import { MeshData } from './services/meshdata';
-import { MeshFlow } from './services/meshflow';
 import { HotMesh } from './services/hotmesh';
 import { HotMeshConfig } from './types/hotmesh';
-export { HotMesh, MeshCall, MeshData, MeshFlow, HotMeshConfig };
+import { MeshCall } from './services/meshcall';
+import { MeshFlow } from './services/meshflow';
+import { MeshData } from './services/meshdata';
+import { MeshOS } from './services/meshos';
+export { HotMesh, HotMeshConfig, MeshCall, MeshData, MeshFlow, MeshOS };
 export * as Types from './types';

package/build/index.js

@@ -23,13 +23,15 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Types = exports.MeshFlow = exports.MeshData = exports.MeshCall = exports.HotMesh = void 0;
+exports.Types = exports.MeshOS = exports.MeshFlow = exports.MeshData = exports.MeshCall = exports.HotMesh = void 0;
+const hotmesh_1 = require("./services/hotmesh");
+Object.defineProperty(exports, "HotMesh", { enumerable: true, get: function () { return hotmesh_1.HotMesh; } });
 const meshcall_1 = require("./services/meshcall");
 Object.defineProperty(exports, "MeshCall", { enumerable: true, get: function () { return meshcall_1.MeshCall; } });
-const meshdata_1 = require("./services/meshdata");
-Object.defineProperty(exports, "MeshData", { enumerable: true, get: function () { return meshdata_1.MeshData; } });
 const meshflow_1 = require("./services/meshflow");
 Object.defineProperty(exports, "MeshFlow", { enumerable: true, get: function () { return meshflow_1.MeshFlow; } });
-const hotmesh_1 = require("./services/hotmesh");
-Object.defineProperty(exports, "HotMesh", { enumerable: true, get: function () { return hotmesh_1.HotMesh; } });
+const meshdata_1 = require("./services/meshdata");
+Object.defineProperty(exports, "MeshData", { enumerable: true, get: function () { return meshdata_1.MeshData; } });
+const meshos_1 = require("./services/meshos");
+Object.defineProperty(exports, "MeshOS", { enumerable: true, get: function () { return meshos_1.MeshOS; } });
 exports.Types = __importStar(require("./types"));

package/build/modules/utils.d.ts

@@ -37,3 +37,4 @@ export declare function getValueByPath(obj: {
 export declare function restoreHierarchy(obj: StringAnyType): StringAnyType;
 export declare function isValidCron(cronExpression: string): boolean;
 export declare const s: (input: string) => number;
+export declare const arrayToHash: (response: [number, ...Array<string | string[]>]) => Record<string, string>[];

package/build/modules/utils.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.s = exports.isValidCron = exports.restoreHierarchy = exports.getValueByPath = exports.getIndexedHash = exports.getSymVal = exports.getSymKey = exports.formatISODate = exports.getTimeSeries = exports.getSubscriptionTopic = exports.findSubscriptionForTrigger = exports.findTopKey = exports.matchesStatus = exports.matchesStatusCode = exports.identifyRedisTypeFromClass = exports.polyfill = exports.identifyRedisType = exports.XSleepFor = exports.sleepImmediate = exports.sleepFor = exports.guid = exports.deterministicRandom = exports.deepCopy = exports.getSystemHealth = exports.hashOptions = void 0;
+exports.arrayToHash = exports.s = exports.isValidCron = exports.restoreHierarchy = exports.getValueByPath = exports.getIndexedHash = exports.getSymVal = exports.getSymKey = exports.formatISODate = exports.getTimeSeries = exports.getSubscriptionTopic = exports.findSubscriptionForTrigger = exports.findTopKey = exports.matchesStatus = exports.matchesStatusCode = exports.identifyRedisTypeFromClass = exports.polyfill = exports.identifyRedisType = exports.XSleepFor = exports.sleepImmediate = exports.sleepFor = exports.guid = exports.deterministicRandom = exports.deepCopy = exports.getSystemHealth = exports.hashOptions = void 0;
 const os_1 = __importDefault(require("os"));
 const crypto_1 = require("crypto");
 const nanoid_1 = require("nanoid");
@@ -240,3 +240,28 @@ const s = (input) => {
     return (0, ms_1.default)(input) / 1000;
 };
 exports.s = s;
+const arrayToHash = (response) => {
+    const results = [];
+    let key;
+    for (let i = 1; i < response.length; i++) {
+        const row = response[i];
+        const result = {};
+        if (Array.isArray(row)) {
+            for (let j = 0; j < row.length; j += 2) {
+                const key = row[j];
+                const value = row[j + 1];
+                result[key] = value;
+            }
+            if (key) {
+                result.$ = key;
+            }
+            results.push(result);
+            key = undefined;
+        }
+        else {
+            key = row;
+        }
+    }
+    return results;
+};
+exports.arrayToHash = arrayToHash;

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.3.5",
+    "version": "0.3.6",
     "description": "Unbreakable Workflows",
     "main": "./build/index.js",
     "types": "./build/index.d.ts",
@@ -28,7 +28,7 @@
         "test:cycle": "NODE_ENV=test jest ./tests/functional/cycle/index.test.ts --detectOpenHandles --forceExit --verbose",
         "test:meshflow": "NODE_ENV=test jest ./tests/meshflow/*/index.test.ts --detectOpenHandles --forceExit --verbose",
         "test:meshflow:basic": "NODE_ENV=test jest ./tests/meshflow/basic/index.test.ts --detectOpenHandles --forceExit --verbose",
-        "test:meshflow:collision": "NODE_ENV=test jest ./tests/meshflow/collision/index.test.ts --detectOpenHandles --forceExit --verbose",
+        "test:meshflow:collision": "NODE_ENV=test HMSH_LOGLEVEL=debug jest ./tests/meshflow/collision/index.test.ts --detectOpenHandles --forceExit --verbose",
         "test:meshflow:fatal": "NODE_ENV=test jest ./tests/meshflow/fatal/index.test.ts --detectOpenHandles --forceExit --verbose",
         "test:meshflow:goodbye": "NODE_ENV=test jest ./tests/meshflow/goodbye/index.test.ts --detectOpenHandles --forceExit --verbose",
         "test:meshflow:hello": "HMSH_IS_CLUSTER=true NODE_ENV=test jest ./tests/meshflow/helloworld/index.test.ts --detectOpenHandles --forceExit --verbose",
@@ -64,6 +64,7 @@
         "test:sub:redis": "NODE_ENV=test jest ./tests/functional/sub/clients/redis.test.ts --detectOpenHandles --forceExit --verbose",
         "test:trigger": "NODE_ENV=test jest ./tests/unit/services/activities/trigger.test.ts --detectOpenHandles --forceExit --verbose",
         "test:meshdata": "NODE_ENV=test HMSH_IS_CLUSTER=true jest ./tests/meshdata/index.test.ts --forceExit --verbose --detectOpenHandles",
+        "test:meshos": "NODE_ENV=test HMSH_IS_CLUSTER=true jest ./tests/meshos/index.test.ts --forceExit --verbose --detectOpenHandles",
         "test:meshcall": "NODE_ENV=test HMSH_LOGLEVEL=debug jest ./tests/meshcall/index.test.ts --forceExit --verbose --detectOpenHandles",
         "test:unit": "NODE_ENV=test jest ./tests/unit/*/*/index.test.ts --detectOpenHandles --forceExit --verbose"
     },

package/build/services/activities/trigger.d.ts

@@ -25,7 +25,6 @@ declare class Trigger extends Activity {
     resolveJobId(context: Partial<JobState>): string;
     resolveJobKey(context: Partial<JobState>): string;
     setStateNX(status?: number): Promise<void>;
-    registerJobDependency(multi?: RedisMulti): Promise<void>;
     setStats(multi?: RedisMulti): Promise<void>;
 }
 export { Trigger };

package/build/services/activities/trigger.js

@@ -37,9 +37,6 @@ class Trigger extends activity_1.Activity {
             if (options?.pending) {
                 await this.setExpired(options?.pending, multi);
             }
-            else {
-                await this.registerJobDependency(multi);
-            }
             await collator_1.CollatorService.notarizeInception(this, this.context.metadata.guid, multi);
             await multi.exec();
             this.execAdjacentParent();
@@ -228,35 +225,6 @@ class Trigger extends activity_1.Activity {
             throw new errors_1.DuplicateJobError(jobId);
         }
     }
-    async registerJobDependency(multi) {
-        const depKey = this.config.stats?.parent ?? this.context.metadata.pj;
-        let resolvedDepKey = depKey ? pipe_1.Pipe.resolve(depKey, this.context) : '';
-        const adjKey = this.config.stats?.adjacent;
-        const resolvedAdjKey = depKey ? pipe_1.Pipe.resolve(adjKey, this.context) : '';
-        if (!resolvedDepKey) {
-            resolvedDepKey = this.context.metadata.pj;
-        }
-        if (resolvedDepKey) {
-            const isParentOrigin = resolvedDepKey === this.context.metadata.pj ||
-                resolvedDepKey === resolvedAdjKey;
-            let type;
-            if (isParentOrigin) {
-                if (this.context.metadata.px) {
-                    type = 'child';
-                }
-                else {
-                    type = 'expire-child';
-                }
-            }
-            else {
-                type = 'expire';
-            }
-            await this.store.registerJobDependency(type, resolvedDepKey, this.context.metadata.tpc, this.context.metadata.jid, this.context.metadata.gid, this.context.metadata.pd, multi);
-        }
-        if (resolvedAdjKey && resolvedAdjKey !== resolvedDepKey) {
-            await this.store.registerJobDependency('child', resolvedAdjKey, this.context.metadata.tpc, this.context.metadata.jid, this.context.metadata.gid, this.context.metadata.pd, multi);
-        }
-    }
     async setStats(multi) {
         const md = this.context.metadata;
         if (md.key && this.config.stats?.measures) {

package/build/services/engine/index.js

@@ -13,14 +13,11 @@ const exporter_1 = require("../exporter");
 const reporter_1 = require("../reporter");
 const router_1 = require("../router");
 const serializer_1 = require("../serializer");
-const redis_1 = require("../store/clients/redis");
-const ioredis_1 = require("../store/clients/ioredis");
-const redis_2 = require("../stream/clients/redis");
-const ioredis_2 = require("../stream/clients/ioredis");
-const ioredis_3 = require("../sub/clients/ioredis");
-const redis_3 = require("../sub/clients/redis");
 const task_1 = require("../task");
 const stream_1 = require("../../types/stream");
+const factory_1 = require("../stream/factory");
+const factory_2 = require("../sub/factory");
+const factory_3 = require("../store/factory");
 class EngineService {
     constructor() {
         this.cacheMode = 'cache';
@@ -56,31 +53,13 @@ class EngineService {
         }
     }
     async initStoreChannel(store) {
-        if ((0, utils_1.identifyRedisType)(store) === 'redis') {
-            this.store = new redis_1.RedisStoreService(store);
-        }
-        else {
-            this.store = new ioredis_1.IORedisStoreService(store);
-        }
-        await this.store.init(this.namespace, this.appId, this.logger);
+        this.store = await factory_3.StoreServiceFactory.init(store, this.namespace, this.appId, this.logger);
     }
     async initSubChannel(sub) {
-        if ((0, utils_1.identifyRedisType)(sub) === 'redis') {
-            this.subscribe = new redis_3.RedisSubService(sub);
-        }
-        else {
-            this.subscribe = new ioredis_3.IORedisSubService(sub);
-        }
-        await this.subscribe.init(this.namespace, this.appId, this.guid, this.logger);
+        this.subscribe = await factory_2.SubServiceFactory.init(sub, this.namespace, this.appId, this.guid, this.logger);
     }
     async initStreamChannel(stream) {
-        if ((0, utils_1.identifyRedisType)(stream) === 'redis') {
-            this.stream = new redis_2.RedisStreamService(stream);
-        }
-        else {
-            this.stream = new ioredis_2.IORedisStreamService(stream);
-        }
-        await this.stream.init(this.namespace, this.appId, this.logger);
+        this.stream = await factory_1.StreamServiceFactory.init(stream, this.namespace, this.appId, this.logger);
     }
     async initRouter(config) {
         const throttle = await this.store.getThrottleRate(':');

package/build/services/meshflow/search.js

@@ -104,7 +104,9 @@ class Search {
             return value;
         }
         catch (error) {
-            this.hotMeshClient.logger.error('meshflow-search-get-error', { ...error });
+            this.hotMeshClient.logger.error('meshflow-search-get-error', {
+                ...error,
+            });
             return '';
         }
     }

package/build/services/meshflow/worker.js

@@ -237,7 +237,7 @@ class WorkerService {
                     isProcessing = true;
                     const workflowInput = data.data;
                     const execIndex = counter.counter - interruptionRegistry.length + 1;
-                    const { workflowId, workflowTopic, workflowDimension, originJobId, expire } = workflowInput;
+                    const { workflowId, workflowTopic, workflowDimension, originJobId, expire, } = workflowInput;
                     const collatorFlowId = `${(0, utils_1.guid)()}$C`;
                     return {
                         status: stream_1.StreamStatus.SUCCESS,

package/build/services/meshflow/workflow.js

@@ -116,7 +116,7 @@ class WorkflowService {
                 }
                 return result.$error;
             }
-            else {
+            else if (!result?.$error) {
                 return result.data;
             }
         }
@@ -135,7 +135,7 @@ class WorkflowService {
             childJobId = options.workflowId;
         }
         else if (options.entity) {
-            childJobId = `${options.entity}-${workflowId.substring(0, 7)}-${(0, utils_1.guid)()}-${workflowDimension}-${execIndex}`;
+            childJobId = `${options.entity}-${(0, utils_1.guid)()}-${workflowDimension}-${execIndex}`;
         }
         else {
             childJobId = `-${options.workflowName}-${(0, utils_1.guid)()}-${workflowDimension}-${execIndex}`;

package/build/services/meshos/index.d.ts

@@ -0,0 +1,81 @@
+import { MeshData } from '../meshdata/index';
+import * as Types from '../../types';
+declare abstract class MeshOS {
+    meshData: MeshData;
+    connected: boolean;
+    namespace: string;
+    namespaceType: string;
+    static databases: Record<string, Types.DB>;
+    static namespaces: Types.Namespaces;
+    static entities: Record<string, Types.Entity>;
+    static schemas: Record<string, Types.WorkflowSearchSchema>;
+    static profiles: Types.Profiles;
+    static classes: Record<string, typeof MeshOS>;
+    static logger: Types.ILogger;
+    constructor(namespace: string, namespaceType: string, config: Types.DBConfig);
+    protected abstract getEntity(): string;
+    abstract getSearchOptions(): Types.WorkflowSearchOptions;
+    protected abstract getTaskQueue(): string;
+    private initializeMeshData;
+    protected defaultTargetFn(): Promise<string>;
+    getNamespace(): string;
+    getRedisUrl: (config: Types.DBConfig) => {
+        url: string;
+    };
+    connect(): Promise<void>;
+    index(): Promise<void>;
+    static shutdown(): Promise<void>;
+    getIndexName(): string;
+    create(body: Record<string, any>): Promise<Types.StringStringType>;
+    retrieve(id: string, sparse?: boolean): Promise<Types.StringStringType>;
+    update(id: string, body: Record<string, any>): Promise<Types.StringStringType>;
+    delete(id: string): Promise<boolean>;
+    find(query?: {
+        field: string;
+        is: '=' | '[]' | '>=' | '<=';
+        value: string;
+    }[], start?: number, size?: number): Promise<{
+        count: number;
+        query: string;
+        data: Types.StringStringType[];
+    }>;
+    count(query: {
+        field: string;
+        is: '=' | '[]' | '>=' | '<=';
+        value: string;
+    }[]): Promise<number>;
+    aggregate(filter?: {
+        field: string;
+        is: '=' | '[]' | '>=' | '<=';
+        value: string;
+    }[], apply?: {
+        expression: string;
+        as: string;
+    }[], rows?: string[], columns?: string[], reduce?: {
+        operation: string;
+        as: string;
+        property?: string;
+    }[], sort?: {
+        field: string;
+        order: 'ASC' | 'DESC';
+    }[], start?: number, size?: number): Promise<{
+        count: number;
+        query: string;
+        data: Types.StringStringType[];
+    }>;
+    private buildAggregateCommand;
+    private buildFilterCommand;
+    init(search?: boolean): Promise<void>;
+    static registerDatabase(id: string, config: Types.DB): void;
+    static registerNamespace(id: string, config: Types.Namespace): void;
+    static registerEntity(id: string, config: Types.Entity): void;
+    static registerSchema(id: string, schema: Types.WorkflowSearchSchema): void;
+    static registerProfile(id: string, config: Types.Profile): void;
+    static registerClass(id: string, entityClass: typeof MeshOS): void;
+    static init(p?: Types.Profiles): Promise<void>;
+    static findEntity(database: string, namespace: string, entity: string): Types.EntityInstanceTypes | undefined;
+    static findSchemas(database: string, ns: string): Record<string, Types.WorkflowSearchSchema>;
+    static toJSON(p?: Types.Profiles): any;
+    workflow: {};
+}
+export { MeshOS };

package/build/services/meshos/index.js

@@ -0,0 +1,339 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MeshOS = void 0;
+const Redis = __importStar(require("redis"));
+const index_1 = require("../meshdata/index");
+const utils_1 = require("../../modules/utils");
+const logger_1 = require("../logger");
+class MeshOS {
+    constructor(namespace, namespaceType, config) {
+        this.connected = false;
+        this.getRedisUrl = (config) => {
+            return {
+                url: `redis${config.REDIS_USE_TLS ? 's' : ''}://${config.REDIS_USERNAME ?? ''}:${config.REDIS_PASSWORD}@${config.REDIS_HOST}:${config.REDIS_PORT}`,
+            };
+        };
+        this.workflow = {};
+        this.namespace = namespace;
+        this.namespaceType = namespaceType;
+        this.meshData = this.initializeMeshData(config);
+    }
+    initializeMeshData(dbConfig) {
+        return new index_1.MeshData(Redis, this.getRedisUrl(dbConfig), this.getSearchOptions());
+    }
+    async defaultTargetFn() {
+        return 'OK';
+    }
+    getNamespace() {
+        return this.namespace;
+    }
+    async connect() {
+        this.connected = await this.meshData.connect({
+            entity: this.getEntity(),
+            target: this.defaultTargetFn,
+            options: {
+                namespace: this.getNamespace(),
+                taskQueue: this.getTaskQueue(),
+            },
+        });
+    }
+    async index() {
+        await this.meshData.createSearchIndex(this.getEntity(), { namespace: this.getNamespace() }, this.getSearchOptions());
+    }
+    static async shutdown() {
+        await index_1.MeshData.shutdown();
+    }
+    getIndexName() {
+        return this.getSearchOptions().index;
+    }
+    async create(body) {
+        const id = body.id || (0, utils_1.guid)();
+        await this.meshData.set(this.getEntity(), id, {
+            search: { data: body },
+            namespace: this.getNamespace(),
+        });
+        return this.retrieve(id);
+    }
+    async retrieve(id, sparse = false) {
+        const opts = this.getSearchOptions();
+        const fields = sparse ? ['id'] : Object.keys(opts?.schema || {});
+        const result = await this.meshData.get(this.getEntity(), id, {
+            fields,
+            namespace: this.getNamespace(),
+        });
+        if (!result?.id)
+            throw new Error(`${this.getEntity()} not found`);
+        return result;
+    }
+    async update(id, body) {
+        await this.retrieve(id);
+        await this.meshData.set(this.getEntity(), id, {
+            search: { data: body },
+            namespace: this.getNamespace(),
+        });
+        return this.retrieve(id);
+    }
+    async delete(id) {
+        await this.retrieve(id);
+        await this.meshData.flush(this.getEntity(), id, this.getNamespace());
+        return true;
+    }
+    async find(query = [], start = 0, size = 100) {
+        const opts = this.getSearchOptions();
+        return this.meshData.findWhere(this.getEntity(), {
+            query,
+            return: Object.keys(opts?.schema || {}),
+            limit: { start, size },
+            options: { namespace: this.getNamespace() },
+        });
+    }
+    async count(query) {
+        return this.meshData.findWhere(this.getEntity(), {
+            query,
+            count: true,
+            options: { namespace: this.getNamespace() },
+        });
+    }
+    async aggregate(filter = [], apply = [], rows = [], columns = [], reduce = [], sort = [], start = 0, size = 100) {
+        const command = this.buildAggregateCommand(filter, apply, rows, columns, reduce, sort);
+        try {
+            const results = await this.meshData.find(this.getEntity(), {
+                index: this.getIndexName(),
+                namespace: this.getNamespace(),
+                taskQueue: this.getTaskQueue(),
+                search: this.getSearchOptions(),
+            }, ...command);
+            return {
+                count: results[0],
+                query: command.join(' '),
+                data: (0, utils_1.arrayToHash)(results),
+            };
+        }
+        catch (e) {
+            throw e;
+        }
+    }
+    buildAggregateCommand(filter, apply, rows, columns, reduce, sort) {
+        const command = ['FT.AGGREGATE', this.getIndexName() || 'default'];
+        const opts = this.getSearchOptions();
+        command.push(this.buildFilterCommand(filter));
+        apply.forEach((a) => command.push('APPLY', a.expression, 'AS', a.as));
+        const groupBy = rows.concat(columns);
+        if (groupBy.length > 0) {
+            command.push('GROUPBY', `${groupBy.length}`, ...groupBy.map((g) => opts?.schema?.[g] ? `@_${g}` : `@${g}`));
+        }
+        reduce.forEach((r) => {
+            const op = r.operation.toUpperCase();
+            if (op === 'COUNT') {
+                command.push('REDUCE', op, '0', 'AS', r.as ?? 'count');
+            }
+            else if ([
+                'COUNT_DISTINCT',
+                'COUNT_DISTINCTISH',
+                'SUM',
+                'AVG',
+                'MIN',
+                'MAX',
+                'STDDEV',
+                'TOLIST',
+            ].includes(op)) {
+                const property = r.property
+                    ? opts?.schema?.[r.property]
+                        ? `@_${r.property}`
+                        : `@${r.property}`
+                    : '';
+                command.push('REDUCE', op, '1', property, 'AS', r.as ?? `${r.operation}_${r.property}`);
+            }
+        });
+        if (sort.length > 0) {
+            command.push('SORTBY', `${2 * sort.length}`, ...sort.flatMap((s) => [
+                opts?.schema?.[s.field] ? `@_${s.field}` : `@${s.field}`,
+                s.order.toUpperCase() || 'DESC',
+            ]));
+        }
+        return command;
+    }
+    buildFilterCommand(filter) {
+        if (filter.length === 0)
+            return '*';
+        const opts = this.getSearchOptions();
+        return filter
+            .map((q) => {
+            const type = opts?.schema?.[q.field]?.type ?? 'TEXT';
+            switch (type) {
+                case 'TAG':
+                    return `@_${q.field}:{${q.value}}`;
+                case 'TEXT':
+                    return `@_${q.field}:${q.value}`;
+                case 'NUMERIC':
+                    return `@_${q.field}:[${q.value}]`;
+            }
+        })
+            .join(' ');
+    }
+    async init(search = true) {
+        await this.connect();
+        if (search) {
+            await this.index();
+        }
+    }
+    static registerDatabase(id, config) {
+        MeshOS.databases[id] = config;
+    }
+    static registerNamespace(id, config) {
+        MeshOS.namespaces[id] = config;
+    }
+    static registerEntity(id, config) {
+        MeshOS.entities[id] = config;
+    }
+    static registerSchema(id, schema) {
+        MeshOS.schemas[id] = schema;
+    }
+    static registerProfile(id, config) {
+        MeshOS.profiles[id] = config;
+    }
+    static registerClass(id, entityClass) {
+        MeshOS.classes[id] = entityClass;
+    }
+    static async init(p = MeshOS.profiles) {
+        for (const key in p) {
+            const profile = p[key];
+            if (profile.db.config.REDIS_HOST) {
+                this.logger.info(`meshos-initializing`, {
+                    db: profile.db.name,
+                    key,
+                });
+                profile.instances = {};
+                for (const ns in profile.namespaces) {
+                    const namespace = profile.namespaces[ns];
+                    this.logger.info(`meshos-initializing-namespace`, {
+                        namespace: ns,
+                        label: namespace.label,
+                    });
+                    let pinstances = profile.instances[ns];
+                    if (!pinstances) {
+                        pinstances = {};
+                        profile.instances[ns] = pinstances;
+                    }
+                    for (const entity of namespace.entities) {
+                        this.logger.info(`meshos-initializing-entity`, {
+                            entity: entity.name,
+                            label: entity.label,
+                        });
+                        const instance = pinstances[entity.name] = new entity.class(ns, namespace.type, profile.db.config);
+                        await instance.init(profile.db.search);
+                    }
+                }
+            }
+        }
+    }
+    static findEntity(database, namespace, entity) {
+        if (!database ||
+            !MeshOS.profiles[database] ||
+            !MeshOS.profiles[database]?.db?.config?.REDIS_HOST) {
+            const activeProfiles = Object.keys(MeshOS.profiles).filter((key) => MeshOS.profiles[key]?.db?.config?.REDIS_HOST);
+            throw new Error(`The database query parameter [${database}] was not found. Use one of: ${activeProfiles.join(', ')}`);
+        }
+        if (!namespace || !MeshOS.profiles[database]?.instances?.[namespace]) {
+            const activeNamespaces = Object.keys(MeshOS.profiles[database]?.instances ?? {});
+            throw new Error(`The namespace query parameter [${namespace}] was not found. Use one of: ${activeNamespaces.join(', ')}`);
+        }
+        const entities = MeshOS.profiles[database]?.instances?.[namespace] ?? {};
+        if (!entity || entity?.startsWith('-') || entity === '*') {
+            entity = Object.keys(entities)[0];
+        }
+        else if (entity?.endsWith('*')) {
+            entity = entity.slice(0, -1);
+        }
+        const target = MeshOS.profiles[database]?.instances?.[namespace]?.[entity];
+        if (!target) {
+            this.logger.error(`meshos-entity-not-found`, {
+                database,
+                namespace,
+                entity,
+            });
+            entity = Object.keys(entities)[0];
+            return MeshOS.profiles[database]?.instances?.[namespace]?.[entity];
+        }
+        return target;
+    }
+    static findSchemas(database, ns) {
+        if (!database ||
+            !MeshOS.profiles[database] ||
+            !MeshOS.profiles[database]?.db?.config?.REDIS_HOST) {
+            const activeProfiles = Object.keys(MeshOS.profiles).filter((key) => MeshOS.profiles[key]?.db?.config?.REDIS_HOST);
+            throw new Error(`The database query parameter [${database}] was not found. Use one of: ${activeProfiles.join(', ')}`);
+        }
+        const profile = MeshOS.profiles[database];
+        const namespacedInstance = profile.instances[ns];
+        const schemas = {};
+        for (const entityName in namespacedInstance) {
+            const entityInstance = namespacedInstance[entityName];
+            const opts = entityInstance.getSearchOptions();
+            schemas[opts.index ?? entityName] = opts.schema;
+        }
+        return schemas;
+    }
+    static toJSON(p = MeshOS.profiles) {
+        const result = {};
+        for (const key in p) {
+            const profile = p[key];
+            if (!profile.db.config.REDIS_HOST) {
+                continue;
+            }
+            else {
+                result[key] = {
+                    db: { ...profile.db, config: undefined },
+                    namespaces: {},
+                };
+            }
+            for (const ns in profile.namespaces) {
+                const namespace = profile.namespaces[ns];
+                result[key].namespaces[ns] = {
+                    name: namespace.name,
+                    label: namespace.label,
+                    entities: [],
+                };
+                for (const entity of namespace.entities) {
+                    result[key].namespaces[ns].entities.push({
+                        name: entity.name,
+                        label: entity.label,
+                        schema: entity.schema,
+                    });
+                }
+            }
+        }
+        return result;
+    }
+}
+exports.MeshOS = MeshOS;
+MeshOS.databases = {};
+MeshOS.namespaces = {};
+MeshOS.entities = {};
+MeshOS.schemas = {};
+MeshOS.profiles = {};
+MeshOS.classes = {};
+MeshOS.logger = new logger_1.LoggerService('hotmesh', 'meshos');

package/build/services/quorum/index.js

@@ -4,11 +4,9 @@ exports.QuorumService = void 0;
 const enums_1 = require("../../modules/enums");
 const utils_1 = require("../../modules/utils");
 const compiler_1 = require("../compiler");
-const ioredis_1 = require("../store/clients/ioredis");
-const redis_1 = require("../store/clients/redis");
-const ioredis_2 = require("../sub/clients/ioredis");
-const redis_2 = require("../sub/clients/redis");
 const hotmesh_1 = require("../../types/hotmesh");
+const factory_1 = require("../sub/factory");
+const factory_2 = require("../store/factory");
 class QuorumService {
     constructor() {
         this.profiles = [];
@@ -42,22 +40,10 @@ class QuorumService {
         }
     }
     async initStoreChannel(store) {
-        if ((0, utils_1.identifyRedisType)(store) === 'redis') {
-            this.store = new redis_1.RedisStoreService(store);
-        }
-        else {
-            this.store = new ioredis_1.IORedisStoreService(store);
-        }
-        await this.store.init(this.namespace, this.appId, this.logger);
+        this.store = await factory_2.StoreServiceFactory.init(store, this.namespace, this.appId, this.logger);
     }
     async initSubChannel(sub) {
-        if ((0, utils_1.identifyRedisType)(sub) === 'redis') {
-            this.subscribe = new redis_2.RedisSubService(sub);
-        }
-        else {
-            this.subscribe = new ioredis_2.IORedisSubService(sub);
-        }
-        await this.subscribe.init(this.namespace, this.appId, this.guid, this.logger);
+        this.subscribe = await factory_1.SubServiceFactory.init(sub, this.namespace, this.appId, this.guid, this.logger);
     }
     subscriptionHandler() {
         const self = this;

package/build/services/store/factory.d.ts

@@ -0,0 +1,7 @@
+import { RedisClient } from '../../types/redis';
+import { ILogger } from '../logger';
+import { StoreService } from './index';
+declare class StoreServiceFactory {
+    static init(redisClient: RedisClient, namespace: string, appId: string, logger: ILogger): Promise<StoreService<any, any>>;
+}
+export { StoreServiceFactory };

package/build/services/store/factory.js

@@ -0,0 +1,20 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StoreServiceFactory = void 0;
+const utils_1 = require("../../modules/utils");
+const ioredis_1 = require("./clients/ioredis");
+const redis_1 = require("./clients/redis");
+class StoreServiceFactory {
+    static async init(redisClient, namespace, appId, logger) {
+        let service;
+        if ((0, utils_1.identifyRedisType)(redisClient) === 'redis') {
+            service = new redis_1.RedisStoreService(redisClient);
+        }
+        else {
+            service = new ioredis_1.IORedisStoreService(redisClient);
+        }
+        await service.init(namespace, appId, logger);
+        return service;
+    }
+}
+exports.StoreServiceFactory = StoreServiceFactory;

package/build/services/store/index.d.ts

@@ -61,7 +61,6 @@ declare abstract class StoreService<T, U extends AbstractRedisClient> {
     setApp(id: string, version: string): Promise<HotMeshApp>;
     activateAppVersion(id: string, version: string): Promise<boolean>;
     registerAppVersion(appId: string, version: string): Promise<any>;
-    registerJobDependency(depType: WorkListTaskType, originJobId: string, topic: string, jobId: string, gId: string, pd?: string, multi?: U): Promise<any>;
     setStats(jobKey: string, jobId: string, dateTime: string, stats: StatsType, appVersion: AppVID, multi?: U): Promise<any>;
     hGetAllResult(result: any): any;
     getJobStats(jobKeys: string[]): Promise<JobStatsRange>;

package/build/services/store/index.js

@@ -355,8 +355,6 @@ class StoreService {
         };
         return await this.redisClient[this.commands.hset](key, payload);
     }
-    async registerJobDependency(depType, originJobId, topic, jobId, gId, pd = '', multi) {
-    }
     async setStats(jobKey, jobId, dateTime, stats, appVersion, multi) {
         const params = {
             appId: appVersion.id,

package/build/services/stream/factory.d.ts

@@ -0,0 +1,7 @@
+import { RedisClient } from '../../types/redis';
+import { ILogger } from '../logger';
+import { StreamService } from './index';
+declare class StreamServiceFactory {
+    static init(redisClient: RedisClient, namespace: string, appId: string, logger: ILogger): Promise<StreamService<any, any>>;
+}
+export { StreamServiceFactory };

package/build/services/stream/factory.js

@@ -0,0 +1,20 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StreamServiceFactory = void 0;
+const utils_1 = require("../../modules/utils");
+const ioredis_1 = require("./clients/ioredis");
+const redis_1 = require("./clients/redis");
+class StreamServiceFactory {
+    static async init(redisClient, namespace, appId, logger) {
+        let service;
+        if ((0, utils_1.identifyRedisType)(redisClient) === 'redis') {
+            service = new redis_1.RedisStreamService(redisClient);
+        }
+        else {
+            service = new ioredis_1.IORedisStreamService(redisClient);
+        }
+        await service.init(namespace, appId, logger);
+        return service;
+    }
+}
+exports.StreamServiceFactory = StreamServiceFactory;

package/build/services/sub/factory.d.ts

@@ -0,0 +1,7 @@
+import { RedisClient } from '../../types/redis';
+import { ILogger } from '../logger';
+import { SubService } from './index';
+declare class SubServiceFactory {
+    static init(redisClient: RedisClient, namespace: string, appId: string, engineId: string, logger: ILogger): Promise<SubService<any, any>>;
+}
+export { SubServiceFactory };

package/build/services/sub/factory.js

@@ -0,0 +1,20 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SubServiceFactory = void 0;
+const utils_1 = require("../../modules/utils");
+const ioredis_1 = require("./clients/ioredis");
+const redis_1 = require("./clients/redis");
+class SubServiceFactory {
+    static async init(redisClient, namespace, appId, engineId, logger) {
+        let service;
+        if ((0, utils_1.identifyRedisType)(redisClient) === 'redis') {
+            service = new redis_1.RedisSubService(redisClient);
+        }
+        else {
+            service = new ioredis_1.IORedisSubService(redisClient);
+        }
+        await service.init(namespace, appId, engineId, logger);
+        return service;
+    }
+}
+exports.SubServiceFactory = SubServiceFactory;

package/build/services/worker/index.js

@@ -5,14 +5,11 @@ const key_1 = require("../../modules/key");
 const utils_1 = require("../../modules/utils");
 const connector_1 = require("../connector");
 const router_1 = require("../router");
-const ioredis_1 = require("../store/clients/ioredis");
-const redis_1 = require("../store/clients/redis");
-const ioredis_2 = require("../stream/clients/ioredis");
-const redis_2 = require("../stream/clients/redis");
-const ioredis_3 = require("../sub/clients/ioredis");
-const redis_3 = require("../sub/clients/redis");
 const stream_1 = require("../../types/stream");
 const enums_1 = require("../../modules/enums");
+const factory_1 = require("../stream/factory");
+const factory_2 = require("../sub/factory");
+const factory_3 = require("../store/factory");
 class WorkerService {
     constructor() {
         this.reporting = false;
@@ -58,31 +55,13 @@ class WorkerService {
         }
     }
     async initStoreChannel(service, store) {
-        if ((0, utils_1.identifyRedisType)(store) === 'redis') {
-            service.store = new redis_1.RedisStoreService(store);
-        }
-        else {
-            service.store = new ioredis_1.IORedisStoreService(store);
-        }
-        await service.store.init(service.namespace, service.appId, service.logger);
+        service.store = await factory_3.StoreServiceFactory.init(store, service.namespace, service.appId, service.logger);
     }
     async initSubChannel(service, sub) {
-        if ((0, utils_1.identifyRedisType)(sub) === 'redis') {
-            service.subscribe = new redis_3.RedisSubService(sub);
-        }
-        else {
-            service.subscribe = new ioredis_3.IORedisSubService(sub);
-        }
-        await service.subscribe.init(service.namespace, service.appId, service.guid, service.logger);
+        service.subscribe = await factory_2.SubServiceFactory.init(sub, service.namespace, service.appId, service.guid, service.logger);
     }
     async initStreamChannel(service, stream) {
-        if ((0, utils_1.identifyRedisType)(stream) === 'redis') {
-            service.stream = new redis_2.RedisStreamService(stream);
-        }
-        else {
-            service.stream = new ioredis_2.IORedisStreamService(stream);
-        }
-        await service.stream.init(service.namespace, service.appId, service.logger);
+        service.stream = await factory_1.StreamServiceFactory.init(stream, service.namespace, service.appId, service.logger);
     }
     async initRouter(worker, logger) {
         const throttle = await this.store.getThrottleRate(worker.topic);

package/build/types/index.d.ts

@@ -9,6 +9,7 @@ export { ActivityAction, DependencyExport, MeshFlowJobExport, ExportCycles, Expo
 export { HookCondition, HookConditions, HookGate, HookInterface, HookRule, HookRules, HookSignal, } from './hook';
 export { ILogger, LogLevel } from './logger';
 export { ExtensionType, JobCompletionOptions, JobData, JobsData, JobInterruptOptions, JobMetadata, JobOutput, JobState, JobStatus, PartialJobState, } from './job';
+export { DB, DBConfig, Profile, Namespaces, Entity, EntityInstanceTypes, SubClassInstance, AllSubclassInstances, SubclassType, Namespace, Instance, Instances, Profiles, } from './manifest';
 export { MappingStatements } from './map';
 export { Pipe, PipeContext, PipeItem, PipeItems, PipeObject, ReduceObject, } from './pipe';
 export { HotMesh, HotMeshApp, HotMeshApps, HotMeshConfig, HotMeshEngine, RedisConfig, HotMeshGraph, HotMeshManifest, HotMeshSettings, HotMeshWorker, KeyStoreParams, KeyType, } from './hotmesh';

package/build/types/manifest.d.ts

@@ -0,0 +1,49 @@
+import { MeshOS } from '../services/meshos';
+import * as Types from './index';
+export type DBConfig = {
+    REDIS_DATABASE: number;
+    REDIS_HOST: string | undefined;
+    REDIS_PORT: number;
+    REDIS_USERNAME: string;
+    REDIS_PASSWORD: string;
+    REDIS_USE_TLS: boolean;
+};
+export type DB = {
+    name: string;
+    label: string;
+    search: boolean;
+    config: DBConfig;
+};
+export type SubClassInstance<T extends typeof MeshOS> = T extends abstract new (...args: any) => infer R ? R : never;
+export type AllSubclassInstances = SubClassInstance<(typeof MeshOS)['classes'][keyof (typeof MeshOS)['classes']]>;
+export type EntityInstanceTypes = MeshOS;
+export type SubclassType<T extends MeshOS = MeshOS> = new (...args: any[]) => T;
+export type Entity = {
+    name: string;
+    label: string;
+    schema: Types.WorkflowSearchSchema;
+    class: SubclassType;
+};
+export type Namespace = {
+    name: string;
+    type: string;
+    label: string;
+    entities: Entity[];
+};
+export type Namespaces = {
+    [key: string]: Namespace;
+};
+export type Instance = {
+    [key: string]: EntityInstanceTypes;
+};
+export type Instances = {
+    [key: string]: Instance;
+};
+export type Profile = {
+    db: DB;
+    namespaces: Namespaces;
+    instances?: Instances;
+};
+export type Profiles = {
+    [key: string]: Profile;
+};

package/build/types/manifest.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/index.ts

@@ -1,9 +1,10 @@
-import { MeshCall } from './services/meshcall';
-import { MeshData } from './services/meshdata';
-import { MeshFlow } from './services/meshflow';
 import { HotMesh } from './services/hotmesh';
 import { HotMeshConfig } from './types/hotmesh';
+import { MeshCall } from './services/meshcall';
+import { MeshFlow } from './services/meshflow';
+import { MeshData } from './services/meshdata';
+import { MeshOS } from './services/meshos';
 
-export { HotMesh, MeshCall, MeshData, MeshFlow, HotMeshConfig };
+export { HotMesh, HotMeshConfig, MeshCall, MeshData, MeshFlow, MeshOS };
 
 export * as Types from './types';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.3.5",
+  "version": "0.3.6",
   "description": "Unbreakable Workflows",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",
@@ -28,7 +28,7 @@
     "test:cycle": "NODE_ENV=test jest ./tests/functional/cycle/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:meshflow": "NODE_ENV=test jest ./tests/meshflow/*/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:meshflow:basic": "NODE_ENV=test jest ./tests/meshflow/basic/index.test.ts --detectOpenHandles --forceExit --verbose",
-    "test:meshflow:collision": "NODE_ENV=test jest ./tests/meshflow/collision/index.test.ts --detectOpenHandles --forceExit --verbose",
+    "test:meshflow:collision": "NODE_ENV=test HMSH_LOGLEVEL=debug jest ./tests/meshflow/collision/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:meshflow:fatal": "NODE_ENV=test jest ./tests/meshflow/fatal/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:meshflow:goodbye": "NODE_ENV=test jest ./tests/meshflow/goodbye/index.test.ts --detectOpenHandles --forceExit --verbose",
     "test:meshflow:hello": "HMSH_IS_CLUSTER=true NODE_ENV=test jest ./tests/meshflow/helloworld/index.test.ts --detectOpenHandles --forceExit --verbose",
@@ -64,6 +64,7 @@
     "test:sub:redis": "NODE_ENV=test jest ./tests/functional/sub/clients/redis.test.ts --detectOpenHandles --forceExit --verbose",
     "test:trigger": "NODE_ENV=test jest ./tests/unit/services/activities/trigger.test.ts --detectOpenHandles --forceExit --verbose",
     "test:meshdata": "NODE_ENV=test HMSH_IS_CLUSTER=true jest ./tests/meshdata/index.test.ts --forceExit --verbose --detectOpenHandles",
+    "test:meshos": "NODE_ENV=test HMSH_IS_CLUSTER=true jest ./tests/meshos/index.test.ts --forceExit --verbose --detectOpenHandles",
     "test:meshcall": "NODE_ENV=test HMSH_LOGLEVEL=debug jest ./tests/meshcall/index.test.ts --forceExit --verbose --detectOpenHandles",
     "test:unit": "NODE_ENV=test jest ./tests/unit/*/*/index.test.ts --detectOpenHandles --forceExit --verbose"
   },

package/typedoc.json

@@ -14,6 +14,7 @@
     "services/meshflow/worker.ts",
     "services/meshflow/workflow.ts",
     "services/meshflow/connection.ts",
+    "services/meshos/index.ts",
     "types/*.ts"
   ],
   "highlightLanguages": [

package/types/index.ts

@@ -91,6 +91,21 @@ export {
   JobStatus,
   PartialJobState,
 } from './job';
+export {
+  DB,
+  DBConfig,
+  Profile,
+  Namespaces,
+  Entity,
+  EntityInstanceTypes,
+  SubClassInstance,
+  AllSubclassInstances,
+  SubclassType,
+  Namespace,
+  Instance,
+  Instances,
+  Profiles,
+} from './manifest';
 export { MappingStatements } from './map';
 export {
   Pipe,

package/types/manifest.ts

@@ -0,0 +1,68 @@
+import { MeshOS } from '../services/meshos';
+
+import * as Types from './index';
+
+export type DBConfig = {
+  REDIS_DATABASE: number;
+  REDIS_HOST: string | undefined;
+  REDIS_PORT: number;
+  REDIS_USERNAME: string;
+  REDIS_PASSWORD: string;
+  REDIS_USE_TLS: boolean;
+};
+
+export type DB = {
+  name: string;
+  label: string;
+  search: boolean;
+  config: DBConfig;
+};
+
+export type SubClassInstance<T extends typeof MeshOS> = T extends abstract new (
+  ...args: any
+) => infer R
+  ? R
+  : never;
+export type AllSubclassInstances = SubClassInstance<
+  (typeof MeshOS)['classes'][keyof (typeof MeshOS)['classes']]
+>;
+
+//export type EntityClassTypes = typeof SubclassType;
+export type EntityInstanceTypes = MeshOS;
+
+export type SubclassType<T extends MeshOS = MeshOS> = new (...args: any[]) => T;
+export type Entity = {
+  name: string;
+  label: string;
+  schema: Types.WorkflowSearchSchema;
+  class: SubclassType;
+};
+
+export type Namespace = {
+  name: string;
+  type: string;
+  label: string;
+  entities: Entity[];
+};
+
+export type Namespaces = {
+  [key: string]: Namespace;
+};
+
+export type Instance = {
+  [key /*entity name*/ : string]: EntityInstanceTypes;
+};
+
+export type Instances = {
+  [key /*namespace abbreviation*/ : string]: Instance;
+};
+
+export type Profile = {
+  db: DB;
+  namespaces: Namespaces;
+  instances?: Instances;
+};
+
+export type Profiles = {
+  [key: string]: Profile;
+};

package/types/meshflow.ts

@@ -511,7 +511,7 @@ type FindJobsOptions = {
   /**
    * application namespace
    * @default meshflow
-  */
+   */
   namespace?: string;
 
   /** The suggested response limit. Reduce batch size to reduce the likelihood of large overages. */