@hotmeshio/hotmesh 0.4.0 → 0.4.1

package/build/services/worker/index.js

@@ -0,0 +1,197 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WorkerService = void 0;
+const key_1 = require("../../modules/key");
+const utils_1 = require("../../modules/utils");
+const factory_1 = require("../connector/factory");
+const router_1 = require("../router");
+const stream_1 = require("../../types/stream");
+const enums_1 = require("../../modules/enums");
+const factory_2 = require("../stream/factory");
+const factory_3 = require("../sub/factory");
+const factory_4 = require("../store/factory");
+class WorkerService {
+    /**
+     * @private
+     */
+    constructor() {
+        this.reporting = false;
+    }
+    /**
+     * @private
+     */
+    static async init(namespace, appId, guid, config, logger) {
+        const services = [];
+        if (Array.isArray(config.workers)) {
+            for (const worker of config.workers) {
+                await factory_1.ConnectorService.initClients(worker);
+                const service = new WorkerService();
+                service.verifyWorkerFields(worker);
+                service.namespace = namespace;
+                service.appId = appId;
+                service.guid = guid;
+                service.callback = worker.callback;
+                service.topic = worker.topic;
+                service.config = config;
+                service.logger = logger;
+                await service.initStoreChannel(service, worker.store);
+                await service.initSubChannel(service, worker.sub, worker.pub ?? worker.store);
+                await service.subscribe.subscribe(key_1.KeyType.QUORUM, service.subscriptionHandler(), appId);
+                await service.subscribe.subscribe(key_1.KeyType.QUORUM, service.subscriptionHandler(), appId, service.topic);
+                await service.subscribe.subscribe(key_1.KeyType.QUORUM, service.subscriptionHandler(), appId, service.guid);
+                await service.initStreamChannel(service, worker.stream, worker.store);
+                service.router = await service.initRouter(worker, logger);
+                const key = service.store.mintKey(key_1.KeyType.STREAMS, {
+                    appId: service.appId,
+                    topic: worker.topic,
+                });
+                await service.router.consumeMessages(key, 'WORKER', service.guid, worker.callback);
+                service.inited = (0, utils_1.formatISODate)(new Date());
+                services.push(service);
+            }
+        }
+        return services;
+    }
+    /**
+     * @private
+     */
+    verifyWorkerFields(worker) {
+        if (!(0, utils_1.identifyProvider)(worker.store) ||
+            !(0, utils_1.identifyProvider)(worker.stream) ||
+            !(0, utils_1.identifyProvider)(worker.sub) ||
+            !(worker.topic && worker.callback)) {
+            throw new Error('worker must include `store`, `stream`, and `sub` fields along with a callback function and topic.');
+        }
+    }
+    /**
+     * @private
+     */
+    async initStoreChannel(service, store) {
+        service.store = await factory_4.StoreServiceFactory.init(store, service.namespace, service.appId, service.logger);
+    }
+    /**
+     * @private
+     */
+    async initSubChannel(service, sub, store) {
+        service.subscribe = await factory_3.SubServiceFactory.init(sub, store, service.namespace, service.appId, service.guid, service.logger);
+    }
+    /**
+     * @private
+     */
+    async initStreamChannel(service, stream, store) {
+        service.stream = await factory_2.StreamServiceFactory.init(stream, store, service.namespace, service.appId, service.logger);
+    }
+    /**
+     * @private
+     */
+    async initRouter(worker, logger) {
+        const throttle = await this.store.getThrottleRate(worker.topic);
+        return new router_1.Router({
+            namespace: this.namespace,
+            appId: this.appId,
+            guid: this.guid,
+            role: stream_1.StreamRole.WORKER,
+            topic: worker.topic,
+            reclaimDelay: worker.reclaimDelay,
+            reclaimCount: worker.reclaimCount,
+            throttle,
+        }, this.stream, logger);
+    }
+    /**
+     * @private
+     */
+    subscriptionHandler() {
+        const self = this;
+        return async (topic, message) => {
+            self.logger.debug('worker-event-received', { topic, type: message.type });
+            if (message.type === 'throttle') {
+                if (message.topic !== null) {
+                    //undefined allows passthrough
+                    self.throttle(message.throttle);
+                }
+            }
+            else if (message.type === 'ping') {
+                self.sayPong(self.appId, self.guid, message.originator, message.details);
+            }
+            else if (message.type === 'rollcall') {
+                if (message.topic !== null) {
+                    //undefined allows passthrough
+                    self.doRollCall(message);
+                }
+            }
+        };
+    }
+    /**
+     * A quorum-wide command to broadcaset system details.
+     *
+     */
+    async doRollCall(message) {
+        let iteration = 0;
+        const max = !isNaN(message.max) ? message.max : enums_1.HMSH_QUORUM_ROLLCALL_CYCLES;
+        if (this.rollCallInterval)
+            clearTimeout(this.rollCallInterval);
+        const base = message.interval / 2;
+        const amount = base + Math.ceil(Math.random() * base);
+        do {
+            await (0, utils_1.sleepFor)(Math.ceil(Math.random() * 1000));
+            await this.sayPong(this.appId, this.guid, null, true, message.signature);
+            if (!message.interval)
+                return;
+            const { promise, timerId } = (0, utils_1.XSleepFor)(amount * 1000);
+            this.rollCallInterval = timerId;
+            await promise;
+        } while (this.rollCallInterval && iteration++ < max - 1);
+    }
+    cancelRollCall() {
+        if (this.rollCallInterval) {
+            clearTimeout(this.rollCallInterval);
+            delete this.rollCallInterval;
+        }
+    }
+    /**
+     * @private
+     */
+    stop() {
+        this.cancelRollCall();
+    }
+    /**
+     * @private
+     */
+    async sayPong(appId, guid, originator, details = false, signature = false) {
+        let profile;
+        if (details) {
+            const params = {
+                appId: this.appId,
+                topic: this.topic,
+            };
+            profile = {
+                engine_id: this.guid,
+                namespace: this.namespace,
+                app_id: this.appId,
+                worker_topic: this.topic,
+                stream: this.store.mintKey(key_1.KeyType.STREAMS, params),
+                counts: this.router?.counts,
+                timestamp: (0, utils_1.formatISODate)(new Date()),
+                inited: this.inited,
+                throttle: this.router?.throttle,
+                reclaimDelay: this.router?.reclaimDelay,
+                reclaimCount: this.router?.reclaimCount,
+                system: await (0, utils_1.getSystemHealth)(),
+                signature: signature ? this.callback.toString() : undefined,
+            };
+        }
+        this.subscribe.publish(key_1.KeyType.QUORUM, {
+            type: 'pong',
+            guid,
+            originator,
+            profile,
+        }, appId);
+    }
+    /**
+     * @private
+     */
+    async throttle(delayInMillis) {
+        this.router?.setThrottle(delayInMillis);
+    }
+}
+exports.WorkerService = WorkerService;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "description": "Permanent-Memory Workflows & AI Agents",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",

package/.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,38 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: ''
-assignees: ''
-
----
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-**To Reproduce**
-Steps to reproduce the behavior:
-1. Go to '...'
-2. Click on '....'
-3. Scroll down to '....'
-4. See error
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
-
-**Desktop (please complete the following information):**
- - OS: [e.g. iOS]
- - Browser [e.g. chrome, safari]
- - Version [e.g. 22]
-
-**Smartphone (please complete the following information):**
- - Device: [e.g. iPhone6]
- - OS: [e.g. iOS8.1]
- - Browser [e.g. stock browser, safari]
- - Version [e.g. 22]
-
-**Additional context**
-Add any other context about the problem here.

package/.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,20 +0,0 @@
----
-name: Feature request
-about: Suggest an idea for this project
-title: ''
-labels: ''
-assignees: ''
-
----
-
-**Is your feature request related to a problem? Please describe.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
-
-**Describe the solution you'd like**
-A clear and concise description of what you want to happen.
-
-**Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
-
-**Additional context**
-Add any other context or screenshots about the feature request here.

package/typedoc.json

@@ -1,47 +0,0 @@
-{
-  "entryPoints": [
-    "modules/enums.ts",
-    "modules/errors.ts",
-    "modules/key.ts",
-    "modules/utils.ts",
-    "services/connector/factory.ts",
-    "services/connector/index.ts",
-    "services/hotmesh/index.ts",
-    "services/meshcall/index.ts",
-    "services/meshdata/index.ts",
-    "services/memflow/index.ts",
-    "services/memflow/client.ts",
-    "services/memflow/handle.ts",
-    "services/memflow/search.ts",
-    "services/memflow/context.ts",
-    "services/memflow/worker.ts",
-    "services/memflow/workflow.ts",
-    "services/memflow/connection.ts",
-    "services/meshos/index.ts",
-    "services/search/index.ts",
-    "services/store/index.ts",
-    "services/stream/index.ts",
-    "services/sub/index.ts",
-    "types/*.ts",
-    "index.ts"
-  ],
-  "highlightLanguages": [
-    "bash",
-    "console",
-    "css",
-    "html",
-    "javascript",
-    "json",
-    "jsonc",
-    "json5",
-    "tsx",
-    "typescript",
-    "yaml"
-  ],
-  "exclude": ["types/index.ts"],
-  "out": "docs/hotmesh",
-  "name": "",
-  "excludePrivate": true,
-  "theme": "default",
-  "customCss": "docs/css/custom.css"
-}

package/types/activity.ts

@@ -1,268 +0,0 @@
-import { MetricTypes } from './stats';
-import { StreamRetryPolicy } from './stream';
-
-type ActivityExecutionType =
-  | 'trigger'
-  | 'await'
-  | 'worker'
-  | 'activity'
-  | 'emit'
-  | 'interrupt'
-  | 'cycle'
-  | 'signal'
-  | 'hook';
-
-type Consumes = Record<string, string[]>;
-
-interface BaseActivity {
-  title?: string;
-  type?: ActivityExecutionType;
-  subtype?: string;
-  statusThreshold?: number; //default is 0; set to 1 to ensure not last standing; message will be ignored as if too late to process
-  statusThresholdType?: 'stop' | 'throw' | 'stall'; //default is stop; must explicitly set to throw to throw an error or stall to stall
-  input?: Record<string, any>;
-  output?: Record<string, any>;
-  settings?: Record<string, any>;
-  job?: Record<string, any>;
-  hook?: Record<string, any>;
-  telemetry?: Record<string, any>;
-  emit?: boolean; //if true, the activity will emit a message to the `publishes` topic immediately before transitioning to adjacent activities
-  sleep?: number; //@pipe /in seconds
-  expire?: number; //-1 forever; 0 persists the flow until the parent flow that expired it is dismissed; 15 seconds is the default (copied from the YAML at compile time)
-  persistent?: boolean; //if true, the job will persist beyond completion (copied from the YAML at compile time)
-  persist?: boolean; //when true, the activity will emit the job completed event and start the `persistence countdown`, using the 'expire' value
-  retry?: StreamRetryPolicy;
-  cycle?: boolean; //if true, the `notary` will leave leg 2 open, so it can be re/cycled
-  collationInt?: number; //compiler
-  consumes?: Consumes; //compiler
-  PRODUCES?: string[]; //compiler
-  produces?: string[]; //compiler
-  publishes?: string; //compiler
-  subscribes?: string; //compiler
-  trigger?: string; //compiler
-  parent?: string; //compiler
-  ancestors?: string[]; //compiler
-}
-
-interface Measure {
-  measure: MetricTypes;
-  target: string;
-}
-
-interface TriggerActivityStats {
-  /**
-   * dependent parent job id; including this allows the parent's
-   * expiration/interruption events to cascade; set
-   * `expire` in the YAML for the dependent graph
-   * to 0 and provide the parent for dependent,
-   * cascading interruption and cleanup
-   */
-  parent?: string;
-  /**
-   * adjacent parent job id; this is the actual adjacent
-   * parent in the graph, but it is not used for cascading expiration
-   */
-  adjacent?: string;
-  id?: { [key: string]: unknown } | string;
-  key?: { [key: string]: unknown } | string;
-  /**
-   * @deprecated
-   * return 'infinity' to disable; default behavior
-   * is to always segment keys by time to ensure
-   * indexes (Redis LIST) never grow unbounded
-   * as a default behavior; for now, 5m is default
-   * and infinity can be set to override
-   */
-  granularity?: string;
-  /**
-   * @deprecated
-   * what to capture
-   */
-  measures?: Measure[];
-}
-
-interface TriggerActivity extends BaseActivity {
-  type: 'trigger';
-  stats?: TriggerActivityStats;
-}
-
-interface AwaitActivity extends BaseActivity {
-  type: 'await';
-  eventName: string;
-  timeout: number;
-  await?: boolean; //if exlicitly false do not await the response
-}
-
-interface WorkerActivity extends BaseActivity {
-  type: 'worker';
-  topic: string;
-  timeout: number;
-}
-
-interface CycleActivity extends BaseActivity {
-  type: 'cycle';
-  ancestor: string; //ancestor activity id
-}
-
-interface HookActivity extends BaseActivity {
-  type: 'hook';
-}
-
-interface SignalActivity extends BaseActivity {
-  type: 'signal'; //signal activities call hook/hookAll
-  subtype: 'one' | 'all'; //trigger: hook(One) or hookAll
-  topic: string; //e.g., 'hook.resume'
-  key_name?: string; //e.g., 'parent_job_id'
-  key_value?: string; //e.g., '1234567890'
-  scrub?: boolean; //if true, the index will be deleted after use
-  signal?: Record<string, any>; //used to define/map the signal input data (what to send/singnal into the job(s))
-  resolver?: Record<string, any>; //used to define/map the signal key resolver (the key used to lookup the job(s that are assigned to the key)
-  status?: string; //pending, success (default), error
-  code?: number; //202, 200 (default)
-}
-
-interface InterruptActivity extends BaseActivity {
-  type: 'interrupt';
-
-  /**
-   * Optional Reason; will be used as the error `message` when thrown
-   * @default 'Job Interrupted'
-   */
-  reason?: string;
-
-  /**
-   * throw JobInterrupted error upon interrupting
-   * @default true
-   */
-  throw?: boolean;
-
-  /**
-   * Interrupt child/descendant jobs
-   * @default false
-   */
-  descend?: boolean;
-
-  /**
-   * Target job id (if not present the current job will be targeted)
-   */
-  target?: string;
-
-  /**
-   * Optional topic to publish the interrupt message (if not present the current job topic will be used
-   */
-  topic?: string;
-
-  /**
-   * Optional Error Code; will be used as the error `code` when thrown
-   * @default 410
-   */
-  code?: number;
-
-  /**
-   * Optional stack trace
-   */
-  stack?: string;
-}
-
-type ActivityType =
-  | BaseActivity
-  | TriggerActivity
-  | AwaitActivity
-  | WorkerActivity
-  | InterruptActivity
-  | HookActivity
-  | SignalActivity
-  | CycleActivity;
-
-type ActivityData = Record<string, any>;
-
-/**
- * Type definition for activity metadata.
- */
-type ActivityMetadata = {
-  /**
-   * Unique identifier for the activity.
-   */
-  aid: string;
-
-  /**
-   * Type of the activity.
-   */
-  atp: string;
-
-  /**
-   * Subtype of the activity.
-   */
-  stp: string;
-
-  /**
-   * Timestamp when the activity was created, in GMT.
-   */
-  ac: string;
-
-  /**
-   * Timestamp when the activity was last updated, in GMT.
-   */
-  au: string;
-
-  /**
-   * Optional stringified JSON containing error details such as message, code, and an optional error object.
-   */
-  err?: string;
-
-  /**
-   * OpenTelemetry span context for the first leg of the activity.
-   */
-  l1s?: string;
-
-  /**
-   * OpenTelemetry span context for the second leg of the activity.
-   */
-  l2s?: string;
-
-  /**
-   * Dimensional address, used for additional metadata.
-   */
-  dad?: string;
-
-  /**
-   * Status of the activity, could be codes representing different states.
-   */
-  as?: string;
-};
-
-type ActivityContext = {
-  data?: ActivityData | null;
-  metadata: ActivityMetadata;
-  hook?: ActivityData;
-};
-
-type ActivityDuplex = 1 | 2;
-
-type ActivityDataType = {
-  data?: Record<string, unknown>;
-  metadata?: Record<string, unknown>;
-  hook?: Record<string, unknown>;
-};
-
-type ActivityLeg = 1 | 2;
-
-export {
-  ActivityContext,
-  ActivityData,
-  ActivityDataType,
-  ActivityDuplex,
-  ActivityLeg,
-  ActivityMetadata,
-  ActivityType,
-  Consumes,
-  TriggerActivityStats,
-  AwaitActivity,
-  CycleActivity,
-  HookActivity,
-  SignalActivity,
-  BaseActivity,
-  InterruptActivity,
-  TriggerActivity,
-  WorkerActivity,
-};

package/types/app.ts

@@ -1,20 +0,0 @@
-interface App {
-  name: string;
-  title: string;
-  description: string;
-}
-
-type AppVID = {
-  version: string;
-  id: string;
-};
-
-type AppTransitions = {
-  [key: string]: Record<string, unknown>;
-};
-
-type AppSubscriptions = {
-  [key: string]: string;
-};
-
-export { App, AppVID, AppTransitions, AppSubscriptions };

package/types/async.ts

@@ -1,6 +0,0 @@
-type AsyncSignal = {
-  topic: string;
-  jobId: string;
-};
-
-export { AsyncSignal };

package/types/cache.ts

@@ -1 +0,0 @@
-export type CacheMode = 'nocache' | 'cache';

package/types/collator.ts

@@ -1,9 +0,0 @@
-export type CollationStage = 'enter' | 'exit' | 'confirm';
-
-export enum CollationFaultType {
-  MISSING = 'missing', //`as` uninitialized; leg1 entry not allowed
-  DUPLICATE = 'duplicate', //1st digit < 8
-  INACTIVE = 'inactive', //3rd digit is 8
-  INVALID = 'invalid', //unknown value (corrupt for unknown reasons)
-  FORBIDDEN = 'forbidden', //leg 1 not completed; reentry (leg 2) not allowed
-}

package/types/error.ts

@@ -1,56 +0,0 @@
-export type MemFlowChildErrorType = {
-  arguments: string[];
-  await?: boolean;
-  backoffCoefficient?: number;
-  index: number;
-  expire?: number;
-  persistent?: boolean;
-  signalIn?: boolean;
-  maximumAttempts?: number;
-  maximumInterval?: number;
-  originJobId: string | null;
-  parentWorkflowId: string;
-  workflowDimension: string;
-  workflowId: string;
-  workflowTopic: string;
-};
-
-export type MemFlowWaitForAllErrorType = {
-  items: string[];
-  workflowId: string;
-  workflowTopic: string;
-  parentWorkflowId: string;
-  originJobId: string | null;
-  size: number;
-  index: number;
-  workflowDimension: string;
-};
-
-export type MemFlowProxyErrorType = {
-  arguments: string[];
-  activityName: string;
-  backoffCoefficient?: number;
-  index: number;
-  expire?: number;
-  maximumAttempts?: number;
-  maximumInterval?: number;
-  originJobId: string | null;
-  parentWorkflowId: string;
-  workflowDimension: string;
-  workflowId: string;
-  workflowTopic: string;
-};
-
-export type MemFlowWaitForErrorType = {
-  signalId: string;
-  index: number;
-  workflowDimension: string;
-  workflowId: string;
-};
-
-export type MemFlowSleepErrorType = {
-  duration: number;
-  index: number;
-  workflowDimension: string;
-  workflowId: string;
-};