@hotmeshio/hotmesh 0.13.0 → 0.14.0

package/build/services/engine/completion.js

@@ -0,0 +1,145 @@
+"use strict";
+/**
+ * Job lifecycle completion — parent notification, cleanup, and expiry.
+ *
+ * When a job finishes (or is interrupted), this module:
+ * 1. Notifies the parent job (if this is a child via execChild)
+ * 2. Publishes to one-time and permanent subscribers
+ * 3. Registers the job for TTL-based cleanup
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.resolveExpires = exports.runJobCompletionTasks = exports.scrub = exports.interrupt = exports.resolveError = exports.hasParentJob = exports.execAdjacentParent = void 0;
+const enums_1 = require("../../modules/enums");
+const utils_1 = require("../../modules/utils");
+const key_1 = require("../../modules/key");
+const stream_1 = require("../../types/stream");
+/**
+ * Sends the child job result back to the waiting parent activity.
+ * Only applies to non-severed children (execChild, not startChild).
+ */
+async function execAdjacentParent(instance, context, jobOutput, emit = false, transaction) {
+    if (hasParentJob(context)) {
+        const error = resolveError(jobOutput.metadata);
+        const spn = context['$self']?.output?.metadata?.l2s ||
+            context['$self']?.output?.metadata?.l1s;
+        const streamData = {
+            metadata: {
+                guid: (0, utils_1.guid)(),
+                jid: context.metadata.pj,
+                gid: context.metadata.pg,
+                dad: context.metadata.pd,
+                aid: context.metadata.pa,
+                trc: context.metadata.trc,
+                spn,
+            },
+            type: stream_1.StreamDataType.RESULT,
+            data: jobOutput.data,
+        };
+        if (error && error.code) {
+            streamData.status = stream_1.StreamStatus.ERROR;
+            streamData.data = error;
+            streamData.code = error.code;
+            streamData.stack = error.stack;
+        }
+        else if (emit) {
+            streamData.status = stream_1.StreamStatus.PENDING;
+            streamData.code = enums_1.HMSH_CODE_PENDING;
+        }
+        else {
+            streamData.status = stream_1.StreamStatus.SUCCESS;
+            streamData.code = enums_1.HMSH_CODE_SUCCESS;
+        }
+        return (await instance.router?.publishMessage(null, streamData, transaction));
+    }
+}
+exports.execAdjacentParent = execAdjacentParent;
+function hasParentJob(context, checkSevered = false) {
+    if (checkSevered) {
+        return Boolean(context.metadata.pj && context.metadata.pa && !context.metadata.px);
+    }
+    return Boolean(context.metadata.pj && context.metadata.pa);
+}
+exports.hasParentJob = hasParentJob;
+function resolveError(metadata) {
+    if (metadata && metadata.err) {
+        return JSON.parse(metadata.err);
+    }
+}
+exports.resolveError = resolveError;
+async function interrupt(instance, topic, jobId, options = {}) {
+    await instance.store.interrupt(topic, jobId, options);
+    const context = (await instance.getState(topic, jobId));
+    const completionOpts = {
+        interrupt: options.descend,
+        expire: options.expire,
+    };
+    return (await runJobCompletionTasks(instance, context, completionOpts));
+}
+exports.interrupt = interrupt;
+async function scrub(instance, jobId) {
+    await instance.store.scrub(jobId);
+}
+exports.scrub = scrub;
+/**
+ * Orchestrates all post-completion work for a finished job:
+ * notify parent, publish to subscribers, schedule cleanup.
+ */
+async function runJobCompletionTasks(instance, context, options = {}, transaction) {
+    const isAwait = hasParentJob(context, true);
+    const isOneTimeSub = hasOneTimeSubscription(context);
+    const topic = await instance.getPublishesTopic(context);
+    let msgId;
+    let jobOutput;
+    if (isAwait || isOneTimeSub || topic) {
+        jobOutput = await instance.getState(context.metadata.tpc, context.metadata.jid);
+        if (isAwait) {
+            msgId = await execAdjacentParent(instance, context, jobOutput, options.emit, transaction);
+        }
+        if (transaction) {
+            await publishOneTimeSubscribers(instance, context, jobOutput, options.emit, transaction);
+            await publishPermanentSubscribers(instance, context, jobOutput, options.emit, transaction);
+        }
+        else {
+            publishOneTimeSubscribers(instance, context, jobOutput, options.emit);
+            publishPermanentSubscribers(instance, context, jobOutput, options.emit);
+        }
+    }
+    if (!options.emit) {
+        if (transaction) {
+            await instance.taskService.registerJobForCleanup(context.metadata.jid, resolveExpires(context, options), options, transaction);
+        }
+        else {
+            instance.taskService.registerJobForCleanup(context.metadata.jid, resolveExpires(context, options), options);
+        }
+    }
+    return msgId;
+}
+exports.runJobCompletionTasks = runJobCompletionTasks;
+function resolveExpires(context, options) {
+    return options.expire ?? context.metadata.expire ?? enums_1.HMSH_EXPIRE_JOB_SECONDS;
+}
+exports.resolveExpires = resolveExpires;
+function hasOneTimeSubscription(context) {
+    return Boolean(context.metadata.ngn);
+}
+async function publishOneTimeSubscribers(instance, context, jobOutput, emit = false, transaction) {
+    if (hasOneTimeSubscription(context)) {
+        const message = {
+            type: 'job',
+            topic: context.metadata.jid,
+            job: (0, utils_1.restoreHierarchy)(jobOutput),
+        };
+        await instance.subscribe.publish(key_1.KeyType.QUORUM, message, instance.appId, context.metadata.ngn, transaction);
+    }
+}
+async function publishPermanentSubscribers(instance, context, jobOutput, emit = false, transaction) {
+    const topic = await instance.getPublishesTopic(context);
+    if (topic) {
+        const message = {
+            type: 'job',
+            topic,
+            job: (0, utils_1.restoreHierarchy)(jobOutput),
+        };
+        await instance.subscribe.publish(key_1.KeyType.QUORUM, message, instance.appId, `${topic}.${context.metadata.jid}`, transaction);
+    }
+}

package/build/services/engine/dispatch.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Stream message routing — the engine's main re-entry point.
+ *
+ * Every message consumed from the engine stream arrives here.
+ * The dispatcher inspects the message type and delegates to the
+ * appropriate activity handler (Hook, Trigger, Await, Worker).
+ *
+ * Message types:
+ *   TIMEHOOK   → Hook.processTimeHookEvent   (sleep/cron awaken)
+ *   WEBHOOK    → Hook.processWebHookEvent     (signal-in awaken)
+ *   TRANSITION → Activity.process             (adjacent activity)
+ *   AWAIT      → Trigger.process              (child job spawn)
+ *   RESULT     → Await.processEvent           (child job result)
+ *   (default)  → Worker.processEvent          (worker response)
+ */
+import { ILogger } from '../logger';
+import { JobState, JobData } from '../../types/job';
+import { StreamDataResponse } from '../../types/stream';
+interface DispatchContext {
+    logger: ILogger;
+    initActivity(topic: string, data?: JobData, context?: JobState): Promise<any>;
+}
+export declare function processStreamMessage(instance: DispatchContext, streamData: StreamDataResponse): Promise<void>;
+export {};

package/build/services/engine/dispatch.js

@@ -0,0 +1,98 @@
+"use strict";
+/**
+ * Stream message routing — the engine's main re-entry point.
+ *
+ * Every message consumed from the engine stream arrives here.
+ * The dispatcher inspects the message type and delegates to the
+ * appropriate activity handler (Hook, Trigger, Await, Worker).
+ *
+ * Message types:
+ *   TIMEHOOK   → Hook.processTimeHookEvent   (sleep/cron awaken)
+ *   WEBHOOK    → Hook.processWebHookEvent     (signal-in awaken)
+ *   TRANSITION → Activity.process             (adjacent activity)
+ *   AWAIT      → Trigger.process              (child job spawn)
+ *   RESULT     → Await.processEvent           (child job result)
+ *   (default)  → Worker.processEvent          (worker response)
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.processStreamMessage = void 0;
+const stream_1 = require("../../types/stream");
+async function processStreamMessage(instance, streamData) {
+    instance.logger.debug('engine-process', {
+        jid: streamData.metadata.jid,
+        gid: streamData.metadata.gid,
+        dad: streamData.metadata.dad,
+        aid: streamData.metadata.aid,
+        status: streamData.status || stream_1.StreamStatus.SUCCESS,
+        code: streamData.code || 200,
+        type: streamData.type,
+    });
+    const context = {
+        metadata: {
+            guid: streamData.metadata.guid,
+            jid: streamData.metadata.jid,
+            gid: streamData.metadata.gid,
+            dad: streamData.metadata.dad,
+            aid: streamData.metadata.aid,
+        },
+        data: streamData.data,
+    };
+    if (streamData.type === stream_1.StreamDataType.TIMEHOOK) {
+        await dispatchTimeHook(instance, streamData, context);
+    }
+    else if (streamData.type === stream_1.StreamDataType.WEBHOOK) {
+        await dispatchWebHook(instance, streamData, context);
+    }
+    else if (streamData.type === stream_1.StreamDataType.TRANSITION) {
+        await dispatchTransition(instance, streamData, context);
+    }
+    else if (streamData.type === stream_1.StreamDataType.AWAIT) {
+        await dispatchAwait(instance, streamData, context);
+    }
+    else if (streamData.type === stream_1.StreamDataType.RESULT) {
+        await dispatchResult(instance, streamData, context);
+    }
+    else {
+        await dispatchWorkerResult(instance, streamData, context);
+    }
+    instance.logger.debug('engine-process-end', {
+        jid: streamData.metadata.jid,
+        gid: streamData.metadata.gid,
+        aid: streamData.metadata.aid,
+    });
+}
+exports.processStreamMessage = processStreamMessage;
+async function dispatchTimeHook(instance, streamData, context) {
+    const handler = (await instance.initActivity(`.${streamData.metadata.aid}`, context.data, context));
+    await handler.processTimeHookEvent(streamData.metadata.jid);
+}
+async function dispatchWebHook(instance, streamData, context) {
+    const handler = (await instance.initActivity(`.${streamData.metadata.aid}`, context.data, context));
+    await handler.processWebHookEvent(streamData.status, streamData.code);
+}
+async function dispatchTransition(instance, streamData, context) {
+    const handler = await instance.initActivity(`.${streamData.metadata.aid}`, context.data, context);
+    await handler.process();
+}
+async function dispatchAwait(instance, streamData, context) {
+    context.metadata = {
+        ...context.metadata,
+        pj: streamData.metadata.jid,
+        pg: streamData.metadata.gid,
+        pd: streamData.metadata.dad,
+        pa: streamData.metadata.aid,
+        px: streamData.metadata.await === false,
+        trc: streamData.metadata.trc,
+        spn: streamData.metadata.spn,
+    };
+    const handler = (await instance.initActivity(streamData.metadata.topic, streamData.data, context));
+    await handler.process();
+}
+async function dispatchResult(instance, streamData, context) {
+    const handler = (await instance.initActivity(`.${context.metadata.aid}`, streamData.data, context));
+    await handler.processEvent(streamData.status, streamData.code);
+}
+async function dispatchWorkerResult(instance, streamData, context) {
+    const handler = (await instance.initActivity(`.${streamData.metadata.aid}`, streamData.data, context));
+    await handler.processEvent(streamData.status, streamData.code, 'output');
+}

package/build/services/engine/index.d.ts

@@ -1,10 +1,25 @@
-import { Await } from '../activities/await';
-import { Cycle } from '../activities/cycle';
-import { Hook } from '../activities/hook';
-import { Interrupt } from '../activities/interrupt';
-import { Signal } from '../activities/signal';
-import { Worker } from '../activities/worker';
-import { Trigger } from '../activities/trigger';
+/**
+ * EngineService — the workflow execution engine.
+ *
+ * Consumes stream messages from the router and dispatches them
+ * to the appropriate activity handler (Trigger, Worker, Hook, …).
+ *
+ * Each section delegates to a purpose-specific module inside `engine/`.
+ * Open the module when you need implementation detail; read this file
+ * when you need the big picture.
+ *
+ * Lifecycle (maps to modules):
+ *   1. INIT       → init.ts       (channel setup, router, config)
+ *   2. VERSION    → version.ts    (app version resolution, caching)
+ *   3. SCHEMA     → schema.ts     (activity lookup, handler factory)
+ *   4. COMPILE    → compiler.ts   (YAML plan & deploy)
+ *   5. REPORT     → reporting.ts  (stats, IDs, query resolution)
+ *   6. DISPATCH   → dispatch.ts   (stream message → activity handler)
+ *   7. COMPLETION → completion.ts (parent notify, cleanup, expiry)
+ *   8. SIGNAL     → signal.ts     (webhook/timehook delivery, fan-out)
+ *   9. PUB/SUB    → pubsub.ts     (topic messaging, subscriptions)
+ *  10. STATE      → state.ts      (job state retrieval, export)
+ */
 import { ExporterService } from '../exporter';
 import { ILogger } from '../logger';
 import { Router } from '../router';
@@ -17,19 +32,20 @@ import { AppVID } from '../../types/app';
 import { ActivityType } from '../../types/activity';
 import { CacheMode } from '../../types/cache';
 import { ExportOptions, JobExport } from '../../types/exporter';
-import { JobState, JobData, JobMetadata, JobOutput, JobStatus, JobInterruptOptions, JobCompletionOptions, ExtensionType } from '../../types/job';
+import { JobState, JobData, JobOutput, JobStatus, JobInterruptOptions, JobCompletionOptions, ExtensionType } from '../../types/job';
 import { HotMeshApps, HotMeshConfig, HotMeshManifest, HotMeshSettings } from '../../types/hotmesh';
 import { ProviderClient, ProviderTransaction } from '../../types/provider';
 import { JobMessageCallback } from '../../types/quorum';
 import { StringAnyType, StringStringType } from '../../types/serializer';
 import { GetStatsOptions, IdsResponse, JobStatsInput, StatsResponse } from '../../types/stats';
-import { StreamCode, StreamData, StreamDataResponse, StreamError, StreamStatus } from '../../types/stream';
+import { StreamCode, StreamData, StreamDataResponse, StreamStatus } from '../../types/stream';
 import { WorkListTaskType } from '../../types/task';
 declare class EngineService {
     namespace: string;
     apps: HotMeshApps | null;
     appId: string;
     guid: string;
+    inited: string;
     exporter: ExporterService | null;
     /** @hidden */
     search: SearchService<ProviderClient> | null;
@@ -47,9 +63,6 @@ declare class EngineService {
     cacheMode: CacheMode;
     untilVersion: string | null;
     jobCallbacks: Record<string, JobMessageCallback>;
-    reporting: boolean;
-    jobId: number;
-    inited: string;
     /**
      * @private
      */
@@ -61,33 +74,10 @@ declare class EngineService {
     /**
      * @private
      */
-    verifyEngineFields(config: HotMeshConfig): void;
-    /**
-     * @private
-     */
-    initSearchChannel(search: ProviderClient, store?: ProviderClient): Promise<void>;
-    /**
-     * @private
-     */
-    initStoreChannel(store: ProviderClient): Promise<void>;
-    /**
-     * @private
-     */
-    initSubChannel(sub: ProviderClient, store: ProviderClient): Promise<void>;
-    /**
-     * @private
-     */
-    initStreamChannel(stream: ProviderClient, store: ProviderClient): Promise<void>;
-    /**
-     * @private
-     */
-    initRouter(config: HotMeshConfig): Promise<Router<StreamService<ProviderClient, ProviderTransaction>>>;
+    getSettings(): Promise<HotMeshSettings>;
     /**
-     * resolves the distributed executable version using a delay
-     * to allow deployment race conditions to resolve
      * @private
      */
-    fetchAndVerifyVID(vid: AppVID, count?: number): Promise<AppVID>;
     getVID(vid?: AppVID): Promise<AppVID>;
     /**
      * @private
@@ -96,28 +86,11 @@ declare class EngineService {
     /**
      * @private
      */
-    routeToSubscribers(topic: string, message: JobOutput): Promise<void>;
+    initActivity(topic: string, data?: JobData, context?: JobState): Promise<any>;
     /**
      * @private
      */
-    processWebHooks(): Promise<void>;
-    /**
-     * @private
-     */
-    processTimeHooks(): Promise<void>;
-    /**
-     * @private
-     */
-    throttle(delayInMillis: number): Promise<void>;
-    /**
-     * @private
-     */
-    initActivity(topic: string, data?: JobData, context?: JobState): Promise<Await | Cycle | Hook | Signal | Trigger | Worker | Interrupt>;
     getSchema(topic: string): Promise<[activityId: string, schema: ActivityType]>;
-    /**
-     * @private
-     */
-    getSettings(): Promise<HotMeshSettings>;
     /**
      * @private
      */
@@ -137,7 +110,7 @@ declare class EngineService {
     /**
      * @private
      */
-    getIds(topic: string, query: JobStatsInput, queryFacets?: any[]): Promise<IdsResponse>;
+    getIds(topic: string, query: JobStatsInput, queryFacets?: string[]): Promise<IdsResponse>;
     /**
      * @private
      */
@@ -157,22 +130,16 @@ declare class EngineService {
     /**
      * @private
      */
-    resolveError(metadata: JobMetadata): StreamError | undefined;
+    interrupt(topic: string, jobId: string, options?: JobInterruptOptions): Promise<string>;
     /**
      * @private
      */
-    interrupt(topic: string, jobId: string, options?: JobInterruptOptions): Promise<string>;
+    scrub(jobId: string): Promise<void>;
     /**
      * @private
      */
-    scrub(jobId: string): Promise<void>;
+    runJobCompletionTasks(context: JobState, options?: JobCompletionOptions, transaction?: ProviderTransaction): Promise<string | void>;
     /**
-     * Delivers a signal (data payload) to a paused hook activity,
-     * resuming its Leg 2 execution. The `topic` must match a hook rule
-     * defined in the YAML graph's `hooks` section. The engine locates
-     * the target activity and dimension for reentry based on the hook
-     * rule's match conditions.
-     *
      * @private
      */
     signal(topic: string, data: JobData, status?: StreamStatus, code?: StreamCode, transaction?: ProviderTransaction): Promise<string>;
@@ -181,13 +148,25 @@ declare class EngineService {
      */
     hookTime(jobId: string, gId: string, topicOrActivity: string, type?: WorkListTaskType): Promise<string | void>;
     /**
-     * Fan-out variant of `signal()` that delivers data to **all**
-     * paused workflows matching a search query. Useful for resuming
-     * a batch of workflows waiting on the same external event.
-     *
      * @private
      */
     signalAll(hookTopic: string, data: JobData, keyResolver: JobStatsInput, queryFacets?: string[]): Promise<string[]>;
+    /**
+     * @private
+     */
+    routeToSubscribers(topic: string, message: JobOutput): Promise<void>;
+    /**
+     * @private
+     */
+    processWebHooks(): Promise<void>;
+    /**
+     * @private
+     */
+    processTimeHooks(): Promise<void>;
+    /**
+     * @private
+     */
+    throttle(delayInMillis: number): Promise<void>;
     /**
      * @private
      */
@@ -219,11 +198,11 @@ declare class EngineService {
     /**
      * @private
      */
-    getPublishesTopic(context: JobState): Promise<string>;
+    pubPermSubs(context: JobState, jobOutput: JobOutput, emit?: boolean, transaction?: ProviderTransaction): Promise<void>;
     /**
      * @private
      */
-    pubPermSubs(context: JobState, jobOutput: JobOutput, emit?: boolean, transaction?: ProviderTransaction): Promise<void>;
+    getPublishesTopic(context: JobState): Promise<string>;
     /**
      * @private
      */
@@ -240,17 +219,6 @@ declare class EngineService {
      * @private
      */
     hasOneTimeSubscription(context: JobState): boolean;
-    /**
-     * @private
-     */
-    runJobCompletionTasks(context: JobState, options?: JobCompletionOptions, transaction?: ProviderTransaction): Promise<string | void>;
-    /**
-     * Job hash expiration is typically reliant on the metadata field
-     * if the activity concludes normally. However, if the job is `interrupted`,
-     * it will be expired immediately.
-     * @private
-     */
-    resolveExpires(context: JobState, options: JobCompletionOptions): number;
     /**
      * @private
      */