@hotmeshio/hotmesh 0.7.0 → 0.9.0

package/build/services/activities/signal.js

@@ -7,6 +7,111 @@ const mapper_1 = require("../mapper");
 const pipe_1 = require("../pipe");
 const telemetry_1 = require("../telemetry");
 const activity_1 = require("./activity");
+/**
+ * Sends a signal to one or more paused flows, resuming their execution.
+ * The `signal` activity is the counterpart to a `Hook` activity
+ * configured with a webhook listener. It allows any flow to reach into
+ * another flow and deliver data to a waiting hook, regardless of the
+ * relationship between the flows.
+ *
+ * ## YAML Configuration — Signal One
+ *
+ * Resumes a single paused flow by publishing to the hook's topic. Use
+ * `subtype: one` when you know the specific hook topic to signal.
+ *
+ * ```yaml
+ * app:
+ *   id: myapp
+ *   version: '1'
+ *   graphs:
+ *     - subscribes: signal.start
+ *       expire: 120
+ *
+ *       activities:
+ *         t1:
+ *           type: trigger
+ *
+ *         resume_hook:
+ *           type: signal
+ *           subtype: one
+ *           topic: my.hook.topic          # the hook's registered topic
+ *           status: success               # optional: success (default) or pending
+ *           code: 200                     # optional: 200 (default) or 202 (keep-alive)
+ *           signal:
+ *             schema:
+ *               type: object
+ *               properties:
+ *                 approved: { type: boolean }
+ *             maps:
+ *               approved: true            # data delivered to the hook
+ *
+ *         done:
+ *           type: hook
+ *
+ *       transitions:
+ *         t1:
+ *           - to: resume_hook
+ *         resume_hook:
+ *           - to: done
+ * ```
+ *
+ * A `code: 202` signal delivers data but keeps the hook alive for
+ * additional signals. A `code: 200` (default) closes the hook.
+ *
+ * ## YAML Configuration — Signal All
+ *
+ * Resumes all paused flows that share a common job key facet. Use
+ * `subtype: all` for fan-out patterns where multiple waiting flows
+ * should be resumed simultaneously.
+ *
+ * ```yaml
+ * app:
+ *   id: myapp
+ *   version: '1'
+ *   graphs:
+ *     - subscribes: signal.fan.out
+ *       expire: 120
+ *
+ *       activities:
+ *         t1:
+ *           type: trigger
+ *
+ *         resume_all:
+ *           type: signal
+ *           subtype: all
+ *           topic: hook.resume
+ *           key_name: parent_job_id       # index facet name
+ *           key_value: '{$job.metadata.jid}'
+ *           scrub: true                   # clean up indexes after use
+ *           resolver:
+ *             maps:
+ *               data:
+ *                 parent_job_id: '{$job.metadata.jid}'
+ *               scrub: true
+ *           signal:
+ *             maps:
+ *               done: true                # data delivered to all matching hooks
+ *
+ *         done:
+ *           type: hook
+ *
+ *       transitions:
+ *         t1:
+ *           - to: resume_all
+ *         resume_all:
+ *           - to: done
+ * ```
+ *
+ * ## Execution Model
+ *
+ * Signal is a **Category B (Leg1-only with children)** activity:
+ * - Bundles the hook signal with the Leg 1 completion marker in a
+ *   single transaction (`hookOne`) or fires best-effort (`hookAll`).
+ * - Executes the crash-safe `executeLeg1StepProtocol` to transition
+ *   to adjacent activities.
+ *
+ * @see {@link SignalActivity} for the TypeScript interface
+ */
 class Signal extends activity_1.Activity {
     constructor(config, data, metadata, hook, engine, context) {
         super(config, data, metadata, hook, engine, context);
@@ -20,34 +125,36 @@ class Signal extends activity_1.Activity {
         });
         let telemetry;
         try {
-            await this.verifyEntry();
+            //Category B: entry with step resume
+            await this.verifyLeg1Entry();
             telemetry = new telemetry_1.TelemetryService(this.engine.appId, this.config, this.metadata, this.context);
             telemetry.startActivitySpan(this.leg);
-            //save state and notarize early completion (signals only run leg1)
-            const transaction = this.store.transact();
             this.adjacencyList = await this.filterAdjacent();
             this.mapOutputData();
             this.mapJobData();
-            await this.setState(transaction);
-            await collator_1.CollatorService.notarizeEarlyCompletion(this, transaction);
-            await this.setStatus(this.adjacencyList.length - 1, transaction);
-            const multiResponse = (await transaction.exec());
-            //todo: this should execute BEFORE the status is decremented
-            if (this.config.subtype === 'all') {
-                await this.hookAll();
-            }
-            else {
-                await this.hookOne();
-            }
-            //transition to adjacent activities
-            const jobStatus = this.resolveStatus(multiResponse);
-            const attrs = { 'app.job.jss': jobStatus };
-            const messageIds = await this.transition(this.adjacencyList, jobStatus);
-            if (messageIds.length) {
-                attrs['app.activity.mids'] = messageIds.join(',');
+            //Step A: Bundle signal hook with Leg1 completion marker.
+            //hookOne is transactional; hookAll is best-effort (complex multi-step).
+            if (!collator_1.CollatorService.isGuidStep1Done(this.guidLedger)) {
+                const txn1 = this.store.transact();
+                if (this.config.subtype === 'all') {
+                    await this.signalAll();
+                }
+                else {
+                    await this.signalOne(txn1);
+                }
+                await this.setState(txn1);
+                if (this.adjacentIndex === 0) {
+                    await collator_1.CollatorService.notarizeLeg1Completion(this, txn1);
+                }
+                await collator_1.CollatorService.notarizeStep1(this, this.context.metadata.guid, txn1);
+                await txn1.exec();
+                //update in-memory guidLedger so executeLeg1StepProtocol skips Step A
+                this.guidLedger += collator_1.CollatorService.WEIGHTS.STEP1_WORK;
             }
+            //Steps B and C: spawn children + semaphore + edge capture
+            await this.executeLeg1StepProtocol(this.adjacencyList.length - 1);
             telemetry.mapActivityAttributes();
-            telemetry.setActivityAttributes(attrs);
+            telemetry.setActivityAttributes({});
             return this.context.metadata.aid;
         }
         catch (error) {
@@ -102,19 +209,20 @@ class Signal extends activity_1.Activity {
         }
     }
     /**
-     * The signal activity will hook one
+     * The signal activity will hook one. Accepts an optional transaction
+     * so the hook publish can be bundled with the Leg1 completion marker.
      */
-    async hookOne() {
+    async signalOne(transaction) {
         const topic = pipe_1.Pipe.resolve(this.config.topic, this.context);
         const signalInputData = this.mapSignalData();
         const status = pipe_1.Pipe.resolve(this.config.status, this.context);
         const code = pipe_1.Pipe.resolve(this.config.code, this.context);
-        return await this.engine.hook(topic, signalInputData, status, code);
+        return await this.engine.signal(topic, signalInputData, status, code, transaction);
     }
     /**
-     * The signal activity will hook all paused jobs that share the same job key.
+     * Signals all paused jobs that share the same job key, resuming their execution.
      */
-    async hookAll() {
+    async signalAll() {
         //prep 1) generate `input signal data` (essentially the webhook payload)
         const signalInputData = this.mapSignalData();
         //prep 2) generate data that resolves the job key (per the YAML config)
@@ -127,8 +235,8 @@ class Signal extends activity_1.Activity {
         const key_name = pipe_1.Pipe.resolve(this.config.key_name, this.context);
         const key_value = pipe_1.Pipe.resolve(this.config.key_value, this.context);
         const indexQueryFacets = [`${key_name}:${key_value}`];
-        //execute: `hookAll` will now resume all paused jobs that share the same job key
-        return await this.engine.hookAll(this.config.topic, signalInputData, keyResolverData, indexQueryFacets);
+        //execute: `signalAll` will now resume all paused jobs that share the same job key
+        return await this.engine.signalAll(this.config.topic, signalInputData, keyResolverData, indexQueryFacets);
     }
 }
 exports.Signal = Signal;

package/build/services/activities/trigger.d.ts

@@ -1,14 +1,67 @@
 import { EngineService } from '../engine';
 import { ActivityData, ActivityMetadata, ActivityType, TriggerActivity } from '../../types/activity';
-import { JobState, ExtensionType, JobStatus } from '../../types/job';
+import { JobState, ExtensionType } from '../../types/job';
 import { ProviderTransaction } from '../../types/provider';
-import { StringScalarType } from '../../types/serializer';
 import { Activity } from './activity';
+/**
+ * The entry point for every workflow graph. Each graph must have exactly
+ * one `trigger` activity, which executes when the graph's `subscribes`
+ * topic receives a message (via `hotMesh.pub` or `hotMesh.pubsub`).
+ *
+ * The trigger initializes the job, sets its unique ID and key, binds
+ * the incoming payload as the trigger's output data, and transitions
+ * to adjacent activities defined in the `transitions` section.
+ *
+ * ## YAML Configuration
+ *
+ * ```yaml
+ * app:
+ *   id: myapp
+ *   version: '1'
+ *   graphs:
+ *     - subscribes: order.placed       # the trigger fires when this topic receives a message
+ *       publishes: order.processed     # emitted when the graph completes
+ *       expire: 120
+ *
+ *       activities:
+ *         t1:
+ *           type: trigger
+ *           entity: '{$self.input.data.entityType}'
+ *           job:
+ *             maps:
+ *               myField: '{$self.output.data.inputField}'
+ *           stats:
+ *             id: '{$self.input.data.workflowId}'
+ *             key: '{$self.input.data.parentId}'
+ *             parent: '{$self.input.data.originJobId}'
+ *             adjacent: '{$self.input.data.parentJobId}'
+ *
+ *         process:
+ *           type: worker
+ *           topic: order.process
+ *
+ *       transitions:
+ *         t1:
+ *           - to: process
+ * ```
+ *
+ * ## Key Behaviors
+ *
+ * - **Job ID Resolution**: If `stats.id` is provided, it resolves via `@pipe`
+ *   expressions against the input data. Otherwise a UUID is generated.
+ * - **Duplicate Detection**: If a job with the same ID already exists,
+ *   a `DuplicateJobError` is thrown (unless it's a crash-recovery scenario).
+ * - **Pending Mode**: When invoked with `{ pending: <seconds> }`, the trigger
+ *   creates the job but does not transition to children until resumed.
+ * - **Crash Recovery**: Uses a 3-step inception protocol with GUID ledger
+ *   to ensure atomic job creation survives process crashes.
+ *
+ * @see {@link TriggerActivity} for the TypeScript interface
+ */
 declare class Trigger extends Activity {
     config: TriggerActivity;
     constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService, context?: JobState);
     process(options?: ExtensionType): Promise<string>;
-    transitionAndLogAdjacent(options: ExtensionType, jobStatus: JobStatus, attrs: StringScalarType): Promise<void>;
     /**
      * `pending` flows will not transition from the trigger to adjacent children until resumed
      */
@@ -31,7 +84,6 @@ declare class Trigger extends Activity {
     getJobStatus(): number;
     resolveJobId(context: Partial<JobState>): string;
     resolveJobKey(context: Partial<JobState>): string;
-    setStateNX(status?: number, entity?: string | undefined): Promise<void>;
     setStats(transaction?: ProviderTransaction): Promise<void>;
 }
 export { Trigger };

package/build/services/activities/trigger.js

@@ -10,6 +10,61 @@ const serializer_1 = require("../serializer");
 const telemetry_1 = require("../telemetry");
 const mapper_1 = require("../mapper");
 const activity_1 = require("./activity");
+/**
+ * The entry point for every workflow graph. Each graph must have exactly
+ * one `trigger` activity, which executes when the graph's `subscribes`
+ * topic receives a message (via `hotMesh.pub` or `hotMesh.pubsub`).
+ *
+ * The trigger initializes the job, sets its unique ID and key, binds
+ * the incoming payload as the trigger's output data, and transitions
+ * to adjacent activities defined in the `transitions` section.
+ *
+ * ## YAML Configuration
+ *
+ * ```yaml
+ * app:
+ *   id: myapp
+ *   version: '1'
+ *   graphs:
+ *     - subscribes: order.placed       # the trigger fires when this topic receives a message
+ *       publishes: order.processed     # emitted when the graph completes
+ *       expire: 120
+ *
+ *       activities:
+ *         t1:
+ *           type: trigger
+ *           entity: '{$self.input.data.entityType}'
+ *           job:
+ *             maps:
+ *               myField: '{$self.output.data.inputField}'
+ *           stats:
+ *             id: '{$self.input.data.workflowId}'
+ *             key: '{$self.input.data.parentId}'
+ *             parent: '{$self.input.data.originJobId}'
+ *             adjacent: '{$self.input.data.parentJobId}'
+ *
+ *         process:
+ *           type: worker
+ *           topic: order.process
+ *
+ *       transitions:
+ *         t1:
+ *           - to: process
+ * ```
+ *
+ * ## Key Behaviors
+ *
+ * - **Job ID Resolution**: If `stats.id` is provided, it resolves via `@pipe`
+ *   expressions against the input data. Otherwise a UUID is generated.
+ * - **Duplicate Detection**: If a job with the same ID already exists,
+ *   a `DuplicateJobError` is thrown (unless it's a crash-recovery scenario).
+ * - **Pending Mode**: When invoked with `{ pending: <seconds> }`, the trigger
+ *   creates the job but does not transition to children until resumed.
+ * - **Crash Recovery**: Uses a 3-step inception protocol with GUID ledger
+ *   to ensure atomic job creation survives process crashes.
+ *
+ * @see {@link TriggerActivity} for the TypeScript interface
+ */
 class Trigger extends activity_1.Activity {
     constructor(config, data, metadata, hook, engine, context) {
         super(config, data, metadata, hook, engine, context);
@@ -30,40 +85,84 @@ class Trigger extends activity_1.Activity {
             const initialStatus = this.initStatus(options, this.adjacencyList.length);
             //config.entity is a pipe expression; if 'entity' exists, it will resolve
             const resolvedEntity = new mapper_1.MapperService({ entity: this.config.entity }, this.context).mapRules()?.entity;
-            await this.setStateNX(initialStatus, options?.entity || resolvedEntity);
+            const msgGuid = this.context.metadata.guid || (0, utils_1.guid)();
+            this.context.metadata.guid = msgGuid;
+            const { id: appId } = await this.engine.getVID();
+            //═══ Step 1: Inception (atomic job creation + GUID seed) ═══
+            const txn1 = this.store.transact();
+            await this.store.setStateNX(this.context.metadata.jid, appId, initialStatus, options?.entity || resolvedEntity, txn1);
+            await this.store.collateSynthetic(this.context.metadata.jid, msgGuid, collator_1.CollatorService.WEIGHTS.STEP1_WORK, txn1);
+            const results1 = (await txn1.exec());
+            const jobCreated = Number(results1[0]) > 0;
+            const guidValue = Number(results1[1]);
+            if (!jobCreated) {
+                if (guidValue > collator_1.CollatorService.WEIGHTS.STEP1_WORK) {
+                    //crash recovery: GUID was seeded on a prior attempt; resume
+                    this.guidLedger = guidValue;
+                    this.logger.info('trigger-crash-recovery', {
+                        job_id: this.context.metadata.jid,
+                        guid: msgGuid,
+                        guidLedger: guidValue,
+                    });
+                }
+                else {
+                    //true duplicate: another process owns this job
+                    throw new errors_1.DuplicateJobError(this.context.metadata.jid);
+                }
+            }
             await this.setStatus(initialStatus);
             this.bindSearchData(options);
             this.bindMarkerData(options);
-            const transaction = this.store.transact();
-            await this.setState(transaction);
-            await this.setStats(transaction);
-            if (options?.pending) {
-                await this.setExpired(options?.pending, transaction);
+            //═══ Step 2: Work (state + children + GUID Step 2) ═══
+            if (!collator_1.CollatorService.isGuidStep2Done(this.guidLedger)) {
+                const txn2 = this.store.transact();
+                await this.setState(txn2);
+                await this.setStats(txn2);
+                if (options?.pending) {
+                    await this.setExpired(options.pending, txn2);
+                }
+                //publish children (unless pending or job already complete)
+                if (isNaN(options?.pending) && this.adjacencyList.length && initialStatus > 0) {
+                    for (const child of this.adjacencyList) {
+                        await this.engine.router?.publishMessage(null, child, txn2);
+                    }
+                }
+                await collator_1.CollatorService.notarizeStep2(this, msgGuid, txn2);
+                await txn2.exec();
             }
-            await collator_1.CollatorService.notarizeInception(this, this.context.metadata.guid, transaction);
-            await transaction.exec();
+            //best-effort parent notification
             this.execAdjacentParent();
-            telemetry.mapActivityAttributes();
+            //═══ Step 3: Completion (if job immediately complete) ═══
+            //NOTE: runJobCompletionTasks is non-transactional here because
+            //the trigger runs inline within pub(). Subscribers register AFTER
+            //pub() returns, so pub notifications must be fire-and-forget to
+            //avoid a race. The GUID marker commits in its own transaction.
             const jobStatus = Number(this.context.metadata.js);
+            const needsCompletion = this.shouldEmit() ||
+                this.isJobComplete(jobStatus) ||
+                this.shouldPersistJob();
+            if (needsCompletion && !collator_1.CollatorService.isGuidStep3Done(this.guidLedger)) {
+                await this.engine.runJobCompletionTasks(this.context, { emit: !this.isJobComplete(jobStatus) && !this.shouldPersistJob() });
+                //NOTE: notarizeStep3 is fire-and-forget for the trigger.
+                //pubOneTimeSubs (inside runJobCompletionTasks) sends a NOTIFY
+                //that must not be processed until registerJobCallback runs
+                //AFTER pub() returns. An `await` here yields to the event loop,
+                //which could deliver the NOTIFY before the callback is registered.
+                const txn3 = this.store.transact();
+                collator_1.CollatorService.notarizeStep3(this, msgGuid, txn3)
+                    .then(() => txn3.exec())
+                    .catch((err) => this.logger.error('trigger-notarize-step3-error', { error: err }));
+            }
+            //telemetry
+            telemetry.mapActivityAttributes();
             telemetry.setJobAttributes({ 'app.job.jss': jobStatus });
             const attrs = { 'app.job.jss': jobStatus };
-            await this.transitionAndLogAdjacent(options, jobStatus, attrs);
             telemetry.setActivityAttributes(attrs);
             return this.context.metadata.jid;
         }
         catch (error) {
             telemetry?.setActivityError(error.message);
             if (error instanceof errors_1.DuplicateJobError) {
-                //todo: verify baseline in x-AZ rollover
-                await (0, utils_1.sleepFor)(1000);
-                const isOverage = await collator_1.CollatorService.isInceptionOverage(this, this.context.metadata.guid);
-                if (isOverage) {
-                    this.logger.info('trigger-collation-overage', {
-                        job_id: error.jobId,
-                        guid: this.context.metadata.guid,
-                    });
-                    return;
-                }
                 this.logger.error('duplicate-job-error', {
                     job_id: error.jobId,
                     guid: this.context.metadata.guid,
@@ -84,15 +183,6 @@ class Trigger extends activity_1.Activity {
             });
         }
     }
-    async transitionAndLogAdjacent(options = {}, jobStatus, attrs) {
-        //todo: enable resume from pending state
-        if (isNaN(options.pending)) {
-            const messageIds = await this.transition(this.adjacencyList, jobStatus);
-            if (messageIds.length) {
-                attrs['app.activity.mids'] = messageIds.join(',');
-            }
-        }
-    }
     /**
      * `pending` flows will not transition from the trigger to adjacent children until resumed
      */
@@ -231,12 +321,6 @@ class Trigger extends activity_1.Activity {
         const jobKey = this.config.stats?.key;
         return jobKey ? pipe_1.Pipe.resolve(jobKey, context) : '';
     }
-    async setStateNX(status, entity) {
-        const jobId = this.context.metadata.jid;
-        if (!await this.store.setStateNX(jobId, this.engine.appId, status, entity)) {
-            throw new errors_1.DuplicateJobError(jobId);
-        }
-    }
     async setStats(transaction) {
         const md = this.context.metadata;
         if (md.key && this.config.stats?.measures) {

package/build/services/activities/worker.d.ts

@@ -3,6 +3,113 @@ import { ActivityData, ActivityMetadata, ActivityType, WorkerActivity } from '..
 import { JobState } from '../../types/job';
 import { ProviderTransaction } from '../../types/provider';
 import { Activity } from './activity';
+/**
+ * Dispatches work to a registered callback function. The `worker` activity
+ * publishes a message to its configured `topic` stream, where a worker
+ * process picks it up, executes the callback, and returns a response
+ * that the engine captures as the activity's output.
+ *
+ * ## YAML Configuration
+ *
+ * ```yaml
+ * app:
+ *   id: myapp
+ *   version: '1'
+ *   graphs:
+ *     - subscribes: order.placed
+ *       expire: 120
+ *
+ *       activities:
+ *         t1:
+ *           type: trigger
+ *
+ *         a1:
+ *           type: worker
+ *           topic: work.do              # matches the registered worker topic
+ *           input:
+ *             schema:
+ *               type: object
+ *               properties:
+ *                 x: { type: string }
+ *             maps:
+ *               x: '{t1.output.data.inputField}'
+ *           output:
+ *             schema:
+ *               type: object
+ *               properties:
+ *                 y: { type: string }
+ *           job:
+ *             maps:
+ *               result: '{$self.output.data.y}'
+ *
+ *       transitions:
+ *         t1:
+ *           - to: a1
+ * ```
+ *
+ * ## Worker Registration (JavaScript)
+ *
+ * Workers are registered at initialization time via the `workers` array
+ * in `HotMesh.init`. Each worker binds a `topic` to a `callback` function.
+ *
+ * ```typescript
+ * const hotMesh = await HotMesh.init({
+ *   appId: 'myapp',
+ *   engine: { connection },
+ *   workers: [{
+ *     topic: 'work.do',
+ *     connection,
+ *     callback: async (data: StreamData) => ({
+ *       metadata: { ...data.metadata },
+ *       data: { y: `${data.data.x} transformed` }
+ *     })
+ *   }]
+ * });
+ * ```
+ *
+ * ## Retry Policy
+ *
+ * Retry behavior is configured at the **worker level** (not in YAML) via
+ * the `retryPolicy` option. Failed callbacks are retried with exponential
+ * backoff until `maximumAttempts` is exhausted. The `maximumInterval` caps
+ * the delay between retries.
+ *
+ * ```typescript
+ * const hotMesh = await HotMesh.init({
+ *   appId: 'myapp',
+ *   engine: { connection },
+ *   workers: [{
+ *     topic: 'work.backoff',
+ *     connection,
+ *     retryPolicy: {
+ *       maximumAttempts: 5,          // retry up to 5 times
+ *       backoffCoefficient: 2,       // exponential: 2^0, 2^1, 2^2, ... seconds
+ *       maximumInterval: '30s',      // cap delay at 30 seconds
+ *     },
+ *     callback: async (data: StreamData) => {
+ *       const result = await doWork(data.data);
+ *       return {
+ *         code: 200,
+ *         status: StreamStatus.SUCCESS,
+ *         metadata: { ...data.metadata },
+ *         data: { result },
+ *       } as StreamDataResponse;
+ *     },
+ *   }]
+ * });
+ * ```
+ *
+ * ## Execution Model
+ *
+ * Worker is a **Category A (duplex)** activity:
+ * - **Leg 1** (`process`): Maps input data and publishes a message to the
+ *   worker's topic stream.
+ * - **Leg 2** (`processEvent`, inherited): Receives the worker's response,
+ *   maps output data, and executes the step protocol to transition to
+ *   adjacent activities.
+ *
+ * @see {@link WorkerActivity} for the TypeScript interface
+ */
 declare class Worker extends Activity {
     config: WorkerActivity;
     constructor(config: ActivityType, data: ActivityData, metadata: ActivityMetadata, hook: ActivityData | null, engine: EngineService, context?: JobState);