@hotmeshio/hotmesh 0.10.2 → 0.12.0

package/build/services/stream/registry.js

@@ -0,0 +1,198 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StreamConsumerRegistry = void 0;
+const router_1 = require("../router");
+const stream_1 = require("../../types/stream");
+const key_1 = require("../../modules/key");
+/**
+ * Process-wide singleton registry that manages one consumer per task queue (workers)
+ * and one per appId (engines). Instead of N consumers each polling independently,
+ * one consumer fetches batches from the stream and dispatches to registered callbacks
+ * based on the `workflow_name` column (workers) or round-robin (engines).
+ */
+class StreamConsumerRegistry {
+    /**
+     * Register a worker callback for a (taskQueue, workflowName) pair.
+     * If no consumer exists for this taskQueue, a singleton Router is created.
+     */
+    static async registerWorker(namespace, appId, guid, taskQueue, workflowName, callback, stream, store, logger, config) {
+        const key = `${namespace}:${appId}:worker:${taskQueue}`;
+        let entry = StreamConsumerRegistry.workerConsumers.get(key);
+        if (!entry) {
+            // Create the singleton consumer for this task queue
+            const throttle = await store.getThrottleRate(taskQueue);
+            const router = new router_1.Router({
+                namespace,
+                appId,
+                guid,
+                role: stream_1.StreamRole.WORKER,
+                topic: taskQueue,
+                reclaimDelay: config?.reclaimDelay,
+                reclaimCount: config?.reclaimCount,
+                throttle,
+                retryPolicy: config?.retryPolicy,
+            }, stream, logger);
+            entry = {
+                router,
+                callbacks: new Map(),
+                stream,
+                logger,
+            };
+            StreamConsumerRegistry.workerConsumers.set(key, entry);
+            // Create the dispatch callback that routes by workflow_name
+            const dispatchCallback = StreamConsumerRegistry.createWorkerDispatcher(key);
+            // Start consuming from the task queue stream
+            const streamKey = stream.mintKey(key_1.KeyType.STREAMS, {
+                appId,
+                topic: taskQueue,
+            });
+            router.consumeMessages(streamKey, 'WORKER', guid, dispatchCallback);
+        }
+        // Register the callback for this workflow name
+        entry.callbacks.set(workflowName, callback);
+        logger.info('stream-consumer-registry-worker-registered', {
+            taskQueue,
+            workflowName,
+            totalCallbacks: entry.callbacks.size,
+        });
+    }
+    /**
+     * Register an engine callback for an appId.
+     * If no consumer exists for this appId, a singleton Router is created.
+     */
+    static async registerEngine(namespace, appId, guid, callback, stream, store, logger, config) {
+        const key = `${namespace}:${appId}:engine`;
+        let entry = StreamConsumerRegistry.engineConsumers.get(key);
+        if (!entry) {
+            const throttle = await store.getThrottleRate();
+            const router = new router_1.Router({
+                namespace,
+                appId,
+                guid,
+                role: stream_1.StreamRole.ENGINE,
+                reclaimDelay: config?.reclaimDelay,
+                reclaimCount: config?.reclaimCount,
+                throttle,
+            }, stream, logger);
+            entry = {
+                router,
+                callbacks: [],
+                stream,
+                logger,
+            };
+            StreamConsumerRegistry.engineConsumers.set(key, entry);
+            // Create the dispatch callback
+            const dispatchCallback = StreamConsumerRegistry.createEngineDispatcher(key);
+            // Start consuming from the engine stream
+            const streamKey = stream.mintKey(key_1.KeyType.STREAMS, { appId });
+            router.consumeMessages(streamKey, 'ENGINE', guid, dispatchCallback);
+        }
+        entry.callbacks.push(callback);
+        logger.info('stream-consumer-registry-engine-registered', {
+            appId,
+            totalCallbacks: entry.callbacks.length,
+        });
+    }
+    /**
+     * Creates a dispatch callback for worker consumers.
+     * Routes messages to the registered callback based on metadata.wfn (workflow_name).
+     */
+    static createWorkerDispatcher(key) {
+        return async (data) => {
+            const entry = StreamConsumerRegistry.workerConsumers.get(key);
+            if (!entry)
+                return;
+            const wfn = data.metadata?.wfn;
+            if (!wfn) {
+                entry.logger.warn('stream-consumer-registry-no-wfn', {
+                    key,
+                    metadata: data.metadata,
+                });
+                // Fall back to first registered callback if only one exists
+                if (entry.callbacks.size === 1) {
+                    const [, callback] = entry.callbacks.entries().next().value;
+                    return callback(data);
+                }
+                return;
+            }
+            const callback = entry.callbacks.get(wfn);
+            if (!callback) {
+                entry.logger.debug('stream-consumer-registry-no-callback', {
+                    key,
+                    wfn,
+                    registered: [...entry.callbacks.keys()],
+                });
+                // Worker not registered yet. Re-publish with short visibility delay
+                // so it retries after the worker has time to register.
+                // This avoids consuming the error handler's retry budget.
+                const replayData = { ...data };
+                replayData._visibilityDelayMs = 500;
+                const streamKey = entry.stream.mintKey(key_1.KeyType.STREAMS, {
+                    topic: data.metadata?.topic,
+                });
+                await entry.stream.publishMessages(streamKey, [JSON.stringify(replayData)]);
+                // Return void — the original message will be ack'd,
+                // but a new copy is queued with a delay.
+                return;
+            }
+            return callback(data);
+        };
+    }
+    /**
+     * Creates a dispatch callback for engine consumers.
+     * Engines are generic processors — the first registered callback handles the message.
+     */
+    static createEngineDispatcher(key) {
+        return async (data) => {
+            const entry = StreamConsumerRegistry.engineConsumers.get(key);
+            if (!entry || entry.callbacks.length === 0)
+                return;
+            // Engine callbacks are all equivalent — use the first one
+            return entry.callbacks[0](data);
+        };
+    }
+    /**
+     * Unregister a worker callback.
+     */
+    static async unregisterWorker(namespace, appId, taskQueue, workflowName) {
+        const key = `${namespace}:${appId}:worker:${taskQueue}`;
+        const entry = StreamConsumerRegistry.workerConsumers.get(key);
+        if (!entry)
+            return;
+        entry.callbacks.delete(workflowName);
+        if (entry.callbacks.size === 0) {
+            await entry.router.stopConsuming();
+            StreamConsumerRegistry.workerConsumers.delete(key);
+        }
+    }
+    /**
+     * Unregister an engine callback.
+     */
+    static async unregisterEngine(namespace, appId, callback) {
+        const key = `${namespace}:${appId}:engine`;
+        const entry = StreamConsumerRegistry.engineConsumers.get(key);
+        if (!entry)
+            return;
+        entry.callbacks = entry.callbacks.filter(cb => cb !== callback);
+        if (entry.callbacks.length === 0) {
+            await entry.router.stopConsuming();
+            StreamConsumerRegistry.engineConsumers.delete(key);
+        }
+    }
+    /**
+     * Stop all consumers and clear the registry.
+     */
+    static async shutdown() {
+        for (const [, entry] of StreamConsumerRegistry.workerConsumers) {
+            await entry.router.stopConsuming();
+        }
+        for (const [, entry] of StreamConsumerRegistry.engineConsumers) {
+            await entry.router.stopConsuming();
+        }
+        StreamConsumerRegistry.workerConsumers.clear();
+        StreamConsumerRegistry.engineConsumers.clear();
+    }
+}
+exports.StreamConsumerRegistry = StreamConsumerRegistry;
+StreamConsumerRegistry.workerConsumers = new Map();
+StreamConsumerRegistry.engineConsumers = new Map();

package/build/services/worker/index.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.WorkerService = void 0;
 const key_1 = require("../../modules/key");
+const registry_1 = require("../stream/registry");
 const utils_1 = require("../../modules/utils");
 const factory_1 = require("../connector/factory");
 const router_1 = require("../router");
@@ -45,12 +46,25 @@ class WorkerService {
                 await service.subscribe.subscribe(key_1.KeyType.QUORUM, service.subscriptionHandler(), appId, service.topic);
                 await service.subscribe.subscribe(key_1.KeyType.QUORUM, service.subscriptionHandler(), appId, service.guid);
                 await service.initStreamChannel(service, worker.stream, worker.store);
-                service.router = await service.initRouter(worker, logger);
-                const key = service.store.mintKey(key_1.KeyType.STREAMS, {
-                    appId: service.appId,
-                    topic: worker.topic,
-                });
-                await service.router.consumeMessages(key, 'WORKER', service.guid, worker.callback);
+                if (worker.workflowName) {
+                    // Use singleton consumer via registry (batched fetch + dispatch by workflow_name)
+                    await registry_1.StreamConsumerRegistry.registerWorker(namespace, appId, guid, worker.topic, worker.workflowName, worker.callback, service.stream, service.store, logger, {
+                        reclaimDelay: worker.reclaimDelay,
+                        reclaimCount: worker.reclaimCount,
+                        retryPolicy: worker.retryPolicy,
+                    });
+                    // Still need a router for publishing responses back to engine
+                    service.router = await service.initRouter(worker, logger);
+                }
+                else {
+                    // Legacy: own Router per worker (no workflowName = no singleton dispatch)
+                    service.router = await service.initRouter(worker, logger);
+                    const key = service.store.mintKey(key_1.KeyType.STREAMS, {
+                        appId: service.appId,
+                        topic: worker.topic,
+                    });
+                    await service.router.consumeMessages(key, 'WORKER', service.guid, worker.callback);
+                }
                 service.inited = (0, utils_1.formatISODate)(new Date());
                 services.push(service);
             }

package/build/types/durable.d.ts

@@ -75,7 +75,12 @@ type WorkflowContext = {
      */
     workflowDimension: string;
     /**
-     * a concatenation of the task queue and workflow name (e.g., `${taskQueueName}-${workflowName}`)
+     * the task queue name (stream_name in worker_streams)
+     */
+    taskQueue: string;
+    /**
+     * a concatenation of the task queue and workflow name (e.g., `${taskQueueName}-${workflowName}`);
+     * used for engine-internal routing (graph.subscribes)
      */
     workflowTopic: string;
     /**

package/build/types/error.d.ts

@@ -14,6 +14,8 @@ export type DurableChildErrorType = {
     workflowDimension: string;
     workflowId: string;
     workflowTopic: string;
+    taskQueue?: string;
+    workflowName?: string;
 };
 export type DurableWaitForAllErrorType = {
     items: string[];

package/build/types/exporter.d.ts

@@ -18,6 +18,13 @@ export interface ExportOptions {
      * @default true
      */
     values?: boolean;
+    /**
+     * When true, fetches stream message history and produces a structured
+     * `activities` array with input/output per activity, timing, dimensional
+     * cycle info, and retry attempts. This is the dashboard-friendly format.
+     * @default false
+     */
+    enrich_inputs?: boolean;
 }
 export type JobAction = {
     cursor: number;
@@ -74,10 +81,24 @@ export interface DurableJobExport {
     timeline?: TimelineType[];
     transitions?: TransitionType[];
 }
+export interface ActivityDetail {
+    name: string;
+    type: string;
+    dimension: string;
+    input?: Record<string, any>;
+    output?: Record<string, any>;
+    started_at?: string;
+    completed_at?: string;
+    duration_ms?: number;
+    retry_attempt?: number;
+    cycle_iteration?: number;
+    error?: string | null;
+}
 export interface JobExport {
     dependencies: DependencyExport[];
     process: StringAnyType;
     status: string;
+    activities?: ActivityDetail[];
 }
 export type ExportMode = 'sparse' | 'verbose';
 export type WorkflowEventType = 'workflow_execution_started' | 'workflow_execution_completed' | 'workflow_execution_failed' | 'activity_task_scheduled' | 'activity_task_completed' | 'activity_task_failed' | 'child_workflow_execution_started' | 'child_workflow_execution_completed' | 'child_workflow_execution_failed' | 'timer_started' | 'timer_fired' | 'workflow_execution_signaled';
@@ -101,6 +122,7 @@ export interface ActivityTaskScheduledAttributes {
     activity_type: string;
     timeline_key: string;
     execution_index: number;
+    input?: any;
 }
 export interface ActivityTaskCompletedAttributes {
     kind: 'activity_task_completed';
@@ -109,6 +131,7 @@ export interface ActivityTaskCompletedAttributes {
     scheduled_event_id?: number;
     timeline_key: string;
     execution_index: number;
+    input?: any;
 }
 export interface ActivityTaskFailedAttributes {
     kind: 'activity_task_failed';
@@ -117,6 +140,7 @@ export interface ActivityTaskFailedAttributes {
     scheduled_event_id?: number;
     timeline_key: string;
     execution_index: number;
+    input?: any;
 }
 export interface ChildWorkflowExecutionStartedAttributes {
     kind: 'child_workflow_execution_started';
@@ -124,6 +148,7 @@ export interface ChildWorkflowExecutionStartedAttributes {
     awaited: boolean;
     timeline_key: string;
     execution_index: number;
+    input?: any;
 }
 export interface ChildWorkflowExecutionCompletedAttributes {
     kind: 'child_workflow_execution_completed';
@@ -199,10 +224,69 @@ export interface WorkflowExecution {
     events: WorkflowExecutionEvent[];
     summary: WorkflowExecutionSummary;
     children?: WorkflowExecution[];
+    stream_history?: StreamHistoryEntry[];
 }
 export interface ExecutionExportOptions {
     mode?: ExportMode;
     exclude_system?: boolean;
     omit_results?: boolean;
     max_depth?: number;
+    /**
+     * When true, enriches activity and child workflow events with their inputs
+     * by querying the underlying job attributes. This enables full visibility
+     * into activity arguments without requiring separate callback queries.
+     *
+     * @default false (late-binding: only return timeline_key references)
+     */
+    enrich_inputs?: boolean;
+    /**
+     * When true, allows fallback to direct database queries for expired jobs
+     * whose in-memory handles have been pruned. Only supported with providers
+     * that implement the extended exporter query interface (e.g., Postgres).
+     *
+     * @default false
+     */
+    allow_direct_query?: boolean;
+    /**
+     * When true, fetches the full stream message history for this workflow
+     * from the worker_streams table and attaches it as `stream_history`.
+     * This provides raw activity input/output data from the original stream
+     * messages, enabling Temporal-grade export fidelity.
+     *
+     * @default false
+     */
+    include_stream_history?: boolean;
+}
+export interface StreamHistoryEntry {
+    id: number;
+    jid: string;
+    aid: string;
+    dad: string;
+    msg_type: string;
+    topic: string;
+    workflow_name: string;
+    data: Record<string, any>;
+    status?: string;
+    code?: number;
+    created_at: string;
+    expired_at?: string;
+}
+export interface JobAttributesRow {
+    field: string;
+    value: string;
+}
+export interface JobRow {
+    id: string;
+    key: string;
+    status: number;
+    created_at: Date;
+    updated_at: Date;
+    expired_at?: Date;
+    is_live: boolean;
+}
+export interface ActivityInputMap {
+    /** Maps activity job_id to parsed input arguments */
+    byJobId: Map<string, any>;
+    /** Maps "activityName:executionIndex" to parsed input arguments */
+    byNameIndex: Map<string, any>;
 }

package/build/types/hotmesh.d.ts

@@ -134,9 +134,15 @@ type HotMeshEngine = {
 };
 type HotMeshWorker = {
     /**
-     * the topic that the worker subscribes to
+     * the topic/task queue that the worker subscribes to (stream_name)
      */
     topic: string;
+    /**
+     * the workflow function name for dispatch routing (workflow_name column).
+     * When set, workers sharing the same topic use a singleton consumer
+     * that fetches batches and dispatches by workflowName.
+     */
+    workflowName?: string;
     /**
      * set by hotmesh once the connnector service instances the provider
      * AND if the provider requires a separate channel for publishing

package/build/types/index.d.ts

@@ -6,7 +6,7 @@ export { CollationFaultType, CollationStage } from './collator';
 export { ActivityConfig, ActivityInterceptor, ActivityInterceptorContext, ActivityWorkflowDataType, ChildResponseType, ClientConfig, ClientWorkflow, ContextType, Connection, ProxyResponseType, ProxyType, Registry, SignalOptions, FindJobsOptions, FindOptions, FindWhereOptions, FindWhereQuery, HookOptions, SearchResults, WorkflowConfig, WorkerConfig, WorkerOptions, WorkflowContext, WorkflowSearchOptions, WorkflowSearchSchema, WorkflowDataType, WorkflowOptions, WorkflowInterceptor, InterceptorRegistry, } from './durable';
 export { PruneOptions, PruneResult, } from './dba';
 export { DurableChildErrorType, DurableProxyErrorType, DurableSleepErrorType, DurableWaitForAllErrorType, DurableWaitForErrorType, } from './error';
-export { ActivityAction, DependencyExport, DurableJobExport, ExecutionExportOptions, ExportCycles, ExportItem, ExportMode, ExportOptions, ExportTransitions, JobAction, JobExport, JobActionExport, JobTimeline, WorkflowEventAttributes, WorkflowEventCategory, WorkflowEventType, WorkflowExecution, WorkflowExecutionEvent, WorkflowExecutionStatus, WorkflowExecutionSummary, } from './exporter';
+export { ActivityAction, ActivityDetail, ActivityInputMap, ActivityTaskCompletedAttributes, ActivityTaskFailedAttributes, ActivityTaskScheduledAttributes, ChildWorkflowExecutionCompletedAttributes, ChildWorkflowExecutionFailedAttributes, ChildWorkflowExecutionStartedAttributes, DependencyExport, DurableJobExport, ExecutionExportOptions, ExportCycles, ExportFields, ExportItem, ExportMode, ExportOptions, ExportTransitions, JobAction, JobActionExport, JobAttributesRow, JobExport, JobRow, JobTimeline, StreamHistoryEntry, TimelineType, TimerFiredAttributes, TimerStartedAttributes, TransitionType, WorkflowEventAttributes, WorkflowEventCategory, WorkflowEventType, WorkflowExecution, WorkflowExecutionCompletedAttributes, WorkflowExecutionEvent, WorkflowExecutionFailedAttributes, WorkflowExecutionSignaledAttributes, WorkflowExecutionStartedAttributes, WorkflowExecutionStatus, WorkflowExecutionSummary, } from './exporter';
 export { HookCondition, HookConditions, HookGate, HookInterface, HookRule, HookRules, HookSignal, } from './hook';
 export { HotMesh, HotMeshEngine, HotMeshWorker, HotMeshSettings, HotMeshApp, HotMeshApps, HotMeshConfig, HotMeshManifest, HotMeshGraph, KeyType, KeyStoreParams, ScoutType, } from './hotmesh';
 export { ILogger, LogLevel } from './logger';

package/build/types/stream.d.ts

@@ -105,6 +105,8 @@ export interface StreamData {
         spn?: string;
         /** Current try count, used for retry logic */
         try?: number;
+        /** Workflow name (graph subscribes topic) for worker stream routing */
+        wfn?: string;
         /**
          * Indicates if the message should wait for a response.
          * If explicitly false, the connection is severed immediately

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.10.2",
+  "version": "0.12.0",
   "description": "Permanent-Memory Workflows & AI Agents",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",
@@ -46,7 +46,7 @@
     "test:durable:sleep": "vitest run tests/durable/sleep/postgres.test.ts",
     "test:durable:signal": "vitest run tests/durable/signal/postgres.test.ts",
     "test:durable:unknown": "vitest run tests/durable/unknown/postgres.test.ts",
-    "test:durable:exporter": "vitest run tests/durable/exporter/exporter.test.ts",
+    "test:durable:exporter": "HMSH_LOGLEVEL=info vitest run tests/durable/exporter",
     "test:durable:exporter:debug": "EXPORT_DEBUG=1 HMSH_LOGLEVEL=error vitest run tests/durable/basic/postgres.test.ts",
     "test:dba": "vitest run tests/dba",
     "test:cycle": "vitest run tests/functional/cycle",