@hotmeshio/hotmesh 0.10.2 → 0.11.0

package/build/services/durable/worker.d.ts

@@ -200,7 +200,7 @@ export declare class WorkerService {
     /**
      * @private
      */
-    initWorkflowWorker(config: WorkerConfig, workflowTopic: string, workflowFunction: Function): Promise<HotMesh>;
+    initWorkflowWorker(config: WorkerConfig, taskQueue: string, workflowFunctionName: string, workflowTopic: string, workflowFunction: Function): Promise<HotMesh>;
     /**
      * @private
      */
@@ -213,7 +213,7 @@ export declare class WorkerService {
     /**
      * @private
      */
-    wrapWorkflowFunction(workflowFunction: Function, workflowTopic: string, config: WorkerConfig): Function;
+    wrapWorkflowFunction(workflowFunction: Function, workflowTopic: string, workflowFunctionName: string, config: WorkerConfig): Function;
     /**
      * @private
      */

package/build/services/durable/worker.js

@@ -345,13 +345,15 @@ class WorkerService {
     static async create(config) {
         const workflow = config.workflow;
         const [workflowFunctionName, workflowFunction] = WorkerService.resolveWorkflowTarget(workflow);
-        const baseTopic = `${config.taskQueue}-${workflowFunctionName}`;
-        const activityTopic = `${baseTopic}-activity`;
-        const workflowTopic = `${baseTopic}`;
+        // Separate taskQueue from workflowName - no concatenation for stream_name
+        const taskQueue = config.taskQueue;
+        const activityTopic = `${taskQueue}-activity`;
+        // workflowTopic remains concatenated for engine-internal routing (graph.subscribes)
+        const workflowTopic = `${taskQueue}-${workflowFunctionName}`;
         //initialize supporting workflows
         const worker = new WorkerService();
         worker.activityRunner = await worker.initActivityWorker(config, activityTopic);
-        worker.workflowRunner = await worker.initWorkflowWorker(config, workflowTopic, workflowFunction);
+        worker.workflowRunner = await worker.initWorkflowWorker(config, taskQueue, workflowFunctionName, workflowTopic, workflowFunction);
         search_1.Search.configureSearchIndex(worker.workflowRunner, config.search);
         await WorkerService.activateWorkflow(worker.workflowRunner);
         return worker;
@@ -470,7 +472,7 @@ class WorkerService {
     /**
      * @private
      */
-    async initWorkflowWorker(config, workflowTopic, workflowFunction) {
+    async initWorkflowWorker(config, taskQueue, workflowFunctionName, workflowTopic, workflowFunction) {
         const providerConfig = config.connection;
         const targetNamespace = config?.namespace ?? factory_1.APP_ID;
         const optionsHash = WorkerService.hashOptions(config?.connection);
@@ -483,9 +485,10 @@ class WorkerService {
             engine: { connection: providerConfig },
             workers: [
                 {
-                    topic: workflowTopic,
+                    topic: taskQueue,
+                    workflowName: workflowFunctionName,
                     connection: providerConfig,
-                    callback: this.wrapWorkflowFunction(workflowFunction, workflowTopic, config).bind(this),
+                    callback: this.wrapWorkflowFunction(workflowFunction, workflowTopic, workflowFunctionName, config).bind(this),
                 },
             ],
         });
@@ -495,7 +498,7 @@ class WorkerService {
     /**
      * @private
      */
-    wrapWorkflowFunction(workflowFunction, workflowTopic, config) {
+    wrapWorkflowFunction(workflowFunction, workflowTopic, workflowFunctionName, config) {
         return async (data) => {
             const counter = { counter: 0 };
             const interruptionRegistry = [];
@@ -532,7 +535,8 @@ class WorkerService {
                     replayQuery = '-*[ehklptydr]-*';
                 }
                 context.set('workflowTopic', workflowTopic);
-                context.set('workflowName', workflowTopic.split('-').pop());
+                context.set('workflowName', workflowFunctionName);
+                context.set('taskQueue', config.taskQueue);
                 context.set('workflowTrace', data.metadata.trc);
                 context.set('workflowSpan', data.metadata.spn);
                 const store = this.workflowRunner.engine.store;
@@ -693,6 +697,8 @@ class WorkerService {
                             workflowDimension: err.workflowDimension,
                             workflowId: err.workflowId,
                             workflowTopic: err.workflowTopic,
+                            taskQueue: err.taskQueue,
+                            workflowName: err.workflowName,
                         },
                     };
                 }

package/build/services/durable/workflow/context.js

@@ -57,6 +57,7 @@ function getContext() {
     const interruptionRegistry = store.get('interruptionRegistry');
     const workflowDimension = store.get('workflowDimension') ?? '';
     const workflowTopic = store.get('workflowTopic');
+    const taskQueue = store.get('taskQueue');
     const connection = store.get('connection');
     const namespace = store.get('namespace');
     const originJobId = store.get('originJobId');
@@ -78,6 +79,7 @@ function getContext() {
         originJobId,
         raw,
         replay,
+        taskQueue,
         workflowId,
         workflowDimension,
         workflowTopic,

package/build/services/durable/workflow/execChild.js

@@ -9,7 +9,7 @@ const didRun_1 = require("./didRun");
  * @private
  */
 function getChildInterruptPayload(context, options, execIndex) {
-    const { workflowId, originJobId, workflowDimension, expire } = context;
+    const { workflowId, originJobId, workflowDimension, expire, taskQueue: parentTaskQueue } = context;
     let childJobId;
     if (options.workflowId) {
         childJobId = options.workflowId;
@@ -21,7 +21,8 @@ function getChildInterruptPayload(context, options, execIndex) {
         childJobId = `-${options.workflowName}-${(0, common_1.guid)()}-${workflowDimension}-${execIndex}`;
     }
     const parentWorkflowId = workflowId;
-    const taskQueueName = options.taskQueue ?? options.entity;
+    // Use explicit taskQueue, or parent's taskQueue, or entity as fallback
+    const taskQueueName = options.taskQueue ?? parentTaskQueue ?? options.entity;
     const workflowName = options.taskQueue
         ? options.workflowName
         : options.entity ?? options.workflowName;
@@ -42,6 +43,8 @@ function getChildInterruptPayload(context, options, execIndex) {
         workflowDimension: workflowDimension,
         workflowId: childJobId,
         workflowTopic,
+        taskQueue: taskQueueName,
+        workflowName: workflowName,
     };
 }
 /**

package/build/services/durable/workflow/hook.js

@@ -109,10 +109,16 @@ async function hook(options) {
                     entity: options.entity,
                 }, null, 2)}`);
         }
+        // Extract taskQueue and workflowName from targetTopic
+        const hookTaskQueue = options.taskQueue ?? options.entity ?? targetTopic.split('-')[0];
+        const hookWorkflowName = options.workflowName ?? options.entity ??
+            (targetTopic.startsWith(`${hookTaskQueue}-`) ? targetTopic.substring(hookTaskQueue.length + 1) : targetTopic);
         const payload = {
             arguments: [...options.args],
             id: targetWorkflowId,
             workflowTopic: targetTopic,
+            taskQueue: hookTaskQueue,
+            workflowName: hookWorkflowName,
             backoffCoefficient: options.config?.backoffCoefficient || common_1.HMSH_DURABLE_EXP_BACKOFF,
             maximumAttempts: options.config?.maximumAttempts || common_1.HMSH_DURABLE_MAX_ATTEMPTS,
             maximumInterval: (0, common_1.s)(options?.config?.maximumInterval ?? common_1.HMSH_DURABLE_MAX_INTERVAL),

package/build/services/durable/workflow/proxyActivities.js

@@ -9,12 +9,11 @@ const didRun_1 = require("./didRun");
  * @private
  */
 function getProxyInterruptPayload(context, activityName, execIndex, args, options) {
-    const { workflowDimension, workflowId, originJobId, workflowTopic, expire } = context;
-    // Use explicitly provided taskQueue, otherwise derive from workflow (original behavior)
-    // This keeps backward compatibility while allowing explicit global/custom queues
+    const { workflowDimension, workflowId, originJobId, taskQueue, expire } = context;
+    // Activity topic uses the task queue directly (no workflow name concatenation)
     const activityTopic = options?.taskQueue
         ? `${options.taskQueue}-activity`
-        : `${workflowTopic}-activity`;
+        : `${taskQueue}-activity`;
     const activityJobId = `-${workflowId}-$${activityName}${workflowDimension}-${execIndex}`;
     let maximumInterval;
     if (options?.retryPolicy?.maximumInterval) {

package/build/services/engine/index.js

@@ -5,6 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.EngineService = void 0;
 const key_1 = require("../../modules/key");
+const registry_1 = require("../stream/registry");
 const enums_1 = require("../../modules/enums");
 const utils_1 = require("../../modules/utils");
 const activities_1 = __importDefault(require("../activities"));
@@ -47,10 +48,11 @@ class EngineService {
             await instance.initSubChannel(config.engine.sub, config.engine.pub ?? config.engine.store);
             await instance.initStreamChannel(config.engine.stream, config.engine.store);
             instance.router = await instance.initRouter(config);
-            const streamName = instance.store.mintKey(key_1.KeyType.STREAMS, {
-                appId: instance.appId,
+            // Use singleton consumer via registry for engine stream
+            await registry_1.StreamConsumerRegistry.registerEngine(namespace, appId, guid, instance.processStreamMessage.bind(instance), instance.stream, instance.store, logger, {
+                reclaimDelay: config.engine.reclaimDelay,
+                reclaimCount: config.engine.reclaimCount,
             });
-            instance.router.consumeMessages(streamName, 'ENGINE', instance.guid, instance.processStreamMessage.bind(instance));
             instance.taskService = new task_1.TaskService(instance.store, logger);
             instance.exporter = new exporter_1.ExporterService(instance.appId, instance.store, logger);
             instance.inited = (0, utils_1.formatISODate)(new Date());

package/build/services/store/index.d.ts

@@ -91,5 +91,45 @@ declare abstract class StoreService<Provider extends ProviderClient, Transaction
     abstract setThrottleRate(options: ThrottleOptions): Promise<void>;
     abstract getThrottleRates(): Promise<StringStringType>;
     abstract getThrottleRate(topic: string): Promise<number>;
+    /**
+     * Fetch activity inputs for a workflow. Used by the exporter to enrich
+     * timeline events with activity arguments.
+     *
+     * @param workflowId - The workflow ID
+     * @param symbolField - The compressed symbol field for activity arguments
+     * @returns Map of job_id -> parsed input arguments and activityName:index -> parsed inputs
+     */
+    getActivityInputs?(workflowId: string, symbolField: string): Promise<{
+        byJobId: Map<string, any>;
+        byNameIndex: Map<string, any>;
+    }>;
+    /**
+     * Fetch child workflow inputs in batch. Used by the exporter to enrich
+     * child workflow events with their arguments.
+     *
+     * @param childJobKeys - Array of child job keys to fetch
+     * @param symbolField - The compressed symbol field for workflow arguments
+     * @returns Map of child_workflow_id -> parsed input arguments
+     */
+    getChildWorkflowInputs?(childJobKeys: string[], symbolField: string): Promise<Map<string, any>>;
+    /**
+     * Fetch job record and attributes by key. Used by the exporter to
+     * reconstruct execution history for expired jobs.
+     *
+     * @param jobKey - The job key (e.g., "hmsh:durable:j:workflowId")
+     * @returns Job row and all attributes
+     */
+    getJobByKeyDirect?(jobKey: string): Promise<{
+        job: {
+            id: string;
+            key: string;
+            status: number;
+            created_at: Date;
+            updated_at: Date;
+            expired_at?: Date;
+            is_live: boolean;
+        };
+        attributes: Record<string, string>;
+    }>;
 }
 export { StoreService };

package/build/services/store/providers/postgres/exporter-sql.d.ts

@@ -0,0 +1,23 @@
+/**
+ * SQL queries for exporter enrichment (Postgres provider).
+ * These queries support the exporter's input enrichment and direct query features.
+ */
+/**
+ * Fetch job record by key.
+ */
+export declare const GET_JOB_BY_KEY = "\n  SELECT id, key, status, created_at, updated_at, expired_at, is_live\n  FROM {schema}.jobs\n  WHERE key = $1\n  LIMIT 1\n";
+/**
+ * Fetch all attributes for a job.
+ */
+export declare const GET_JOB_ATTRIBUTES = "\n  SELECT field, value\n  FROM {schema}.jobs_attributes\n  WHERE job_id = $1\n  ORDER BY field\n";
+/**
+ * Fetch activity inputs for a workflow.
+ * Matches all activity jobs for the given workflow and extracts their input arguments.
+ */
+export declare const GET_ACTIVITY_INPUTS = "\n  SELECT j.key, ja.value\n  FROM {schema}.jobs j\n  JOIN {schema}.jobs_attributes ja ON ja.job_id = j.id\n  WHERE j.key LIKE $1\n    AND ja.field = $2\n";
+/**
+ * Fetch child workflow inputs in batch.
+ * Uses parameterized IN clause for exact-match efficiency.
+ * Note: This query template must be built dynamically with the correct number of placeholders.
+ */
+export declare function buildChildWorkflowInputsQuery(childCount: number, schema: string): string;

package/build/services/store/providers/postgres/exporter-sql.js

@@ -0,0 +1,52 @@
+"use strict";
+/**
+ * SQL queries for exporter enrichment (Postgres provider).
+ * These queries support the exporter's input enrichment and direct query features.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildChildWorkflowInputsQuery = exports.GET_ACTIVITY_INPUTS = exports.GET_JOB_ATTRIBUTES = exports.GET_JOB_BY_KEY = void 0;
+/**
+ * Fetch job record by key.
+ */
+exports.GET_JOB_BY_KEY = `
+  SELECT id, key, status, created_at, updated_at, expired_at, is_live
+  FROM {schema}.jobs
+  WHERE key = $1
+  LIMIT 1
+`;
+/**
+ * Fetch all attributes for a job.
+ */
+exports.GET_JOB_ATTRIBUTES = `
+  SELECT field, value
+  FROM {schema}.jobs_attributes
+  WHERE job_id = $1
+  ORDER BY field
+`;
+/**
+ * Fetch activity inputs for a workflow.
+ * Matches all activity jobs for the given workflow and extracts their input arguments.
+ */
+exports.GET_ACTIVITY_INPUTS = `
+  SELECT j.key, ja.value
+  FROM {schema}.jobs j
+  JOIN {schema}.jobs_attributes ja ON ja.job_id = j.id
+  WHERE j.key LIKE $1
+    AND ja.field = $2
+`;
+/**
+ * Fetch child workflow inputs in batch.
+ * Uses parameterized IN clause for exact-match efficiency.
+ * Note: This query template must be built dynamically with the correct number of placeholders.
+ */
+function buildChildWorkflowInputsQuery(childCount, schema) {
+    const placeholders = Array.from({ length: childCount }, (_, i) => `$${i + 1}`).join(',');
+    return `
+    SELECT j.key, ja.value
+    FROM ${schema}.jobs j
+    JOIN ${schema}.jobs_attributes ja ON ja.job_id = j.id
+    WHERE j.key IN (${placeholders})
+      AND ja.field = $${childCount + 1}
+  `;
+}
+exports.buildChildWorkflowInputsQuery = buildChildWorkflowInputsQuery;

package/build/services/store/providers/postgres/kvtables.js

@@ -221,6 +221,12 @@ const KVTables = (context) => ({
                         await client.query(`
               CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_pruned_at
               ON ${fullTableName} (pruned_at) WHERE pruned_at IS NULL;
+            `);
+                        // Index for paginated entity listing with sort (dashboard entity queries)
+                        await client.query(`
+              CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_entity_created
+              ON ${fullTableName} (entity, created_at DESC)
+              WHERE entity IS NOT NULL;
             `);
                         // Create function to update is_live flag in the schema
                         await client.query(`

package/build/services/store/providers/postgres/postgres.d.ts

@@ -194,5 +194,39 @@ declare class PostgresStoreService extends StoreService<ProviderClient, Provider
      * Enhanced shouldScout that can handle notifications
      */
     shouldScout(): Promise<boolean>;
+    /**
+     * Fetch activity inputs for a workflow. Used by the exporter to enrich
+     * timeline events with activity arguments.
+     */
+    getActivityInputs(workflowId: string, symbolField: string): Promise<{
+        byJobId: Map<string, any>;
+        byNameIndex: Map<string, any>;
+    }>;
+    /**
+     * Fetch child workflow inputs in batch. Used by the exporter to enrich
+     * child workflow events with their arguments.
+     */
+    getChildWorkflowInputs(childJobKeys: string[], symbolField: string): Promise<Map<string, any>>;
+    /**
+     * Fetch job record and attributes by key. Used by the exporter to
+     * reconstruct execution history for expired jobs.
+     */
+    getJobByKeyDirect(jobKey: string): Promise<{
+        job: {
+            id: string;
+            key: string;
+            status: number;
+            created_at: Date;
+            updated_at: Date;
+            expired_at?: Date;
+            is_live: boolean;
+        };
+        attributes: Record<string, string>;
+    }>;
+    /**
+     * Parse a HotMesh-encoded value string.
+     * Values may be prefixed with `/s` (JSON), `/d` (number), `/t` or `/f` (boolean), `/n` (null).
+     */
+    private parseHmshValue;
 }
 export { PostgresStoreService };

package/build/services/store/providers/postgres/postgres.js

@@ -1284,5 +1284,104 @@ class PostgresStoreService extends __1.StoreService {
         }
         return false;
     }
+    // ── Exporter queries ───────────────────────────────────────────────────────
+    /**
+     * Fetch activity inputs for a workflow. Used by the exporter to enrich
+     * timeline events with activity arguments.
+     */
+    async getActivityInputs(workflowId, symbolField) {
+        const { GET_ACTIVITY_INPUTS } = await Promise.resolve().then(() => __importStar(require('./exporter-sql')));
+        const schemaName = this.kvsql().safeName(this.appId);
+        const sql = GET_ACTIVITY_INPUTS.replace(/{schema}/g, schemaName);
+        const jobKeyPattern = `hmsh:${this.appId}:j:-${workflowId}-%`;
+        const result = await this.pgClient.query(sql, [jobKeyPattern, symbolField]);
+        const byJobId = new Map();
+        const byNameIndex = new Map();
+        for (const row of result.rows) {
+            const jobKey = row.key;
+            const jobId = jobKey.replace(`hmsh:${this.appId}:j:`, '');
+            try {
+                const parsed = this.parseHmshValue(row.value);
+                byJobId.set(jobId, parsed);
+                // Extract activityName and executionIndex from job_id
+                // Format: -workflowId-$activityName-executionIndex
+                const match = jobId.match(/\$([^-]+)-(\d+)$/);
+                if (match) {
+                    const activityName = match[1];
+                    const executionIndex = match[2];
+                    byNameIndex.set(`${activityName}:${executionIndex}`, parsed);
+                }
+            }
+            catch {
+                // Skip unparseable values
+            }
+        }
+        return { byJobId, byNameIndex };
+    }
+    /**
+     * Fetch child workflow inputs in batch. Used by the exporter to enrich
+     * child workflow events with their arguments.
+     */
+    async getChildWorkflowInputs(childJobKeys, symbolField) {
+        if (childJobKeys.length === 0) {
+            return new Map();
+        }
+        const { buildChildWorkflowInputsQuery } = await Promise.resolve().then(() => __importStar(require('./exporter-sql')));
+        const schemaName = this.kvsql().safeName(this.appId);
+        const sql = buildChildWorkflowInputsQuery(childJobKeys.length, schemaName);
+        const result = await this.pgClient.query(sql, [...childJobKeys, symbolField]);
+        const childInputMap = new Map();
+        for (const row of result.rows) {
+            const jobKey = row.key;
+            const childId = jobKey.replace(`hmsh:${this.appId}:j:`, '');
+            try {
+                const parsed = this.parseHmshValue(row.value);
+                childInputMap.set(childId, parsed);
+            }
+            catch {
+                // Skip unparseable values
+            }
+        }
+        return childInputMap;
+    }
+    /**
+     * Fetch job record and attributes by key. Used by the exporter to
+     * reconstruct execution history for expired jobs.
+     */
+    async getJobByKeyDirect(jobKey) {
+        const { GET_JOB_BY_KEY, GET_JOB_ATTRIBUTES } = await Promise.resolve().then(() => __importStar(require('./exporter-sql')));
+        const schemaName = this.kvsql().safeName(this.appId);
+        const jobSql = GET_JOB_BY_KEY.replace(/{schema}/g, schemaName);
+        const jobResult = await this.pgClient.query(jobSql, [jobKey]);
+        if (jobResult.rows.length === 0) {
+            throw new Error(`No job found for key "${jobKey}"`);
+        }
+        const job = jobResult.rows[0];
+        const attrSql = GET_JOB_ATTRIBUTES.replace(/{schema}/g, schemaName);
+        const attrResult = await this.pgClient.query(attrSql, [job.id]);
+        const attributes = {};
+        for (const row of attrResult.rows) {
+            attributes[row.field] = row.value;
+        }
+        return { job, attributes };
+    }
+    /**
+     * Parse a HotMesh-encoded value string.
+     * Values may be prefixed with `/s` (JSON), `/d` (number), `/t` or `/f` (boolean), `/n` (null).
+     */
+    parseHmshValue(raw) {
+        if (typeof raw !== 'string')
+            return undefined;
+        const prefix = raw.slice(0, 2);
+        const rest = raw.slice(2);
+        switch (prefix) {
+            case '/t': return true;
+            case '/f': return false;
+            case '/d': return Number(rest);
+            case '/n': return null;
+            case '/s': return JSON.parse(rest);
+            default: return raw;
+        }
+    }
 }
 exports.PostgresStoreService = PostgresStoreService;

package/build/services/stream/providers/postgres/kvtables.d.ts

@@ -1,3 +1,3 @@
 import { ILogger } from '../../../logger';
 export declare function deploySchema(streamClient: any, appId: string, logger: ILogger): Promise<void>;
-export declare function getNotificationChannelName(streamName: string, groupName: string): string;
+export declare function getNotificationChannelName(streamName: string, isEngine: boolean): string;