@hotmeshio/hotmesh 0.13.0 → 0.14.0

package/build/services/router/consumption/index.js

@@ -7,9 +7,16 @@ const config_1 = require("../config");
 const stream_1 = require("../../../types/stream");
 const key_1 = require("../../../modules/key");
 class ConsumptionManager {
-    constructor(stream, logger, throttleManager, errorHandler, lifecycleManager, reclaimDelay, reclaimCount, appId, role, router, retryPolicy) {
-        this.errorCount = 0;
-        this.counts = {};
+    /**
+     * Consumption stats are written directly to the parent Router so
+     * they are visible in quorum rollcall profiles.
+     */
+    get errorCount() { return this.router.errorCount; }
+    set errorCount(v) { this.router.errorCount = v; }
+    get counts() { return this.router.counts; }
+    get hasReachedMaxBackoff() { return this.router.hasReachedMaxBackoff; }
+    set hasReachedMaxBackoff(v) { this.router.hasReachedMaxBackoff = v; }
+    constructor(stream, logger, throttleManager, errorHandler, lifecycleManager, reclaimDelay, reclaimCount, appId, role, router, retry) {
         this.stream = stream;
         this.logger = logger;
         this.throttleManager = throttleManager;
@@ -20,7 +27,7 @@ class ConsumptionManager {
         this.appId = appId;
         this.role = role;
         this.router = router;
-        this.retryPolicy = retryPolicy;
+        this.retry = retry;
     }
     async createGroup(stream, group) {
         try {
@@ -107,7 +114,7 @@ class ConsumptionManager {
             }
             // Process messages - use parallel processing for PostgreSQL
             const features = this.stream.getProviderSpecificFeatures();
-            const isPostgres = features.supportsNotifications; // Only PostgreSQL supports notifications currently
+            const isPostgres = features.supportsParallelProcessing;
             if (isPostgres && messages.length > 1) {
                 // Parallel processing for PostgreSQL batches
                 this.logger.debug('postgres-stream-parallel-processing', {
@@ -221,7 +228,7 @@ class ConsumptionManager {
                 if (!this.hasReachedMaxBackoff) {
                     // Normal mode: try with backoff and finite retries
                     const features = this.stream.getProviderSpecificFeatures();
-                    const isPostgres = features.supportsNotifications; // Only PostgreSQL supports notifications
+                    const isPostgres = features.supportsParallelProcessing;
                     const batchSize = isPostgres ? 10 : 1; // Use batch size of 10 for PostgreSQL, 1 for others
                     messages = await this.stream.consumeMessages(stream, group, consumer, {
                         blockTimeout: streamDuration,
@@ -235,7 +242,7 @@ class ConsumptionManager {
                 else {
                     // Fallback mode: just try once, no backoff
                     const features = this.stream.getProviderSpecificFeatures();
-                    const isPostgres = features.supportsNotifications; // Only PostgreSQL supports notifications
+                    const isPostgres = features.supportsParallelProcessing;
                     const batchSize = isPostgres ? 10 : 1; // Use batch size of 10 for PostgreSQL, 1 for others
                     messages = await this.stream.consumeMessages(stream, group, consumer, {
                         blockTimeout: streamDuration,
@@ -256,7 +263,7 @@ class ConsumptionManager {
                     this.hasReachedMaxBackoff = false;
                     // Process messages - use parallel processing for PostgreSQL
                     const features = this.stream.getProviderSpecificFeatures();
-                    const isPostgres = features.supportsNotifications; // Only PostgreSQL supports notifications currently
+                    const isPostgres = features.supportsParallelProcessing;
                     if (isPostgres && messages.length > 1) {
                         // Parallel processing for PostgreSQL batches
                         this.logger.debug('postgres-stream-parallel-processing-polling', {
@@ -360,13 +367,13 @@ class ConsumptionManager {
         // the normal retry mechanism (ErrorHandler.handleRetry / shouldRetry).
         //
         // Normal retry flow: handleRetry() checks metadata.try against the
-        // configured retryPolicy.maximumAttempts (or _streamRetryConfig) and
+        // configured retry.maximumAttempts (or _streamRetryConfig) and
         // applies exponential backoff + visibility delays. That mechanism is
         // the primary retry budget and is what developers configure via
-        // HotMesh.init({ workers: [{ retryPolicy: { maximumAttempts, ... } }] }).
+        // HotMesh.init({ workers: [{ retry: { maximumAttempts, ... } }] }).
         //
         // This check catches messages that have somehow exceeded the normal
-        // budget — e.g., when no retryPolicy is configured, when the retry
+        // budget — e.g., when no retry is configured, when the retry
         // logic is bypassed by an infrastructure error, or when a message
         // re-enters the stream through a path that doesn't increment
         // metadata.try. The threshold is the HIGHER of the configured retry
@@ -374,7 +381,7 @@ class ConsumptionManager {
         // never interferes with legitimate developer-configured retries.
         const retryAttempt = input._retryAttempt || 0;
         const configuredMax = input._streamRetryConfig?.max_retry_attempts
-            ?? this.retryPolicy?.maximumAttempts
+            ?? this.retry?.maximumAttempts
             ?? 0;
         const poisonThreshold = Math.max(configuredMax, config_1.HMSH_POISON_MESSAGE_THRESHOLD);
         if (retryAttempt >= poisonThreshold) {
@@ -476,7 +483,7 @@ class ConsumptionManager {
         if (ids.length === 0)
             return;
         const features = this.stream.getProviderSpecificFeatures();
-        const isPostgres = features.supportsNotifications; // Only PostgreSQL supports notifications
+        const isPostgres = features.supportsParallelProcessing;
         if (isPostgres && ids.length > 1) {
             // Batch acknowledgment for PostgreSQL
             await this.stream.ackAndDelete(stream, group, ids);
@@ -493,15 +500,15 @@ class ConsumptionManager {
             if (output.status === 'error') {
                 // Extract retry policy with priority:
                 // 1. Use message-level _streamRetryConfig (from database columns or previous retry)
-                // 2. Fall back to router-level retryPolicy (from worker config)
-                const retryPolicy = input._streamRetryConfig
+                // 2. Fall back to router-level retry (from worker config)
+                const retry = input._streamRetryConfig
                     ? {
                         maximumAttempts: input._streamRetryConfig.max_retry_attempts,
                         backoffCoefficient: input._streamRetryConfig.backoff_coefficient,
                         maximumInterval: input._streamRetryConfig.maximum_interval_seconds,
                     }
-                    : this.retryPolicy;
-                return await this.errorHandler.handleRetry(input, output, this.publishMessage.bind(this), retryPolicy);
+                    : this.retry;
+                return await this.errorHandler.handleRetry(input, output, this.publishMessage.bind(this), retry);
             }
             else if (typeof output.metadata !== 'object') {
                 output.metadata = { ...input.metadata, guid: (0, utils_1.guid)() };

package/build/services/router/error-handling/index.d.ts

@@ -1,8 +1,8 @@
 import { StreamData, StreamDataResponse, RetryPolicy } from '../../../types/stream';
 export declare class ErrorHandler {
-    shouldRetry(input: StreamData, output: StreamDataResponse, retryPolicy?: RetryPolicy): [boolean, number];
+    shouldRetry(input: StreamData, output: StreamDataResponse, retry?: RetryPolicy): [boolean, number];
     structureUnhandledError(input: StreamData, err: Error): StreamDataResponse;
     structureUnacknowledgedError(input: StreamData): StreamDataResponse;
     structureError(input: StreamData, output: StreamDataResponse): StreamDataResponse;
-    handleRetry(input: StreamData, output: StreamDataResponse, publishMessage: (topic: string, streamData: StreamData | StreamDataResponse) => Promise<string>, retryPolicy?: RetryPolicy): Promise<string>;
+    handleRetry(input: StreamData, output: StreamDataResponse, publishMessage: (topic: string, streamData: StreamData | StreamDataResponse) => Promise<string>, retry?: RetryPolicy): Promise<string>;
 }

package/build/services/router/error-handling/index.js

@@ -5,15 +5,15 @@ const utils_1 = require("../../../modules/utils");
 const config_1 = require("../config");
 const stream_1 = require("../../../types/stream");
 class ErrorHandler {
-    shouldRetry(input, output, retryPolicy) {
+    shouldRetry(input, output, retry) {
         const tryCount = input.metadata.try || 0;
         // Priority 1: Use structured retry policy (from stream columns or config)
-        if (retryPolicy) {
-            const maxAttempts = retryPolicy.maximumAttempts || 3;
-            const backoffCoeff = retryPolicy.backoffCoefficient || 10;
-            const maxInterval = typeof retryPolicy.maximumInterval === 'string'
-                ? parseInt(retryPolicy.maximumInterval)
-                : (retryPolicy.maximumInterval || 120);
+        if (retry) {
+            const maxAttempts = retry.maximumAttempts || 3;
+            const backoffCoeff = retry.backoffCoefficient || 10;
+            const maxInterval = typeof retry.maximumInterval === 'string'
+                ? parseInt(retry.maximumInterval)
+                : (retry.maximumInterval || 120);
             // Check if we can retry (next attempt would be attempt #tryCount+2, must be <= maxAttempts)
             // tryCount=0 is 1st attempt, tryCount=1 is 2nd attempt, etc.
             // So after tryCount, we've made (tryCount + 1) attempts
@@ -97,12 +97,12 @@ class ErrorHandler {
             data,
         };
     }
-    async handleRetry(input, output, publishMessage, retryPolicy) {
-        const [shouldRetry, timeout] = this.shouldRetry(input, output, retryPolicy);
+    async handleRetry(input, output, publishMessage, retry) {
+        const [shouldRetry, timeout] = this.shouldRetry(input, output, retry);
         if (shouldRetry) {
-            // Only sleep if no retryPolicy (legacy behavior for backward compatibility)
-            // With retryPolicy, use visibility timeout instead of in-memory sleep
-            if (!retryPolicy) {
+            // Only sleep if no retry (legacy behavior for backward compatibility)
+            // With retry, use visibility timeout instead of in-memory sleep
+            if (!retry) {
                 await (0, utils_1.sleepFor)(timeout);
             }
             // Create new message with incremented try count
@@ -115,8 +115,8 @@ class ErrorHandler {
             if (input._streamRetryConfig) {
                 newMessage._streamRetryConfig = input._streamRetryConfig;
             }
-            // Add visibility delay for production-ready retry with retryPolicy
-            if (retryPolicy && timeout > 0) {
+            // Add visibility delay for production-ready retry with retry
+            if (retry && timeout > 0) {
                 newMessage._visibilityDelayMs = timeout;
             }
             // Track retry attempt count in database

package/build/services/router/index.d.ts

@@ -13,7 +13,7 @@ declare class Router<S extends StreamService<ProviderClient, ProviderTransaction
     reclaimCount: number;
     logger: ILogger;
     readonly: boolean;
-    retryPolicy: import('../../types/stream').RetryPolicy | undefined;
+    retry: import('../../types/stream').RetryPolicy | undefined;
     errorCount: number;
     counts: {
         [key: string]: number;

package/build/services/router/index.js

@@ -29,12 +29,12 @@ class Router {
         this.reclaimCount = enhancedConfig.reclaimCount;
         this.logger = logger;
         this.readonly = enhancedConfig.readonly;
-        this.retryPolicy = enhancedConfig.retryPolicy;
+        this.retry = enhancedConfig.retry;
         // Initialize submodule managers
         this.throttleManager = new throttling_1.ThrottleManager(enhancedConfig.throttle);
         this.errorHandler = new error_handling_1.ErrorHandler();
         this.lifecycleManager = new lifecycle_1.LifecycleManager(this.readonly, this.topic, this.logger, this.stream);
-        this.consumptionManager = new consumption_1.ConsumptionManager(this.stream, this.logger, this.throttleManager, this.errorHandler, this.lifecycleManager, this.reclaimDelay, this.reclaimCount, this.appId, this.role, this, this.retryPolicy);
+        this.consumptionManager = new consumption_1.ConsumptionManager(this.stream, this.logger, this.throttleManager, this.errorHandler, this.lifecycleManager, this.reclaimDelay, this.reclaimCount, this.appId, this.role, this, this.retry);
         this.resetThrottleState();
     }
     // Legacy compatibility methods

package/build/services/serializer/index.d.ts

@@ -1,4 +1,5 @@
 import { Consumes } from '../../types/activity';
+import { PayloadCodec } from '../../types/codec';
 import { StringStringType, StringAnyType, SymbolMap, SymbolMaps, SymbolSets, Symbols } from '../../types/serializer';
 export declare const MDATA_SYMBOLS: {
     SLOTS: number;
@@ -21,6 +22,27 @@ export declare class SerializerService {
     symReverseKeys: SymbolMaps;
     symValMaps: SymbolMap;
     symValReverseMaps: SymbolMap;
+    private static codec;
+    /**
+     * Register a global payload codec for encoding/decoding serialized
+     * object data. Once registered, all object values are stored as
+     * `/b{encoded}` instead of `/s{json}`.
+     *
+     * Pass `null` to remove a previously registered codec.
+     *
+     * @example
+     * ```typescript
+     * SerializerService.registerCodec({
+     *   encode(json) { return Buffer.from(json).toString('base64'); },
+     *   decode(encoded) { return Buffer.from(encoded, 'base64').toString('utf8'); },
+     * });
+     * ```
+     */
+    static registerCodec(codec: PayloadCodec | null): void;
+    /**
+     * Returns the currently registered codec, or `null` if none.
+     */
+    static getCodec(): PayloadCodec | null;
     constructor();
     abbreviate(consumes: Consumes, symbolNames: string[], fields?: string[]): string[];
     resolveDimensionalIndex(path: string): string;

package/build/services/serializer/index.js

@@ -39,6 +39,30 @@ exports.MDATA_SYMBOLS = {
     },
 };
 class SerializerService {
+    /**
+     * Register a global payload codec for encoding/decoding serialized
+     * object data. Once registered, all object values are stored as
+     * `/b{encoded}` instead of `/s{json}`.
+     *
+     * Pass `null` to remove a previously registered codec.
+     *
+     * @example
+     * ```typescript
+     * SerializerService.registerCodec({
+     *   encode(json) { return Buffer.from(json).toString('base64'); },
+     *   decode(encoded) { return Buffer.from(encoded, 'base64').toString('utf8'); },
+     * });
+     * ```
+     */
+    static registerCodec(codec) {
+        SerializerService.codec = codec;
+    }
+    /**
+     * Returns the currently registered codec, or `null` if none.
+     */
+    static getCodec() {
+        return SerializerService.codec;
+    }
     constructor() {
         this.resetSymbols({}, {}, {});
     }
@@ -231,7 +255,13 @@ class SerializerService {
                     value = '/n';
                 }
                 else {
-                    value = '/s' + JSON.stringify(value);
+                    const json = JSON.stringify(value);
+                    if (SerializerService.codec) {
+                        value = '/b' + SerializerService.codec.encode(json);
+                    }
+                    else {
+                        value = '/s' + json;
+                    }
                 }
                 break;
         }
@@ -251,6 +281,13 @@ class SerializerService {
                 return Number(rest);
             case '/n': // null
                 return null;
+            case '/b': { // encoded object (codec)
+                const json = SerializerService.codec.decode(rest);
+                if (dateReg.exec(json)) {
+                    return new Date(JSON.parse(json));
+                }
+                return JSON.parse(json);
+            }
             case '/s': // object (JSON string)
                 if (dateReg.exec(rest)) {
                     return new Date(JSON.parse(rest));
@@ -279,4 +316,5 @@ class SerializerService {
         return obj;
     }
 }
+SerializerService.codec = null;
 exports.SerializerService = SerializerService;

package/build/services/store/index.d.ts

@@ -88,6 +88,7 @@ declare abstract class StoreService<Provider extends ProviderClient, Transaction
     abstract scrub(jobId: string): Promise<void>;
     abstract findJobs(queryString?: string, limit?: number, batchSize?: number, cursor?: string): Promise<[string, string[]]>;
     abstract findJobFields(jobId: string, fieldMatchPattern?: string, limit?: number, batchSize?: number, cursor?: string): Promise<[string, StringStringType]>;
+    abstract setCancel(jobId: string, appId: string): Promise<void>;
     abstract setThrottleRate(options: ThrottleOptions): Promise<void>;
     abstract getThrottleRates(): Promise<StringStringType>;
     abstract getThrottleRate(topic: string): Promise<number>;

package/build/services/store/providers/postgres/exporter-sql.d.ts

@@ -9,12 +9,12 @@ export declare const GET_JOB_BY_KEY = "\n  SELECT id, key, status, created_at, u
 /**
  * Fetch all attributes for a job.
  */
-export declare const GET_JOB_ATTRIBUTES = "\n  SELECT field, value\n  FROM {schema}.jobs_attributes\n  WHERE job_id = $1\n  ORDER BY field\n";
+export declare const GET_JOB_ATTRIBUTES = "\n  SELECT symbol || dimension AS field, value\n  FROM {schema}.jobs_attributes\n  WHERE job_id = $1\n  ORDER BY symbol, dimension\n";
 /**
  * Fetch activity inputs for a workflow.
  * Matches all activity jobs for the given workflow and extracts their input arguments.
  */
-export declare const GET_ACTIVITY_INPUTS = "\n  SELECT j.key, ja.value\n  FROM {schema}.jobs j\n  JOIN {schema}.jobs_attributes ja ON ja.job_id = j.id\n  WHERE j.key LIKE $1\n    AND ja.field = $2\n";
+export declare const GET_ACTIVITY_INPUTS = "\n  SELECT j.key, ja.value\n  FROM {schema}.jobs j\n  JOIN {schema}.jobs_attributes ja ON ja.job_id = j.id\n  WHERE j.key LIKE $1\n    AND ja.symbol = $2 AND ja.dimension = $3\n";
 /**
  * Fetch all worker stream messages for a job AND its child activities.
  * Child activity jobs use the pattern: -{parentJobId}-$activityName-N

package/build/services/store/providers/postgres/exporter-sql.js

@@ -18,10 +18,10 @@ exports.GET_JOB_BY_KEY = `
  * Fetch all attributes for a job.
  */
 exports.GET_JOB_ATTRIBUTES = `
-  SELECT field, value
+  SELECT symbol || dimension AS field, value
   FROM {schema}.jobs_attributes
   WHERE job_id = $1
-  ORDER BY field
+  ORDER BY symbol, dimension
 `;
 /**
  * Fetch activity inputs for a workflow.
@@ -32,7 +32,7 @@ exports.GET_ACTIVITY_INPUTS = `
   FROM {schema}.jobs j
   JOIN {schema}.jobs_attributes ja ON ja.job_id = j.id
   WHERE j.key LIKE $1
-    AND ja.field = $2
+    AND ja.symbol = $2 AND ja.dimension = $3
 `;
 /**
  * Fetch all worker stream messages for a job AND its child activities.
@@ -86,7 +86,7 @@ function buildChildWorkflowInputsQuery(childCount, schema) {
     FROM ${schema}.jobs j
     JOIN ${schema}.jobs_attributes ja ON ja.job_id = j.id
     WHERE j.key IN (${placeholders})
-      AND ja.field = $${childCount + 1}
+      AND ja.symbol = $${childCount + 1} AND ja.dimension = $${childCount + 2}
   `;
 }
 exports.buildChildWorkflowInputsQuery = buildChildWorkflowInputsQuery;

package/build/services/store/providers/postgres/kvtables.js

@@ -275,12 +275,13 @@ const KVTables = (context) => ({
                         await client.query(`
               CREATE TABLE IF NOT EXISTS ${attributesTableName} (
                 job_id UUID NOT NULL,
-                field TEXT NOT NULL,
+                symbol TEXT NOT NULL,
+                dimension TEXT NOT NULL DEFAULT '',
                 value TEXT,
                 type ${schemaName}.type_enum NOT NULL,
                 created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
                 updated_at TIMESTAMP WITH TIME ZONE DEFAULT NOW(),
-                PRIMARY KEY (job_id, field),
+                PRIMARY KEY (job_id, symbol, dimension),
                 FOREIGN KEY (job_id) REFERENCES ${fullTableName} (id) ON DELETE CASCADE
               ) PARTITION BY HASH (job_id);
             `);
@@ -320,12 +321,12 @@ const KVTables = (context) => ({
             `);
                         // Create indexes for attributes table
                         await client.query(`
-              CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_attributes_type_field 
-              ON ${attributesTableName} (type, field);
+              CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_attributes_type_symbol
+              ON ${attributesTableName} (type, symbol);
             `);
                         await client.query(`
-              CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_attributes_field 
-              ON ${attributesTableName} (field);
+              CREATE INDEX IF NOT EXISTS idx_${tableDef.name}_attributes_symbol
+              ON ${attributesTableName} (symbol);
             `);
                         break;
                     case 'list':