@hotmeshio/hotmesh 0.14.3 → 0.14.5

package/build/services/store/providers/postgres/postgres.js

@@ -754,16 +754,133 @@ class PostgresStoreService extends __1.StoreService {
             return patterns;
         }
     }
+    /**
+     * Leg1: set hook signal, atomically detecting a pending signal.
+     *
+     * Standalone (no transaction): single CTE query that reads any existing
+     * pending value, then inserts the hook signal (overwriting pending or
+     * expired entries). Returns `{success, pendingData}` in one round trip.
+     *
+     * In a transaction: queues the setnxex; pending detection deferred.
+     */
     async setHookSignal(hook, transaction) {
         const key = this.mintKey(key_1.KeyType.SIGNALS, { appId: this.appId });
         const { topic, resolved, jobId } = hook;
         const signalKey = `${topic}:${resolved}`;
-        await this.kvsql(transaction).setnxex(`${key}:${signalKey}`, jobId, Math.max(hook.expire, enums_1.HMSH_SIGNAL_EXPIRE));
+        const fullKey = `${key}:${signalKey}`;
+        const delay = Math.max(hook.expire, enums_1.HMSH_SIGNAL_EXPIRE);
+        if (transaction) {
+            await this.kvsql(transaction).setnxex(fullKey, jobId, delay);
+            return { success: true };
+        }
+        const kv = this.kvsql();
+        const tableName = kv.tableForKey(fullKey);
+        const storedKey = kv.storageKey(fullKey);
+        const sql = `
+      WITH pre AS (
+        SELECT value FROM ${tableName}
+        WHERE key = $1 AND (expiry IS NULL OR expiry > NOW())
+      ),
+      ins AS (
+        INSERT INTO ${tableName} (key, value, expiry)
+        VALUES ($1, $2, NOW() + INTERVAL '${delay} seconds')
+        ON CONFLICT (key) DO UPDATE
+          SET value = EXCLUDED.value, expiry = EXCLUDED.expiry
+          WHERE ${tableName}.expiry IS NULL
+            OR ${tableName}.expiry <= NOW()
+            OR ${tableName}.value LIKE '$pending::%'
+        RETURNING true as success
+      )
+      SELECT
+        COALESCE((SELECT success FROM ins), false) as success,
+        (SELECT value FROM pre) as existing_value
+    `;
+        try {
+            const res = await this.pgClient.query(sql, [storedKey, jobId]);
+            const row = res.rows[0] || {};
+            const success = row.success === true;
+            const existing = row.existing_value;
+            if (success && existing?.startsWith('$pending::')) {
+                return {
+                    success: true,
+                    pendingData: existing.slice('$pending::'.length),
+                };
+            }
+            return { success };
+        }
+        catch (error) {
+            if (error?.message?.includes('closed') ||
+                error?.message?.includes('queryable')) {
+                return { success: false };
+            }
+            throw error;
+        }
     }
-    async getHookSignal(topic, resolved) {
+    /**
+     * Leg2: get hook signal OR atomically set a pending signal.
+     *
+     * When `pendingData` is provided and no hook signal exists, the
+     * pending value is inserted in the SAME SQL statement — no second
+     * round trip. This is the transactional edge that prevents the
+     * signal from being lost: by the time the query returns, the
+     * pending key is already visible to leg1's setnxex.
+     *
+     * When `pendingData` is omitted, behaves as a plain read.
+     */
+    async getHookSignal(topic, resolved, pendingData, pendingExpire) {
         const key = this.mintKey(key_1.KeyType.SIGNALS, { appId: this.appId });
-        const response = await this.kvsql().get(`${key}:${topic}:${resolved}`);
-        return response ? response.toString() : undefined;
+        const fullKey = `${key}:${topic}:${resolved}`;
+        if (!pendingData) {
+            //plain read (used by deleteWebHookSignal path, tests, etc.)
+            const response = await this.kvsql().get(fullKey);
+            if (!response)
+                return undefined;
+            const value = response.toString();
+            if (value.startsWith('$pending::'))
+                return undefined;
+            return value;
+        }
+        //atomic get-or-set-pending: one round trip
+        const kv = this.kvsql();
+        const tableName = kv.tableForKey(fullKey);
+        const storedKey = kv.storageKey(fullKey);
+        const expire = pendingExpire || enums_1.HMSH_PENDING_SIGNAL_EXPIRE;
+        const pendingValue = `$pending::${pendingData}`;
+        const sql = `
+      WITH existing AS (
+        SELECT value FROM ${tableName}
+        WHERE key = $1 AND (expiry IS NULL OR expiry > NOW())
+      ),
+      pending AS (
+        INSERT INTO ${tableName} (key, value, expiry)
+        SELECT $1, $2, NOW() + INTERVAL '${expire} seconds'
+        WHERE NOT EXISTS (SELECT 1 FROM existing)
+        ON CONFLICT (key) DO UPDATE
+          SET value = EXCLUDED.value, expiry = EXCLUDED.expiry
+          WHERE ${tableName}.expiry IS NULL OR ${tableName}.expiry <= NOW()
+        RETURNING true as inserted
+      )
+      SELECT
+        (SELECT value FROM existing) as hook_value,
+        (SELECT inserted FROM pending) as pending_inserted
+    `;
+        try {
+            const res = await this.pgClient.query(sql, [storedKey, pendingValue]);
+            const row = res.rows[0] || {};
+            const hookValue = row.hook_value;
+            if (hookValue && !hookValue.startsWith('$pending::')) {
+                return hookValue;
+            }
+            //no hook signal; pending was inserted (or already existed)
+            return undefined;
+        }
+        catch (error) {
+            if (error?.message?.includes('closed') ||
+                error?.message?.includes('queryable')) {
+                return undefined;
+            }
+            throw error;
+        }
     }
     async deleteHookSignal(topic, resolved) {
         const key = this.mintKey(key_1.KeyType.SIGNALS, { appId: this.appId });

package/build/services/stream/registry.d.ts

@@ -23,6 +23,7 @@ declare class StreamConsumerRegistry {
     }, logger: ILogger, config?: {
         reclaimDelay?: number;
         reclaimCount?: number;
+        readonly?: boolean;
         retry?: any;
     }): Promise<void>;
     /**

package/build/services/stream/registry.js

@@ -29,6 +29,7 @@ class StreamConsumerRegistry {
                 topic: taskQueue,
                 reclaimDelay: config?.reclaimDelay,
                 reclaimCount: config?.reclaimCount,
+                readonly: config?.readonly || false,
                 throttle,
                 retry: config?.retry,
             }, stream, logger);
@@ -39,14 +40,17 @@ class StreamConsumerRegistry {
                 logger,
             };
             StreamConsumerRegistry.workerConsumers.set(key, entry);
-            // Create the dispatch callback that routes by workflow_name
-            const dispatchCallback = StreamConsumerRegistry.createWorkerDispatcher(key);
-            // Start consuming from the task queue stream
-            const streamKey = stream.mintKey(key_1.KeyType.STREAMS, {
-                appId,
-                topic: taskQueue,
-            });
-            router.consumeMessages(streamKey, 'WORKER', guid, dispatchCallback);
+            // Only start consuming if not readonly
+            if (!config?.readonly) {
+                // Create the dispatch callback that routes by workflow_name
+                const dispatchCallback = StreamConsumerRegistry.createWorkerDispatcher(key);
+                // Start consuming from the task queue stream
+                const streamKey = stream.mintKey(key_1.KeyType.STREAMS, {
+                    appId,
+                    topic: taskQueue,
+                });
+                router.consumeMessages(streamKey, 'WORKER', guid, dispatchCallback);
+            }
         }
         // Register the callback for this workflow name
         entry.callbacks.set(workflowName, callback);

package/build/services/task/index.d.ts

@@ -29,7 +29,10 @@ declare class TaskService {
     processTimeHooks(timeEventCallback: (jobId: string, gId: string, activityId: string, type: WorkListTaskType) => Promise<void>, listKey?: string): Promise<void>;
     cancelCleanup(): void;
     getHookRule(topic: string): Promise<HookRule | undefined>;
-    registerWebHook(topic: string, context: JobState, dad: string, expire: number, transaction?: ProviderTransaction): Promise<string>;
+    registerWebHook(topic: string, context: JobState, dad: string, expire: number): Promise<{
+        jobId: string;
+        pending?: string;
+    }>;
     processWebHookSignal(topic: string, data: Record<string, unknown>): Promise<[string, string, string, string] | undefined>;
     deleteWebHookSignal(topic: string, data: Record<string, unknown>): Promise<number>;
     /**

package/build/services/task/index.js

@@ -134,7 +134,7 @@ class TaskService {
         const rules = await this.store.getHookRules();
         return rules?.[topic]?.[0];
     }
-    async registerWebHook(topic, context, dad, expire, transaction) {
+    async registerWebHook(topic, context, dad, expire) {
         const hookRule = await this.getHookRule(topic);
         if (hookRule) {
             const mapExpression = hookRule.conditions.match[0].expected;
@@ -150,8 +150,27 @@ class TaskService {
                 jobId: compositeJobKey,
                 expire,
             };
-            await this.store.setHookSignal(hook, transaction);
-            return jobId;
+            //called standalone (no transaction) so the single CTE query can
+            //atomically detect and return pending signal data on collision
+            const result = await this.store.setHookSignal(hook);
+            if (result.pendingData) {
+                this.logger.warn('task-signal-race-pending-consumed', {
+                    topic,
+                    resolved,
+                    jobId,
+                });
+                return { jobId, pending: result.pendingData };
+            }
+            if (!result.success) {
+                //setnxex failed but no pending signal; likely a retry where
+                //our own hook signal was already set. continue normally.
+                this.logger.debug('task-signal-hook-already-set', {
+                    topic,
+                    resolved,
+                    jobId,
+                });
+            }
+            return { jobId };
         }
         else {
             throw new Error('signaler.registerWebHook:error: hook rule not found');
@@ -165,10 +184,19 @@ class TaskService {
             const context = { $self: { hook: { data } }, $hook: { data } };
             const mapExpression = hookRule.conditions.match[0].actual;
             const resolved = pipe_1.Pipe.resolve(mapExpression, context);
-            const hookSignalId = await this.store.getHookSignal(topic, resolved);
+            //resolve $expire override from the signal data (e.g., '1h', '30d')
+            const pendingExpire = typeof data.$expire === 'string'
+                ? (0, utils_1.s)(data.$expire)
+                : enums_1.HMSH_PENDING_SIGNAL_EXPIRE;
+            //atomic: returns the hook signal, or stores a pending signal
+            //in the same SQL statement if no hook is registered yet
+            const hookSignalId = await this.store.getHookSignal(topic, resolved, JSON.stringify(data), pendingExpire);
             if (!hookSignalId) {
-                //messages can be double-processed; not an issue; return `undefined`
-                //users can also provide a bogus topic; not an issue; return `undefined`
+                this.logger.warn('task-signal-race-pending-stored', {
+                    topic,
+                    resolved,
+                    expire: pendingExpire,
+                });
                 return undefined;
             }
             //`aid` is part of composite key, but the hook `topic` is its public interface;

package/build/services/worker/index.js

@@ -51,6 +51,7 @@ class WorkerService {
                     await registry_1.StreamConsumerRegistry.registerWorker(namespace, appId, guid, worker.topic, worker.workflowName, worker.callback, service.stream, service.store, logger, {
                         reclaimDelay: worker.reclaimDelay,
                         reclaimCount: worker.reclaimCount,
+                        readonly: worker.readonly,
                         retry: worker.retry,
                     });
                     // Still need a router for publishing responses back to engine
@@ -114,6 +115,7 @@ class WorkerService {
             reclaimDelay: worker.reclaimDelay,
             reclaimCount: worker.reclaimCount,
             throttle,
+            readonly: worker.readonly || false,
             retry: worker.retry,
         }, this.stream, logger);
     }

package/build/types/dba.d.ts

@@ -103,6 +103,15 @@ export interface PruneOptions {
      * @default false
      */
     keepHmark?: boolean;
+    /**
+     * If true, hard-deletes expired rows from `signal_registry`.
+     * These include consumed hook signals and stale pending signals
+     * (signals that arrived before hook registration but were never
+     * claimed). All signal_registry entries have a natural `expiry`
+     * column; this operation removes rows whose expiry has passed.
+     * @default true
+     */
+    signals?: boolean;
 }
 /**
  * Result returned by `DBA.prune()`, providing deletion
@@ -123,4 +132,6 @@ export interface PruneResult {
     transient: number;
     /** Number of jobs marked as pruned (pruned_at set) */
     marked: number;
+    /** Number of expired signal_registry rows hard-deleted */
+    signals: number;
 }

package/build/types/hotmesh.d.ts

@@ -284,6 +284,14 @@ type HotMeshWorker = {
         user: string;
         password: string;
     };
+    /**
+     * If true, the worker's router will not consume messages from the
+     * stream. The worker can still publish responses but will never
+     * dequeue or process messages. This is inherited from the
+     * connection's `readonly` flag by the Durable layer.
+     * @default false
+     */
+    readonly?: boolean;
 };
 type HotMeshConfig = {
     appId: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.14.3",
+  "version": "0.14.5",
   "description": "Durable Workflow",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",
@@ -14,8 +14,8 @@
     "obfuscate": "ts-node scripts/obfuscate.ts",
     "clean-build": "npm run clean && npm run build",
     "clean-build-obfuscate": "npm run clean-build && npm run obfuscate",
-    "docs": "typedoc",
-    "docs:clean": "rimraf ./docs/hotmesh && typedoc",
+    "docs": "typedoc && cp -R docs/hotmesh/* docs/ && rm -rf docs/hotmesh",
+    "docs:clean": "rimraf ./docs/hotmesh && typedoc && cp -R docs/hotmesh/* docs/ && rm -rf docs/hotmesh",
     "lint": "eslint . --ext .ts",
     "lint:fix": "eslint . --fix --ext .ts",
     "start": "ts-node src/index.ts",
@@ -47,6 +47,7 @@
     "test:durable:retrypolicy": "vitest run tests/durable/retry-policy",
     "test:durable:sleep": "vitest run tests/durable/sleep/postgres.test.ts",
     "test:durable:signal": "vitest run tests/durable/signal/postgres.test.ts",
+    "test:durable:readonly": "docker compose --profile readonly up -d --build && docker compose exec hotmesh-readonly npx vitest run --config tests/durable/readonly/vitest.config.mts",
     "test:durable:unknown": "vitest run tests/durable/unknown/postgres.test.ts",
     "test:durable:exporter": "HMSH_LOGLEVEL=info vitest run tests/durable/exporter",
     "test:durable:exporter:debug": "EXPORT_DEBUG=1 HMSH_LOGLEVEL=error vitest run tests/durable/basic/postgres.test.ts",

package/vitest.config.mts

@@ -5,7 +5,7 @@ export default defineConfig({
     globals: true,
     environment: 'node',
     include: ['tests/**/*.test.ts'],
-    exclude: ['node_modules', 'build', 'config'],
+    exclude: ['node_modules', 'build', 'config', 'tests/durable/readonly/**'],
     testTimeout: 60_000,
     hookTimeout: 120_000,
     fileParallelism: false,