@hotmeshio/hotmesh 0.4.1 → 0.4.2

package/README.md

@@ -1,15 +1,16 @@
-# HotMesh MemFlow
+# HotMesh
 
 **Permanent-Memory Workflows & AI Agents**
 
 ![beta release](https://img.shields.io/badge/release-beta-blue.svg)  ![made with typescript](https://img.shields.io/badge/built%20with-typescript-lightblue.svg)
 
-MemFlow is a drop-in Temporal-style engine that runs natively on Postgres — but with a twist:
-every workflow owns a *permanent*, JSON-backed context that lives beyond the main workflow.
-Any number of *hooks* (lightweight, thread-safe workers) can attach to that record at any
-time, read it, and safely write back incremental knowledge.
-Think **durable execution** + **shared, evolving memory** → perfect for human-in-the-loop
-processes and AI agents that learn over time.
+**HotMesh** is a Temporal-style workflow engine that runs natively on PostgreSQL — with a powerful twist: every workflow maintains a permanent, JSON-backed context that persists independently of the workflow itself.
+
+This means:
+
+* Any number of lightweight, thread-safe **hook workers** can attach to the same workflow record at any time.
+* These hooks can safely **read and incrementally write** to shared state.
+* The result is a **durable execution model** with **evolving memory**, ideal for **human-in-the-loop processes** and **AI agents that learn over time**.
 
 ---
 
@@ -61,10 +62,11 @@ async function main() {
 main().catch(console.error);
 ```
 
+---
 
 ## 🧠 How Permanent Memory Works
 
-* **Context = JSONB row** in `<yourappname>.jobs` table
+* **Context = persistent JSON record** – each workflow's memory is stored as a JSONB row in your Postgres database
 * **Atomic operations** (`set`, `merge`, `append`, `increment`, `toggle`, `delete`, …)
 * **Transactional** – every update participates in the workflow/DB transaction
 * **Time-travel-safe** – full replay compatibility; side-effect detector guarantees determinism
@@ -135,14 +137,37 @@ export async function hook2(name: string, kind: string): Promise<void> {
   await ctx.merge({ user: { lastSeen: new Date().toISOString() } });
   await MemFlow.workflow.signal('hook2-complete', { ok: true });
 }
-```
 
-**Highlights**
+/* ------------ Worker/Hook Registration ------------ */
+async function startWorker() {
+  const mf = await MemFlow.init({
+    appId: 'my-app',
+    engine: {
+      connection: {
+        class: Postgres,
+        options: { connectionString: process.env.DATABASE_URL }
+      }
+    }
+  });
+
+  const worker = await mf.worker.create({
+    taskQueue: 'contextual',
+    workflow: example
+  });
 
-* Hook functions are replay-safe.
-* Hook functions can safely read and write to the the *same* JSON context.
-* All context operations (`set`, `merge`, `append`, etc.) execute transactionally.
-* Context data is stored as JSONB; add partial indexes for improved query analysis.
+  await mf.worker.create({
+    taskQueue: 'contextual',
+    workflow: hook1
+  });
+
+  await mf.worker.create({
+    taskQueue: 'contextual',
+    workflow: hook2
+  });
+
+  console.log('Workers and hooks started and listening...');
+}
+```
 
 ---
 

package/build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@hotmeshio/hotmesh",
-    "version": "0.4.1",
+    "version": "0.4.2",
     "description": "Permanent-Memory Workflows & AI Agents",
     "main": "./build/index.js",
     "types": "./build/index.d.ts",

package/build/services/memflow/workflow/signal.d.ts

@@ -1,6 +1,28 @@
 /**
  * Sends a signal payload to any paused workflow thread awaiting this signal.
- * @param {string} signalId - Unique signal identifier.
+ * This method is commonly used to coordinate between workflows, hook functions,
+ * and external events.
+ *
+ * @example
+ * // Basic usage - send a simple signal with data
+ * await MemFlow.workflow.signal('signal-id', { name: 'WarmMash' });
+ *
+ * @example
+ * // Hook function signaling completion
+ * export async function exampleHook(name: string): Promise<void> {
+ *   const result = await processData(name);
+ *   await MemFlow.workflow.signal('hook-complete', { data: result });
+ * }
+ *
+ * @example
+ * // Signal with complex data structure
+ * await MemFlow.workflow.signal('process-complete', {
+ *   status: 'success',
+ *   data: { id: 123, name: 'test' },
+ *   timestamp: new Date().toISOString()
+ * });
+ *
+ * @param {string} signalId - Unique signal identifier that matches a waitFor() call.
  * @param {Record<any, any>} data - The payload to send with the signal.
  * @returns {Promise<string>} The resulting hook/stream ID.
  */

package/build/services/memflow/workflow/signal.js

@@ -5,7 +5,29 @@ const common_1 = require("./common");
 const isSideEffectAllowed_1 = require("./isSideEffectAllowed");
 /**
  * Sends a signal payload to any paused workflow thread awaiting this signal.
- * @param {string} signalId - Unique signal identifier.
+ * This method is commonly used to coordinate between workflows, hook functions,
+ * and external events.
+ *
+ * @example
+ * // Basic usage - send a simple signal with data
+ * await MemFlow.workflow.signal('signal-id', { name: 'WarmMash' });
+ *
+ * @example
+ * // Hook function signaling completion
+ * export async function exampleHook(name: string): Promise<void> {
+ *   const result = await processData(name);
+ *   await MemFlow.workflow.signal('hook-complete', { data: result });
+ * }
+ *
+ * @example
+ * // Signal with complex data structure
+ * await MemFlow.workflow.signal('process-complete', {
+ *   status: 'success',
+ *   data: { id: 123, name: 'test' },
+ *   timestamp: new Date().toISOString()
+ * });
+ *
+ * @param {string} signalId - Unique signal identifier that matches a waitFor() call.
  * @param {Record<any, any>} data - The payload to send with the signal.
  * @returns {Promise<string>} The resulting hook/stream ID.
  */

package/build/services/memflow/workflow/sleepFor.d.ts

@@ -2,7 +2,23 @@
  * Sleeps the workflow for a specified duration, deterministically.
  * On replay, it will not actually sleep again, but resume after sleep.
  *
- * @param {string} duration - A human-readable duration string (e.g., '1m', '2 hours').
+ * @example
+ * // Basic usage - sleep for a specific duration
+ * await MemFlow.workflow.sleepFor('2 seconds');
+ *
+ * @example
+ * // Using with Promise.all for parallel operations
+ * const [greeting, timeInSeconds] = await Promise.all([
+ *   someActivity(name),
+ *   MemFlow.workflow.sleepFor('1 second')
+ * ]);
+ *
+ * @example
+ * // Multiple sequential sleeps
+ * await MemFlow.workflow.sleepFor('1 seconds');  // First pause
+ * await MemFlow.workflow.sleepFor('2 seconds');  // Second pause
+ *
+ * @param {string} duration - A human-readable duration string (e.g., '1m', '2 hours', '30 seconds').
  * @returns {Promise<number>} The resolved duration in seconds.
  */
 export declare function sleepFor(duration: string): Promise<number>;

package/build/services/memflow/workflow/sleepFor.js

@@ -7,7 +7,23 @@ const didRun_1 = require("./didRun");
  * Sleeps the workflow for a specified duration, deterministically.
  * On replay, it will not actually sleep again, but resume after sleep.
  *
- * @param {string} duration - A human-readable duration string (e.g., '1m', '2 hours').
+ * @example
+ * // Basic usage - sleep for a specific duration
+ * await MemFlow.workflow.sleepFor('2 seconds');
+ *
+ * @example
+ * // Using with Promise.all for parallel operations
+ * const [greeting, timeInSeconds] = await Promise.all([
+ *   someActivity(name),
+ *   MemFlow.workflow.sleepFor('1 second')
+ * ]);
+ *
+ * @example
+ * // Multiple sequential sleeps
+ * await MemFlow.workflow.sleepFor('1 seconds');  // First pause
+ * await MemFlow.workflow.sleepFor('2 seconds');  // Second pause
+ *
+ * @param {string} duration - A human-readable duration string (e.g., '1m', '2 hours', '30 seconds').
  * @returns {Promise<number>} The resolved duration in seconds.
  */
 async function sleepFor(duration) {

package/build/services/memflow/workflow/waitFor.d.ts

@@ -1,7 +1,28 @@
 /**
  * Pauses the workflow until a signal with the given `signalId` is received.
+ * This method is commonly used to coordinate between the main workflow and hook functions,
+ * or to wait for external events.
  *
- * @template T
+ * @example
+ * // Basic usage - wait for a single signal
+ * const payload = await MemFlow.workflow.waitFor<PayloadType>('abcdefg');
+ *
+ * @example
+ * // Wait for multiple signals in parallel
+ * const [signal1, signal2] = await Promise.all([
+ *   MemFlow.workflow.waitFor<Record<string, any>>('signal1'),
+ *   MemFlow.workflow.waitFor<Record<string, any>>('signal2')
+ * ]);
+ *
+ * @example
+ * // Typical pattern with hook functions
+ * // In main workflow:
+ * await MemFlow.workflow.waitFor<ResponseType>('hook-complete');
+ *
+ * // In hook function:
+ * await MemFlow.workflow.signal('hook-complete', { data: result });
+ *
+ * @template T - The type of data expected in the signal payload
  * @param {string} signalId - A unique signal identifier shared by the sender and receiver.
  * @returns {Promise<T>} The data payload associated with the received signal.
  */

package/build/services/memflow/workflow/waitFor.js

@@ -5,8 +5,29 @@ const common_1 = require("./common");
 const didRun_1 = require("./didRun");
 /**
  * Pauses the workflow until a signal with the given `signalId` is received.
+ * This method is commonly used to coordinate between the main workflow and hook functions,
+ * or to wait for external events.
  *
- * @template T
+ * @example
+ * // Basic usage - wait for a single signal
+ * const payload = await MemFlow.workflow.waitFor<PayloadType>('abcdefg');
+ *
+ * @example
+ * // Wait for multiple signals in parallel
+ * const [signal1, signal2] = await Promise.all([
+ *   MemFlow.workflow.waitFor<Record<string, any>>('signal1'),
+ *   MemFlow.workflow.waitFor<Record<string, any>>('signal2')
+ * ]);
+ *
+ * @example
+ * // Typical pattern with hook functions
+ * // In main workflow:
+ * await MemFlow.workflow.waitFor<ResponseType>('hook-complete');
+ *
+ * // In hook function:
+ * await MemFlow.workflow.signal('hook-complete', { data: result });
+ *
+ * @template T - The type of data expected in the signal payload
  * @param {string} signalId - A unique signal identifier shared by the sender and receiver.
  * @returns {Promise<T>} The data payload associated with the received signal.
  */

package/build/services/store/providers/postgres/postgres.d.ts

@@ -18,6 +18,7 @@ import { KVTables } from './kvtables';
 declare class PostgresStoreService extends StoreService<ProviderClient, ProviderTransaction> {
     pgClient: PostgresClientType;
     kvTables: ReturnType<typeof KVTables>;
+    isScout: boolean;
     transact(): ProviderTransaction;
     constructor(storeClient: ProviderClient);
     init(namespace: string, appId: string, logger: ILogger): Promise<HotMeshApps>;
@@ -141,5 +142,37 @@ declare class PostgresStoreService extends StoreService<ProviderClient, Provider
     setThrottleRate(options: ThrottleOptions): Promise<void>;
     getThrottleRates(): Promise<StringStringType>;
     getThrottleRate(topic: string): Promise<number>;
+    /**
+     * Deploy time-aware notification triggers and functions
+     */
+    private deployTimeNotificationTriggers;
+    /**
+     * Enhanced time scout that uses LISTEN/NOTIFY to reduce polling
+     */
+    startTimeScoutWithNotifications(timeEventCallback: (jobId: string, gId: string, activityId: string, type: WorkListTaskType) => Promise<void>): Promise<void>;
+    /**
+     * Handle time notifications from PostgreSQL
+     */
+    private handleTimeNotification;
+    /**
+     * Process time hooks that are ready to be awakened
+     */
+    private processReadyTimeHooks;
+    /**
+     * Enhanced time scout process that uses notifications
+     */
+    private processTimeHooksWithNotifications;
+    /**
+     * Get the next awakening time from the database
+     */
+    private getNextAwakeningTime;
+    /**
+     * Update the time scout's sleep timing based on schedule changes
+     */
+    private updateTimeScoutSleep;
+    /**
+     * Enhanced shouldScout that can handle notifications
+     */
+    shouldScout(): Promise<boolean>;
 }
 export { PostgresStoreService };

package/build/services/store/providers/postgres/postgres.js

@@ -39,6 +39,7 @@ class PostgresStoreService extends __1.StoreService {
     }
     constructor(storeClient) {
         super(storeClient);
+        this.isScout = false;
         //Instead of directly referencing the 'pg' package and methods like 'query',
         //  the PostgresStore wraps the 'pg' client in a class that implements
         //  the Redis client interface. This allows the same methods to be called
@@ -58,6 +59,8 @@ class PostgresStoreService extends __1.StoreService {
         this.logger = logger;
         //confirm db tables exist
         await this.kvTables.deploy(appId);
+        // Deploy time notification triggers
+        await this.deployTimeNotificationTriggers(appId);
         //note: getSettings will contact db to confirm r/w access
         const settings = await this.getSettings(true);
         this.cache = new cache_1.Cache(appId, settings);
@@ -1032,5 +1035,210 @@ class PostgresStoreService extends __1.StoreService {
         }
         return resolveRate(response, topic);
     }
+    /**
+     * Deploy time-aware notification triggers and functions
+     */
+    async deployTimeNotificationTriggers(appId) {
+        const schemaName = this.kvsql().safeName(appId);
+        const client = this.pgClient;
+        try {
+            // Read the SQL template and replace schema placeholder
+            const fs = await Promise.resolve().then(() => __importStar(require('fs')));
+            const path = await Promise.resolve().then(() => __importStar(require('path')));
+            const sqlTemplate = fs.readFileSync(path.join(__dirname, 'time-notify.sql'), 'utf8');
+            const sql = sqlTemplate.replace(/{schema}/g, schemaName);
+            // Execute the entire SQL as one statement (functions contain $$ blocks with semicolons)
+            await client.query(sql);
+            this.logger.info('postgres-time-notifications-deployed', {
+                appId,
+                schemaName,
+                message: 'Time-aware notifications ENABLED - using LISTEN/NOTIFY instead of polling'
+            });
+        }
+        catch (error) {
+            this.logger.error('postgres-time-notifications-deploy-error', {
+                appId,
+                schemaName,
+                error: error.message
+            });
+            // Don't throw - fall back to polling mode
+        }
+    }
+    /**
+     * Enhanced time scout that uses LISTEN/NOTIFY to reduce polling
+     */
+    async startTimeScoutWithNotifications(timeEventCallback) {
+        const channelName = `time_hooks_${this.appId}`;
+        try {
+            // Set up LISTEN for time notifications
+            await this.pgClient.query(`LISTEN "${channelName}"`);
+            // Set up notification handler
+            this.pgClient.on('notification', (notification) => {
+                this.handleTimeNotification(notification, timeEventCallback);
+            });
+            this.logger.debug('postgres-time-scout-notifications-started', {
+                appId: this.appId,
+                channelName
+            });
+            // Start the enhanced time scout loop
+            await this.processTimeHooksWithNotifications(timeEventCallback);
+        }
+        catch (error) {
+            this.logger.error('postgres-time-scout-notifications-error', {
+                appId: this.appId,
+                error
+            });
+            // Fall back to regular polling mode
+            throw error;
+        }
+    }
+    /**
+     * Handle time notifications from PostgreSQL
+     */
+    async handleTimeNotification(notification, timeEventCallback) {
+        try {
+            const payload = JSON.parse(notification.payload);
+            const { type, app_id, next_awakening, ready_at } = payload;
+            if (app_id !== this.appId) {
+                return; // Not for this app
+            }
+            this.logger.debug('postgres-time-notification-received', {
+                type,
+                appId: app_id,
+                nextAwakening: next_awakening,
+                readyAt: ready_at
+            });
+            if (type === 'time_hooks_ready') {
+                // Process any ready time hooks immediately
+                await this.processReadyTimeHooks(timeEventCallback);
+            }
+            else if (type === 'time_schedule_updated') {
+                // Update our sleep timing if we're the time scout
+                await this.updateTimeScoutSleep(next_awakening);
+            }
+        }
+        catch (error) {
+            this.logger.error('postgres-time-notification-handle-error', {
+                notification,
+                error
+            });
+        }
+    }
+    /**
+     * Process time hooks that are ready to be awakened
+     */
+    async processReadyTimeHooks(timeEventCallback) {
+        let hasMoreTasks = true;
+        while (hasMoreTasks) {
+            const workListTask = await this.getNextTask();
+            if (Array.isArray(workListTask)) {
+                const [listKey, target, gId, activityId, type] = workListTask;
+                if (type === 'child') {
+                    // Skip child tasks - they're handled by ancestors
+                }
+                else if (type === 'delist') {
+                    // Delist the signal key
+                    const key = this.mintKey(key_1.KeyType.SIGNALS, { appId: this.appId });
+                    await this.delistSignalKey(key, target);
+                }
+                else {
+                    // Process the task
+                    await timeEventCallback(target, gId, activityId, type);
+                }
+            }
+            else if (workListTask === true) {
+                // A worklist was emptied, continue processing
+                continue;
+            }
+            else {
+                // No more tasks ready
+                hasMoreTasks = false;
+            }
+        }
+    }
+    /**
+     * Enhanced time scout process that uses notifications
+     */
+    async processTimeHooksWithNotifications(timeEventCallback) {
+        let currentSleepTimeout = null;
+        // Function to calculate next sleep duration
+        const calculateNextSleep = async () => {
+            const nextAwakeningTime = await this.getNextAwakeningTime();
+            if (nextAwakeningTime) {
+                const sleepMs = nextAwakeningTime - Date.now();
+                return Math.max(sleepMs, 0);
+            }
+            // Default sleep if no tasks scheduled
+            return enums_1.HMSH_FIDELITY_SECONDS * 1000;
+        };
+        // Main loop
+        while (true) {
+            try {
+                if (await this.shouldScout()) {
+                    // Process any ready tasks
+                    await this.processReadyTimeHooks(timeEventCallback);
+                    // Calculate next sleep
+                    const sleepMs = await calculateNextSleep();
+                    // Sleep with ability to be interrupted by notifications
+                    await new Promise((resolve) => {
+                        currentSleepTimeout = setTimeout(resolve, sleepMs);
+                    });
+                }
+                else {
+                    // Not the scout, sleep longer
+                    await (0, utils_1.sleepFor)(enums_1.HMSH_SCOUT_INTERVAL_SECONDS * 1000);
+                }
+            }
+            catch (error) {
+                this.logger.error('postgres-time-scout-loop-error', { error });
+                await (0, utils_1.sleepFor)(1000);
+            }
+        }
+    }
+    /**
+     * Get the next awakening time from the database
+     */
+    async getNextAwakeningTime() {
+        const schemaName = this.kvsql().safeName(this.appId);
+        const appKey = `${this.appId}:time_range`;
+        try {
+            const result = await this.pgClient.query(`SELECT ${schemaName}.get_next_awakening_time($1) as next_time`, [appKey]);
+            if (result.rows[0]?.next_time) {
+                return new Date(result.rows[0].next_time).getTime();
+            }
+            return null;
+        }
+        catch (error) {
+            this.logger.error('postgres-get-next-awakening-error', { error });
+            return null;
+        }
+    }
+    /**
+     * Update the time scout's sleep timing based on schedule changes
+     */
+    async updateTimeScoutSleep(nextAwakening) {
+        // This could be used to interrupt current sleep and recalculate
+        // For now, just log the schedule update
+        this.logger.debug('postgres-time-schedule-updated', {
+            nextAwakening,
+            currentTime: Date.now()
+        });
+    }
+    /**
+     * Enhanced shouldScout that can handle notifications
+     */
+    async shouldScout() {
+        const wasScout = this.isScout;
+        const isScout = wasScout || (this.isScout = await this.reserveScoutRole('time'));
+        if (isScout) {
+            if (!wasScout) {
+                setTimeout(() => {
+                    this.isScout = false;
+                }, enums_1.HMSH_SCOUT_INTERVAL_SECONDS * 1000);
+            }
+            return true;
+        }
+        return false;
+    }
 }
 exports.PostgresStoreService = PostgresStoreService;

package/build/services/task/index.d.ts

@@ -32,5 +32,17 @@ declare class TaskService {
     registerWebHook(topic: string, context: JobState, dad: string, expire: number, transaction?: ProviderTransaction): Promise<string>;
     processWebHookSignal(topic: string, data: Record<string, unknown>): Promise<[string, string, string, string] | undefined>;
     deleteWebHookSignal(topic: string, data: Record<string, unknown>): Promise<number>;
+    /**
+     * Enhanced processTimeHooks that uses notifications for PostgreSQL stores
+     */
+    processTimeHooksWithNotifications(timeEventCallback: (jobId: string, gId: string, activityId: string, type: WorkListTaskType) => Promise<void>): Promise<void>;
+    /**
+     * Check if this is a PostgreSQL store
+     */
+    private isPostgresStore;
+    /**
+     * Check if the store supports notifications
+     */
+    private supportsNotifications;
 }
 export { TaskService };

package/build/services/task/index.js

@@ -202,5 +202,52 @@ class TaskService {
             throw new Error('signaler.process:error: hook rule not found');
         }
     }
+    /**
+     * Enhanced processTimeHooks that uses notifications for PostgreSQL stores
+     */
+    async processTimeHooksWithNotifications(timeEventCallback) {
+        // Check if the store supports notifications
+        if (this.isPostgresStore() && this.supportsNotifications()) {
+            try {
+                this.logger.info('task-using-notification-mode', {
+                    appId: this.store.appId,
+                    message: 'Time scout using PostgreSQL LISTEN/NOTIFY mode for efficient task processing'
+                });
+                // Use the PostgreSQL store's notification-based approach
+                await this.store.startTimeScoutWithNotifications(timeEventCallback);
+            }
+            catch (error) {
+                this.logger.warn('task-notifications-fallback', {
+                    appId: this.store.appId,
+                    error: error.message,
+                    fallbackTo: 'polling',
+                    message: 'Notification mode failed - falling back to traditional polling'
+                });
+                // Fall back to regular polling
+                await this.processTimeHooks(timeEventCallback);
+            }
+        }
+        else {
+            this.logger.info('task-using-polling-mode', {
+                appId: this.store.appId,
+                storeType: this.store.constructor.name,
+                message: 'Time scout using traditional polling mode (notifications not available)'
+            });
+            // Use regular polling for non-PostgreSQL stores
+            await this.processTimeHooks(timeEventCallback);
+        }
+    }
+    /**
+     * Check if this is a PostgreSQL store
+     */
+    isPostgresStore() {
+        return this.store.constructor.name === 'PostgresStoreService';
+    }
+    /**
+     * Check if the store supports notifications
+     */
+    supportsNotifications() {
+        return typeof this.store.startTimeScoutWithNotifications === 'function';
+    }
 }
 exports.TaskService = TaskService;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hotmeshio/hotmesh",
-  "version": "0.4.1",
+  "version": "0.4.2",
   "description": "Permanent-Memory Workflows & AI Agents",
   "main": "./build/index.js",
   "types": "./build/index.d.ts",