@hotmeshio/hotmesh 0.9.0 → 0.10.0

package/build/services/dba/index.js

@@ -0,0 +1,280 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DBA = void 0;
+const utils_1 = require("../../modules/utils");
+const postgres_1 = require("../connector/providers/postgres");
+/**
+ * Database maintenance operations for HotMesh's Postgres backend.
+ *
+ * HotMesh uses soft-delete patterns: expired jobs and stream messages
+ * retain their rows with `expired_at` set but are never physically
+ * removed during normal operation. Over time, three tables accumulate
+ * dead rows:
+ *
+ * | Table | What accumulates |
+ * |---|---|
+ * | `{appId}.jobs` | Completed/expired jobs with `expired_at` set |
+ * | `{appId}.jobs_attributes` | Execution artifacts (`adata`, `hmark`, `jmark`, `status`, `other`) that are only needed during workflow execution |
+ * | `{appId}.streams` | Processed stream messages with `expired_at` set |
+ *
+ * The `DBA` service addresses this with two methods:
+ *
+ * - {@link DBA.prune | prune()} — Targets any combination of jobs,
+ *   streams, and attributes independently. Each table can be pruned on
+ *   its own schedule with its own retention window.
+ * - {@link DBA.deploy | deploy()} — Pre-deploys the Postgres function
+ *   (e.g., during CI/CD migrations) without running a prune.
+ *
+ * ## Independent cron schedules (TypeScript)
+ *
+ * Each table can be targeted independently, allowing different retention
+ * windows and schedules:
+ *
+ * @example
+ * ```typescript
+ * import { Client as Postgres } from 'pg';
+ * import { DBA } from '@hotmeshio/hotmesh';
+ *
+ * const connection = {
+ *   class: Postgres,
+ *   options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' },
+ * };
+ *
+ * // Cron 1 — Nightly: strip execution artifacts from completed jobs
+ * //          Keeps all jobs and their jdata/udata; keeps all streams.
+ * await DBA.prune({
+ *   appId: 'myapp', connection,
+ *   jobs: false, streams: false, attributes: true,
+ * });
+ *
+ * // Cron 2 — Hourly: remove processed stream messages older than 24h
+ * await DBA.prune({
+ *   appId: 'myapp', connection,
+ *   expire: '24 hours',
+ *   jobs: false, streams: true,
+ * });
+ *
+ * // Cron 3 — Weekly: remove expired jobs older than 30 days
+ * await DBA.prune({
+ *   appId: 'myapp', connection,
+ *   expire: '30 days',
+ *   jobs: true, streams: false,
+ * });
+ * ```
+ *
+ * ## Direct SQL (schedulable via pg_cron)
+ *
+ * The underlying Postgres function can be called directly, without
+ * the TypeScript SDK. Schedule it via `pg_cron`, `crontab`, or any
+ * SQL client:
+ *
+ * ```sql
+ * -- Strip attributes only (keep all jobs and streams)
+ * SELECT * FROM myapp.prune('0 seconds', false, false, true);
+ *
+ * -- Prune streams older than 24 hours (keep jobs)
+ * SELECT * FROM myapp.prune('24 hours', false, true, false);
+ *
+ * -- Prune expired jobs older than 30 days (keep streams)
+ * SELECT * FROM myapp.prune('30 days', true, false, false);
+ *
+ * -- Prune everything older than 7 days and strip attributes
+ * SELECT * FROM myapp.prune('7 days', true, true, true);
+ * ```
+ */
+class DBA {
+    /**
+     * @private
+     */
+    constructor() { }
+    /**
+     * Sanitizes an appId for use as a Postgres schema name.
+     * Mirrors the naming logic used during table deployment.
+     * @private
+     */
+    static safeName(input) {
+        if (!input)
+            return 'connections';
+        let name = input.trim().toLowerCase();
+        name = name.replace(/[^a-z0-9]+/g, '_');
+        if (name.length > 63) {
+            name = name.slice(0, 63);
+        }
+        name = name.replace(/_+$/g, '');
+        return name || 'connections';
+    }
+    /**
+     * Acquires a Postgres client from the connection config.
+     * @private
+     */
+    static async getClient(connection) {
+        if (postgres_1.PostgresConnection.isPoolClient(connection.class)) {
+            const poolClient = await connection.class.connect();
+            return {
+                client: poolClient,
+                release: async () => poolClient.release(),
+            };
+        }
+        const pgConnection = await postgres_1.PostgresConnection.connect((0, utils_1.guid)(), connection.class, connection.options);
+        return {
+            client: pgConnection.getClient(),
+            release: async () => { },
+        };
+    }
+    /**
+     * Returns the SQL for the server-side `prune()` function.
+     * @private
+     */
+    static getPruneFunctionSQL(schema) {
+        return `
+      CREATE OR REPLACE FUNCTION ${schema}.prune(
+        retention INTERVAL DEFAULT INTERVAL '7 days',
+        prune_jobs BOOLEAN DEFAULT TRUE,
+        prune_streams BOOLEAN DEFAULT TRUE,
+        strip_attributes BOOLEAN DEFAULT FALSE
+      )
+      RETURNS TABLE(
+        deleted_jobs BIGINT,
+        deleted_streams BIGINT,
+        stripped_attributes BIGINT
+      )
+      LANGUAGE plpgsql
+      AS $$
+      DECLARE
+        v_deleted_jobs BIGINT := 0;
+        v_deleted_streams BIGINT := 0;
+        v_stripped_attributes BIGINT := 0;
+      BEGIN
+        -- 1. Hard-delete expired jobs older than the retention window.
+        --    FK CASCADE on jobs_attributes handles attribute cleanup.
+        IF prune_jobs THEN
+          DELETE FROM ${schema}.jobs
+          WHERE expired_at IS NOT NULL
+            AND expired_at < NOW() - retention;
+          GET DIAGNOSTICS v_deleted_jobs = ROW_COUNT;
+        END IF;
+
+        -- 2. Hard-delete expired stream messages older than the retention window.
+        IF prune_streams THEN
+          DELETE FROM ${schema}.streams
+          WHERE expired_at IS NOT NULL
+            AND expired_at < NOW() - retention;
+          GET DIAGNOSTICS v_deleted_streams = ROW_COUNT;
+        END IF;
+
+        -- 3. Optionally strip execution artifacts from completed, live jobs.
+        --    Retains jdata (workflow return data) and udata (searchable data).
+        IF strip_attributes THEN
+          DELETE FROM ${schema}.jobs_attributes
+          WHERE job_id IN (
+            SELECT id FROM ${schema}.jobs
+            WHERE status = 0
+              AND is_live = TRUE
+          )
+          AND type NOT IN ('jdata', 'udata');
+          GET DIAGNOSTICS v_stripped_attributes = ROW_COUNT;
+        END IF;
+
+        deleted_jobs := v_deleted_jobs;
+        deleted_streams := v_deleted_streams;
+        stripped_attributes := v_stripped_attributes;
+        RETURN NEXT;
+      END;
+      $$;
+    `;
+    }
+    /**
+     * Deploys the `prune()` Postgres function into the target schema.
+     * Idempotent — uses `CREATE OR REPLACE` and can be called repeatedly.
+     *
+     * The function is automatically deployed when {@link DBA.prune} is called,
+     * but this method is exposed for explicit control (e.g., CI/CD
+     * migration scripts that provision database objects before the
+     * application starts).
+     *
+     * @param connection - Postgres provider configuration
+     * @param appId - Application identifier (schema name)
+     *
+     * @example
+     * ```typescript
+     * import { Client as Postgres } from 'pg';
+     * import { DBA } from '@hotmeshio/hotmesh';
+     *
+     * // Pre-deploy during CI/CD migration
+     * await DBA.deploy(
+     *   {
+     *     class: Postgres,
+     *     options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' }
+     *   },
+     *   'myapp',
+     * );
+     * ```
+     */
+    static async deploy(connection, appId) {
+        const schema = DBA.safeName(appId);
+        const { client, release } = await DBA.getClient(connection);
+        try {
+            await client.query(DBA.getPruneFunctionSQL(schema));
+        }
+        finally {
+            await release();
+        }
+    }
+    /**
+     * Prunes expired data and/or strips execution artifacts from
+     * completed jobs. Each operation is independently controlled,
+     * so callers can target a single table per cron schedule.
+     *
+     * Operations (each enabled individually):
+     * 1. **jobs** — Hard-deletes expired jobs older than the retention
+     *    window (FK CASCADE removes their attributes automatically)
+     * 2. **streams** — Hard-deletes expired stream messages older than
+     *    the retention window
+     * 3. **attributes** — Strips non-essential attributes (`adata`,
+     *    `hmark`, `jmark`, `status`, `other`) from completed jobs,
+     *    retaining only `jdata` and `udata`
+     *
+     * @param options - Prune configuration
+     * @returns Counts of deleted/stripped rows
+     *
+     * @example
+     * ```typescript
+     * import { Client as Postgres } from 'pg';
+     * import { DBA } from '@hotmeshio/hotmesh';
+     *
+     * // Strip attributes only — keep all jobs and streams
+     * await DBA.prune({
+     *   appId: 'myapp',
+     *   connection: {
+     *     class: Postgres,
+     *     options: { connectionString: 'postgresql://usr:pwd@localhost:5432/db' }
+     *   },
+     *   jobs: false,
+     *   streams: false,
+     *   attributes: true,
+     * });
+     * ```
+     */
+    static async prune(options) {
+        const schema = DBA.safeName(options.appId);
+        const expire = options.expire ?? '7 days';
+        const jobs = options.jobs ?? true;
+        const streams = options.streams ?? true;
+        const attributes = options.attributes ?? false;
+        await DBA.deploy(options.connection, options.appId);
+        const { client, release } = await DBA.getClient(options.connection);
+        try {
+            const result = await client.query(`SELECT * FROM ${schema}.prune($1::interval, $2::boolean, $3::boolean, $4::boolean)`, [expire, jobs, streams, attributes]);
+            const row = result.rows[0];
+            return {
+                jobs: Number(row.deleted_jobs),
+                streams: Number(row.deleted_streams),
+                attributes: Number(row.stripped_attributes),
+            };
+        }
+        finally {
+            await release();
+        }
+    }
+}
+exports.DBA = DBA;

package/build/services/{memflow → durable}/client.d.ts

@@ -1,7 +1,7 @@
 import { HotMesh } from '../hotmesh';
-import { ClientConfig, ClientWorkflow, Connection, WorkflowOptions } from '../../types/memflow';
+import { ClientConfig, ClientWorkflow, Connection, WorkflowOptions } from '../../types/durable';
 /**
- * The MemFlow `Client` service is functionally
+ * The Durable `Client` service is functionally
  * equivalent to the Temporal `Client` service.
  * Start a new workflow execution by calling
  * `workflow.start`. Note the direct connection to
@@ -81,7 +81,7 @@ export declare class ClientService {
      */
     search: (hotMeshClient: HotMesh, index: string, query: string[]) => Promise<string[]>;
     /**
-     * The MemFlow `Client` service is functionally
+     * The Durable `Client` service is functionally
      * equivalent to the Temporal `Client` service.
      * Starting a workflow is the primary use case and
      * is accessed by calling workflow.start().

package/build/services/{memflow → durable}/client.js

@@ -11,7 +11,7 @@ const search_1 = require("./search");
 const handle_1 = require("./handle");
 const factory_1 = require("./schemas/factory");
 /**
- * The MemFlow `Client` service is functionally
+ * The Durable `Client` service is functionally
  * equivalent to the Temporal `Client` service.
  * Start a new workflow execution by calling
  * `workflow.start`. Note the direct connection to
@@ -83,7 +83,7 @@ class ClientService {
             //resolve, activate, and return the client
             const resolvedClient = await hotMeshClient;
             if (!readonly) {
-                resolvedClient.engine.logger.info('memflow-readonly-client', {
+                resolvedClient.engine.logger.info('durable-readonly-client', {
                     guid: resolvedClient.engine.guid,
                     appId: targetNS,
                 });
@@ -114,7 +114,7 @@ class ClientService {
             return await searchClient.sendIndexedQuery(index, query);
         };
         /**
-         * The MemFlow `Client` service is functionally
+         * The Durable `Client` service is functionally
          * equivalent to the Temporal `Client` service.
          * Starting a workflow is the primary use case and
          * is accessed by calling workflow.start().
@@ -145,9 +145,9 @@ class ClientService {
                     parentWorkflowId: options.parentWorkflowId,
                     workflowId: options.workflowId || hotmesh_1.HotMesh.guid(),
                     workflowTopic: workflowTopic,
-                    backoffCoefficient: options.config?.backoffCoefficient || enums_1.HMSH_MEMFLOW_EXP_BACKOFF,
-                    maximumAttempts: options.config?.maximumAttempts || enums_1.HMSH_MEMFLOW_MAX_ATTEMPTS,
-                    maximumInterval: (0, utils_1.s)(options.config?.maximumInterval || enums_1.HMSH_MEMFLOW_MAX_INTERVAL),
+                    backoffCoefficient: options.config?.backoffCoefficient || enums_1.HMSH_DURABLE_EXP_BACKOFF,
+                    maximumAttempts: options.config?.maximumAttempts || enums_1.HMSH_DURABLE_MAX_ATTEMPTS,
+                    maximumInterval: (0, utils_1.s)(options.config?.maximumInterval || enums_1.HMSH_DURABLE_MAX_INTERVAL),
                 };
                 const context = { metadata: { trc, spn }, data: {} };
                 const jobId = await hotMeshClient.pub(`${options.namespace ?? factory_1.APP_ID}.execute`, payload, context, {
@@ -191,9 +191,9 @@ class ClientService {
                     arguments: [...options.args],
                     id: options.workflowId,
                     workflowTopic,
-                    backoffCoefficient: options.config?.backoffCoefficient || enums_1.HMSH_MEMFLOW_EXP_BACKOFF,
-                    maximumAttempts: options.config?.maximumAttempts || enums_1.HMSH_MEMFLOW_MAX_ATTEMPTS,
-                    maximumInterval: (0, utils_1.s)(options.config?.maximumInterval || enums_1.HMSH_MEMFLOW_MAX_INTERVAL),
+                    backoffCoefficient: options.config?.backoffCoefficient || enums_1.HMSH_DURABLE_EXP_BACKOFF,
+                    maximumAttempts: options.config?.maximumAttempts || enums_1.HMSH_DURABLE_MAX_ATTEMPTS,
+                    maximumInterval: (0, utils_1.s)(options.config?.maximumInterval || enums_1.HMSH_DURABLE_MAX_INTERVAL),
                 };
                 //seed search data before entering
                 const hotMeshClient = await this.getHotMeshClient(taskQueue, options.namespace);
@@ -237,7 +237,7 @@ class ClientService {
              * await client.workflow.search(
              *   'someTaskQueue'
              *   'someWorkflowName',
-             *   'memflow',
+             *   'durable',
              *   'user',
              *   ...args,
              * );
@@ -251,7 +251,7 @@ class ClientService {
                     return await this.search(hotMeshClient, index, query);
                 }
                 catch (error) {
-                    hotMeshClient.engine.logger.error('memflow-client-search-err', {
+                    hotMeshClient.engine.logger.error('durable-client-search-err', {
                         error,
                     });
                     throw error;
@@ -314,7 +314,7 @@ class ClientService {
                 await hotMesh.activate(version);
             }
             catch (error) {
-                hotMesh.engine.logger.error('memflow-client-activate-err', {
+                hotMesh.engine.logger.error('durable-client-activate-err', {
                     error,
                 });
                 throw error;
@@ -326,7 +326,7 @@ class ClientService {
                 await hotMesh.activate(version);
             }
             catch (error) {
-                hotMesh.engine.logger.error('memflow-client-deploy-activate-err', {
+                hotMesh.engine.logger.error('durable-client-deploy-activate-err', {
                     error,
                 });
                 throw error;

package/build/services/{memflow → durable}/connection.d.ts

@@ -1,10 +1,10 @@
-import { Connection } from '../../types/memflow';
+import { Connection } from '../../types/durable';
 import { ProviderConfig, ProvidersConfig } from '../../types/provider';
 /**
  * The Connection service is used to declare the class
  * and connection options but does not connect quite yet. Connection
  * happens at a later lifecycle stage when a workflow
- * is started by the MemFlow Client module (`(new MemFlow.Client())).start()`).
+ * is started by the Durable Client module (`(new Durable.Client())).start()`).
  *
  * The config options optionall support a multi-connection setup
  * where the `store` connection explicitly defined along with `stream`, `sub`, etc.

package/build/services/{memflow → durable}/connection.js

@@ -5,7 +5,7 @@ exports.ConnectionService = void 0;
  * The Connection service is used to declare the class
  * and connection options but does not connect quite yet. Connection
  * happens at a later lifecycle stage when a workflow
- * is started by the MemFlow Client module (`(new MemFlow.Client())).start()`).
+ * is started by the Durable Client module (`(new Durable.Client())).start()`).
  *
  * The config options optionall support a multi-connection setup
  * where the `store` connection explicitly defined along with `stream`, `sub`, etc.

package/build/services/{memflow → durable}/exporter.d.ts

@@ -1,6 +1,6 @@
 import { ILogger } from '../logger';
 import { StoreService } from '../store';
-import { ExportOptions, MemFlowJobExport, TimelineType, TransitionType, ExportFields } from '../../types/exporter';
+import { ExportOptions, DurableJobExport, TimelineType, TransitionType, ExportFields } from '../../types/exporter';
 import { ProviderClient, ProviderTransaction } from '../../types/provider';
 import { StringStringType, Symbols } from '../../types/serializer';
 declare class ExporterService {
@@ -11,17 +11,17 @@ declare class ExporterService {
     private static symbols;
     constructor(appId: string, store: StoreService<ProviderClient, ProviderTransaction>, logger: ILogger);
     /**
-     * Convert the job hash from its compiles format into a MemFlowJobExport object with
+     * Convert the job hash from its compiles format into a DurableJobExport object with
      * facets that describe the workflow in terms relevant to narrative storytelling.
      */
-    export(jobId: string, options?: ExportOptions): Promise<MemFlowJobExport>;
+    export(jobId: string, options?: ExportOptions): Promise<DurableJobExport>;
     /**
-     * Inflates the job data into a MemFlowJobExport object
+     * Inflates the job data into a DurableJobExport object
      * @param jobHash - the job data
      * @param dependencyList - the list of dependencies for the job
      * @returns - the inflated job data
      */
-    inflate(jobHash: StringStringType, options: ExportOptions): MemFlowJobExport;
+    inflate(jobHash: StringStringType, options: ExportOptions): DurableJobExport;
     resolveValue(raw: string, withValues: boolean): Record<string, any> | string | number | null;
     /**
      * Inflates the key
@@ -30,7 +30,7 @@ declare class ExporterService {
      * @private
      */
     inflateKey(key: string): string;
-    filterFields(fullObject: MemFlowJobExport, block?: ExportFields[], allow?: ExportFields[]): Partial<MemFlowJobExport>;
+    filterFields(fullObject: DurableJobExport, block?: ExportFields[], allow?: ExportFields[]): Partial<DurableJobExport>;
     inflateTransition(match: RegExpMatchArray, value: string, transitionsObject: Record<string, TransitionType>): void;
     sortEntriesByCreated(obj: {
         [key: string]: TransitionType;

package/build/services/{memflow → durable}/exporter.js

@@ -10,7 +10,7 @@ class ExporterService {
         this.store = store;
     }
     /**
-     * Convert the job hash from its compiles format into a MemFlowJobExport object with
+     * Convert the job hash from its compiles format into a DurableJobExport object with
      * facets that describe the workflow in terms relevant to narrative storytelling.
      */
     async export(jobId, options = {}) {
@@ -23,7 +23,7 @@ class ExporterService {
         return jobExport;
     }
     /**
-     * Inflates the job data into a MemFlowJobExport object
+     * Inflates the job data into a DurableJobExport object
      * @param jobHash - the job data
      * @param dependencyList - the list of dependencies for the job
      * @returns - the inflated job data

package/build/services/{memflow → durable}/handle.d.ts

@@ -1,5 +1,5 @@
 import { HotMesh } from '../hotmesh';
-import { MemFlowJobExport, ExportOptions } from '../../types/exporter';
+import { DurableJobExport, ExportOptions } from '../../types/exporter';
 import { JobInterruptOptions } from '../../types/job';
 import { StreamError } from '../../types/stream';
 import { ExporterService } from './exporter';
@@ -7,7 +7,7 @@ import { ExporterService } from './exporter';
  * The WorkflowHandleService provides methods to interact with a running
  * workflow. This includes exporting the workflow, sending signals, and
  * querying the state of the workflow. It is instanced/accessed via the
- * MemFlow.Client class.
+ * Durable.Client class.
  *
  * @example
  * ```typescript
@@ -43,11 +43,11 @@ export declare class WorkflowHandleService {
     /**
      * Exports the workflow state to a JSON object.
      */
-    export(options?: ExportOptions): Promise<MemFlowJobExport>;
+    export(options?: ExportOptions): Promise<DurableJobExport>;
     /**
      * Sends a signal to the workflow. This is a way to send
      * a message to a workflow that is paused due to having
-     * executed `MemFlow.workflow.waitFor`. The workflow
+     * executed `Durable.workflow.waitFor`. The workflow
      * will awaken if no other signals are pending.
      */
     signal(signalId: string, data: Record<any, any>): Promise<void>;

package/build/services/{memflow → durable}/handle.js

@@ -6,7 +6,7 @@ const exporter_1 = require("./exporter");
  * The WorkflowHandleService provides methods to interact with a running
  * workflow. This includes exporting the workflow, sending signals, and
  * querying the state of the workflow. It is instanced/accessed via the
- * MemFlow.Client class.
+ * Durable.Client class.
  *
  * @example
  * ```typescript
@@ -46,7 +46,7 @@ class WorkflowHandleService {
     /**
      * Sends a signal to the workflow. This is a way to send
      * a message to a workflow that is paused due to having
-     * executed `MemFlow.workflow.waitFor`. The workflow
+     * executed `Durable.workflow.waitFor`. The workflow
      * will awaken if no other signals are pending.
      */
     async signal(signalId, data) {