@peopl-health/nexus 5.44.0-dev.5413 → 5.44.0-dev.5415

package/README.md

@@ -116,38 +116,6 @@ const bus = nexus.getMessaging().getEventBus();
 bus.on('message:received', (m) => console.log('rx', m.id));
 ```
 
-## Workflows (Optional)
-
-Register a named unit of work, deduplicated by a key you derive from the trigger.
-
-```js
-const runner = nexus.getMessaging().getWorkflowRunner();
-
-await runner.register({
-  kind: 'triage-projection',
-  dedupeKey: (trigger) => String(trigger.submissionId),
-  prepare: async (trigger, checkpoints, save) => { /* ... */ },
-  retrySchedule: [5, 10, 20, 40],   // minutes; omit for fire-and-forget
-});
-
-await runner.enqueue('triage-projection', { submissionId });
-```
-
-With `retrySchedule` the trigger and a checkpoint journal are stored in Mongo and the schedule runs
-on an in-process timer — deferred workflows never touch the queue adapter, so they add no Redis load.
-`prepare` resumes from `checkpoints`; call `save(patch)` before each external call so a retry does
-not repeat completed work. Throw an error carrying `permanent: true` to give up immediately instead
-of walking the schedule; `addRecord` and `addLinkedRecord` tag their own permanent 4xx errors that
-way. The read and update helpers (`getRecords`, `getRecordByFilter`, `updateRecordByFilter`) return
-`undefined` on failure instead of throwing, so `prepare` must check their results and throw itself —
-otherwise an outage reads as success and the work is marked done.
-
-Recovery is automatic: registering sweeps for work left behind by a previous process, and the runner
-keeps sweeping so work whose worker died mid-`prepare` is reclaimed once its claim goes stale.
-`enqueue` on an abandoned key revives it, keeping its checkpoints. Work is retained for 24 hours
-after its last update and then expires — the dedupe key expires with it, so idempotency has the
-same horizon as retention.
-
 ## Assistants (Optional)
 
 Register assistant classes and (optionally) a custom resolver. OpenAI is supported via `llm: 'openai'`.

package/lib/core/NexusMessaging.js

@@ -32,7 +32,6 @@ const { isLlmMonitorEnabled } = require('../clinical/flags/llmMonitorConfig');
 const { createQueueAdapter } = require('../queue');
 const { ScheduledMessageJob } = require('../jobs/ScheduledMessageJob');
 const { TemplateApprovalJob } = require('../jobs/TemplateApprovalJob');
-const { WorkflowRunner } = require('./WorkflowRunner');
 
 const { PhiProcessor } = require('./PhiProcessor');
 
@@ -117,8 +116,6 @@ class NexusMessaging {
       queueAdapter: this.queueAdapter,
     });
 
-    this.workflowRunner = new WorkflowRunner({ queueAdapter: this.queueAdapter });
-
     this.phiProcessor = new PhiProcessor({
       encode: config.phi?.encode || false,
       ner: config.phi?.ner || null,
@@ -289,7 +286,6 @@ class NexusMessaging {
   getAssistantProcessor() { return this.assistantProcessor; }
   getLlmMonitor() { return this.llmMonitor; }
   getQueueAdapter() { return this.queueAdapter; }
-  getWorkflowRunner() { return this.workflowRunner; }
   getPhiProcessor() { return this.phiProcessor; }
   isConnected() { return this.provider?.getConnectionStatus() ?? false; }
   isProcessing(chatId) { return this.batchingManager.isProcessing(chatId); }
@@ -721,7 +717,6 @@ class NexusMessaging {
   async disconnect() {
     this.queueReconciliation?.stop();
     if (this.provider) await this.provider.disconnect();
-    if (this.workflowRunner) this.workflowRunner.stop();
     if (this.queueAdapter) await this.queueAdapter.shutdown();
     this.events.removeAllListeners();
   }

package/lib/core/workflowRunner.js

@@ -0,0 +1,35 @@
+function createWorkflowRunner({ queueAdapter } = {}) {
+  if (!queueAdapter) throw new Error('createWorkflowRunner requires a queueAdapter');
+  const workflows = new Map();
+
+  async function register(workflow = {}) {
+    if (!workflow.kind || typeof workflow.kind !== 'string') throw new Error('workflow requires a kind');
+    if (workflow.kind.includes('__')) throw new Error(`workflow '${workflow.kind}' must not contain '__' (reserved as the jobId delimiter)`);
+    if (typeof workflow.dedupeKey !== 'function') throw new Error(`workflow '${workflow.kind}' requires a dedupeKey function`);
+    if (typeof workflow.prepare !== 'function') throw new Error(`workflow '${workflow.kind}' requires a prepare function`);
+    if (workflows.has(workflow.kind)) throw new Error(`workflow '${workflow.kind}' is already registered`);
+    workflows.set(workflow.kind, workflow);
+    try {
+      await queueAdapter.process(workflow.kind, (trigger) => workflow.prepare(trigger));
+    } catch (error) {
+      workflows.delete(workflow.kind);
+      throw error;
+    }
+    return workflow;
+  }
+
+  function enqueue(kind, trigger, options = {}) {
+    const workflow = workflows.get(kind);
+    if (!workflow) throw new Error(`no workflow registered for '${kind}'`);
+    const dedupeKey = workflow.dedupeKey(trigger);
+    if (dedupeKey == null || (typeof dedupeKey === 'string' && !dedupeKey.trim())) {
+      throw new Error(`workflow '${kind}' produced an empty dedupe key`);
+    }
+    if (typeof dedupeKey !== 'string') throw new Error(`workflow '${kind}' produced a non-string dedupe key`);
+    return queueAdapter.enqueue(kind, trigger, { ...options, jobId: `${kind}__${dedupeKey}` });
+  }
+
+  return { register, enqueue };
+}
+
+module.exports = { createWorkflowRunner };

package/lib/index.d.ts

@@ -242,7 +242,6 @@ declare module '@peopl-health/nexus' {
     getAssistantProcessor(): AssistantProcessor;
     getLlmMonitor(): { start(options?: { scheduleDaily?: boolean }): Promise<{ enabled: boolean }> } | null;
     initializeLlmMonitor(): Promise<{ enabled: boolean; error?: string } | undefined>;
-    getWorkflowRunner(): WorkflowRunner;
     processInstruction(code: string, instruction: string, role?: string, options?: { triggeredBy?: string }): Promise<void>;
     processSystemMessage(code: string, messages: string | string[], role?: string, options?: { triggeredBy?: string; reply?: boolean }): Promise<void>;
     isConnected(): boolean;
@@ -455,17 +454,16 @@ declare module '@peopl-health/nexus' {
   export interface Workflow {
     kind: string;
     dedupeKey: (trigger: any) => string;
-    prepare: (trigger: any, checkpoints?: any, save?: (patch: any) => Promise<void>) => Promise<any>;
-    retrySchedule?: number[];
+    prepare: (trigger: any) => Promise<any>;
   }
 
   export interface WorkflowRunner {
     register(workflow: Workflow): Promise<Workflow>;
-    enqueue(kind: string, trigger: any, options?: QueueJobOptions): Promise<string | null>;
-    sweep(only?: string[]): Promise<string[]>;
-    stop(): void;
+    enqueue(kind: string, trigger: any, options?: QueueJobOptions): Promise<string>;
   }
 
+  export function createWorkflowRunner(options: { queueAdapter: QueueAdapter }): WorkflowRunner;
+
   // Memory System
   export interface PatientMemoryDocument {
     _id: any;

package/lib/index.js

@@ -27,6 +27,7 @@ const { BaileysProvider } = require('./adapters/BaileysProvider');
 const { setPreprocessingHandler, hasPreprocessingHandler, invokePreprocessingHandler } = require('./services/preprocessingService');
 const { requestIdMiddleware, getRequestId } = require('./middleware/requestId');
 const { QueueAdapter, LocalQueueAdapter, RedisQueueAdapter, createQueueAdapter, registerQueueAdapter } = require('./queue');
+const { createWorkflowRunner } = require('./core/workflowRunner');
 const routes = require('./routes');
 const { resetAll } = require('./config/lifecycle');
 const { EvalProvider } = require('./eval/EvalProvider');
@@ -220,6 +221,7 @@ class Nexus {
 }
 
 module.exports = {
+  createWorkflowRunner,
   Nexus,
   TwilioProvider,
   BaileysProvider,

package/lib/services/airtableService.js

@@ -11,11 +11,6 @@ const TRANSIENT_ERROR_CODES = ['ETIMEDOUT', 'ECONNRESET', 'ECONNREFUSED', 'ENOTF
 const MAX_ATTEMPTS = 3;
 const RETRY_BASE_DELAY_MS = 1000;
 
-function isPermanentAirtableError(error) {
-  const status = error?.statusCode;
-  return typeof status === 'number' && status >= 400 && status < 500 && status !== 429;
-}
-
 let isEvalMode = false;
 
 function setEvalMode(enabled) {
@@ -43,7 +38,6 @@ async function withAirtableRetry(operation, label, { retryNetworkErrors = true }
       const isTransient = TRANSIENT_STATUS_CODES.includes(error.statusCode)
         || (retryNetworkErrors && isTransientNetworkError(error));
       if (!isTransient || attempt === MAX_ATTEMPTS) {
-        if (isPermanentAirtableError(error)) error.permanent = true;
         throw error;
       }
       const baseDelay = error.statusCode === RATE_LIMITED_STATUS ? RATE_LIMIT_DELAY_MS : RETRY_BASE_DELAY_MS;
@@ -175,7 +169,6 @@ async function addLinkedRecord(baseID, targetTable, fields, linkConfig, context
 
 module.exports = {
   setEvalMode,
-  isPermanentAirtableError,
   addRecord,
   getRecords,
   getRecordByFilter,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@peopl-health/nexus",
-  "version": "5.44.0-dev.5413",
+  "version": "5.44.0-dev.5415",
   "description": "Core messaging and assistant library for WhatsApp communication platforms",
   "keywords": [
     "whatsapp",

package/lib/core/WorkflowRunner.js

@@ -1,252 +0,0 @@
-const { logger } = require('../utils/logger');
-
-const { DeferredWork, RETENTION_MS } = require('../models/deferredWorkModel');
-
-const STALE_CLAIM_MS = 5 * 60 * 1000;
-const MINUTE_MS = 60 * 1000;
-const INDEX_TIMEOUT_MS = 1000;
-const SWEEP_INTERVAL_MS = 60 * 1000;
-
-let indexesReady = null;
-
-function ensureIndexes() {
-  if (!indexesReady) {
-    indexesReady = DeferredWork.init().catch((error) => {
-      indexesReady = null;
-      logger.error('[WorkflowRunner] Index build failed, dedupe and retention are not guaranteed', { error: error.message });
-    });
-  }
-  return indexesReady;
-}
-
-function withIndexTimeout(promise) {
-  let timer;
-  const timeout = new Promise((resolve) => {
-    timer = setTimeout(resolve, INDEX_TIMEOUT_MS);
-    timer.unref();
-  });
-  return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
-}
-
-class WorkflowRunner {
-  constructor({ queueAdapter, sweepIntervalMs = SWEEP_INTERVAL_MS } = {}) {
-    if (!queueAdapter) throw new Error('WorkflowRunner requires a queueAdapter');
-    this.queueAdapter = queueAdapter;
-    this.sweepIntervalMs = sweepIntervalMs;
-    this.sweepTimer = null;
-    this.timers = new Map();
-    this.workflows = new Map();
-  }
-
-  async register(workflow = {}) {
-    if (!workflow.kind || typeof workflow.kind !== 'string') throw new Error('workflow requires a kind');
-    if (workflow.kind.includes('__')) throw new Error(`workflow '${workflow.kind}' must not contain '__' (reserved as the jobId delimiter)`);
-    if (typeof workflow.dedupeKey !== 'function') throw new Error(`workflow '${workflow.kind}' requires a dedupeKey function`);
-    if (typeof workflow.prepare !== 'function') throw new Error(`workflow '${workflow.kind}' requires a prepare function`);
-    if (this.workflows.has(workflow.kind)) throw new Error(`workflow '${workflow.kind}' is already registered`);
-    if (workflow.retrySchedule !== undefined) {
-      if (!Array.isArray(workflow.retrySchedule) || !workflow.retrySchedule.length) {
-        throw new Error(`workflow '${workflow.kind}' requires a non-empty retrySchedule array`);
-      }
-      if (workflow.retrySchedule.some((minutes) => !Number.isFinite(minutes) || minutes <= 0)) {
-        throw new Error(`workflow '${workflow.kind}' requires retrySchedule entries to be positive minutes`);
-      }
-      if (workflow.retrySchedule.some((minutes) => minutes * MINUTE_MS >= RETENTION_MS)) {
-        throw new Error(`workflow '${workflow.kind}' requires retrySchedule entries under the ${RETENTION_MS / MINUTE_MS}m retention, or the record expires before the attempt fires`);
-      }
-    }
-
-    this.workflows.set(workflow.kind, workflow);
-    if (workflow.retrySchedule) {
-      await this.sweep([workflow.kind]).catch((error) => {
-        logger.error('[WorkflowRunner] Sweep on register failed', { kind: workflow.kind, error: error.message });
-      });
-      this._startSweeping();
-      return workflow;
-    }
-
-    try {
-      await this.queueAdapter.process(workflow.kind, (trigger) => workflow.prepare(trigger));
-    } catch (error) {
-      this.workflows.delete(workflow.kind);
-      throw error;
-    }
-    return workflow;
-  }
-
-  enqueue(kind, trigger, options = {}) {
-    const workflow = this.workflows.get(kind);
-    if (!workflow) throw new Error(`no workflow registered for '${kind}'`);
-    const dedupeKey = workflow.dedupeKey(trigger);
-    if (dedupeKey == null || (typeof dedupeKey === 'string' && !dedupeKey.trim())) {
-      throw new Error(`workflow '${kind}' produced an empty dedupe key`);
-    }
-    if (typeof dedupeKey !== 'string') throw new Error(`workflow '${kind}' produced a non-string dedupe key`);
-    if (!workflow.retrySchedule) {
-      return this.queueAdapter.enqueue(kind, trigger, { ...options, jobId: `${kind}__${dedupeKey}` });
-    }
-    return this._armDeferred(workflow, trigger, dedupeKey);
-  }
-
-  async sweep(only = null) {
-    const deferredKinds = [...this.workflows.values()].filter((w) => w.retrySchedule).map((w) => w.kind);
-    const kinds = only ? deferredKinds.filter((kind) => only.includes(kind)) : deferredKinds;
-    if (!kinds.length) return [];
-
-    const orphaned = await DeferredWork.find({
-      kind: { $in: kinds },
-      $or: [
-        { status: 'pending' },
-        { status: 'processing', claimedAt: { $lt: new Date(Date.now() - STALE_CLAIM_MS) } }
-      ]
-    });
-
-    const rearmed = orphaned.map((work) => this._arm(
-      this.workflows.get(work.kind),
-      work._id,
-      this._delayUntilDue(work),
-    ));
-    if (rearmed.length) logger.info('[WorkflowRunner] Swept', { kinds, rearmed: rearmed.length });
-    return rearmed;
-  }
-
-  _startSweeping() {
-    if (this.sweepTimer || !this.sweepIntervalMs) return;
-    this.sweepTimer = setInterval(() => {
-      this.sweep().catch((error) => logger.error('[WorkflowRunner] Periodic sweep failed', { error: error.message }));
-    }, this.sweepIntervalMs);
-    this.sweepTimer.unref();
-  }
-
-  stop() {
-    if (this.sweepTimer) clearInterval(this.sweepTimer);
-    this.sweepTimer = null;
-    for (const timer of this.timers.values()) clearTimeout(timer);
-    this.timers.clear();
-  }
-
-  async _armDeferred(workflow, trigger, dedupeKey) {
-    const { kind } = workflow;
-    await withIndexTimeout(ensureIndexes());
-    const work = await DeferredWork.findOne({ kind, dedupeKey })
-      || await DeferredWork.create({ kind, dedupeKey, trigger }).catch(async (error) => {
-        if (error.code !== 11000) throw error;
-        return await DeferredWork.findOne({ kind, dedupeKey });
-      });
-    if (work.status === 'abandoned') {
-      const { modifiedCount } = await DeferredWork.updateOne(
-        { _id: work._id, status: 'abandoned' },
-        { $set: { status: 'pending', attempt: 0, nextAttemptAt: new Date(), lastError: null, claimedAt: null } }
-      );
-      if (!modifiedCount) return null;
-      logger.info('[WorkflowRunner] Re-arming abandoned work', { kind, dedupeKey });
-      return this._arm(workflow, work._id, 0);
-    }
-    if (work.status !== 'pending') {
-      logger.info('[WorkflowRunner] Skipping, work is in flight or complete', { kind, dedupeKey, status: work.status });
-      return null;
-    }
-    return this._arm(workflow, work._id, this._delayUntilDue(work));
-  }
-
-  _delayUntilDue(work) {
-    return Math.max(0, new Date(work.nextAttemptAt).getTime() - Date.now());
-  }
-
-  _arm(workflow, workId, delayMs) {
-    const id = String(workId);
-    clearTimeout(this.timers.get(id));
-    const timer = setTimeout(() => {
-      this.timers.delete(id);
-      this._runDeferred(workflow, id).catch((error) => {
-        logger.error('[WorkflowRunner] Deferred run failed', { kind: workflow.kind, deferredWorkId: id, error: error.message });
-      });
-    }, delayMs);
-    timer.unref();
-    this.timers.set(id, timer);
-    return id;
-  }
-
-  async _runDeferred(workflow, deferredWorkId) {
-    const claimedAt = new Date();
-    const prior = await DeferredWork.findOneAndUpdate(
-      {
-        _id: deferredWorkId,
-        $or: [
-          { status: 'pending', nextAttemptAt: { $lte: claimedAt } },
-          { status: 'processing', claimedAt: { $lt: new Date(claimedAt.getTime() - STALE_CLAIM_MS) } }
-        ]
-      },
-      { $set: { status: 'processing', claimedAt } },
-      { new: false }
-    );
-    if (!prior) {
-      logger.info('[WorkflowRunner] Claimed elsewhere or not due', { kind: workflow.kind, deferredWorkId });
-      return { claimed: false };
-    }
-
-    const reclaimed = prior.status === 'processing';
-    const work = { ...prior.toObject(), attempt: prior.attempt + (reclaimed ? 1 : 0) };
-    if (reclaimed) await this._write(work, claimedAt, { attempt: work.attempt });
-
-    const checkpoints = { ...(work.checkpoints || {}) };
-    const save = async (patch) => {
-      Object.assign(checkpoints, patch);
-      const $set = {};
-      for (const [key, value] of Object.entries(patch || {})) $set[`checkpoints.${key}`] = value;
-      if (Object.keys($set).length) await this._write(work, claimedAt, $set);
-    };
-
-    try {
-      await workflow.prepare(work.trigger, checkpoints, save);
-    } catch (error) {
-      return await this._rescheduleOrAbandon(workflow, work, claimedAt, error);
-    }
-    await this._write(work, claimedAt, { status: 'done', claimedAt: null });
-    return { status: 'done' };
-  }
-
-  async _write(work, claimedAt, $set) {
-    const { modifiedCount } = await DeferredWork.updateOne({ _id: work._id, claimedAt }, { $set });
-    if (!modifiedCount) logger.warn('[WorkflowRunner] Claim lost, write discarded', { deferredWorkId: String(work._id) });
-    return modifiedCount > 0;
-  }
-
-  async _rescheduleOrAbandon(workflow, work, claimedAt, error) {
-    const delayMinutes = workflow.retrySchedule[work.attempt];
-    if (error.permanent === true || delayMinutes === undefined) {
-      await this._write(work, claimedAt, { status: 'abandoned', lastError: error.message, claimedAt: null });
-      logger.error('[WorkflowRunner] Abandoned', {
-        kind: workflow.kind,
-        dedupeKey: work.dedupeKey,
-        attempt: work.attempt,
-        permanent: error.permanent === true,
-        error: error.message
-      });
-      return { status: 'abandoned' };
-    }
-
-    const attempt = work.attempt + 1;
-    const delayMs = delayMinutes * MINUTE_MS;
-    const kept = await this._write(work, claimedAt, {
-      status: 'pending',
-      attempt,
-      nextAttemptAt: new Date(Date.now() + delayMs),
-      lastError: error.message,
-      claimedAt: null
-    });
-    if (!kept) return { claimed: false };
-
-    logger.warn('[WorkflowRunner] Rescheduled', {
-      kind: workflow.kind,
-      dedupeKey: work.dedupeKey,
-      attempt,
-      delayMinutes,
-      error: error.message
-    });
-    this._arm(workflow, work._id, delayMs);
-    return { status: 'pending', attempt };
-  }
-}
-
-module.exports = { WorkflowRunner };

package/lib/models/deferredWorkModel.js

@@ -1,24 +0,0 @@
-const mongoose = require('mongoose');
-
-const STATUS_VALUES = ['pending', 'processing', 'done', 'abandoned'];
-const RETENTION_MS = 24 * 60 * 60 * 1000;
-
-const deferredWorkSchema = new mongoose.Schema({
-  kind: { type: String, required: true },
-  dedupeKey: { type: String, required: true },
-  trigger: { type: mongoose.Schema.Types.Mixed, default: null },
-  checkpoints: { type: mongoose.Schema.Types.Mixed, default: () => ({}) },
-  status: { type: String, enum: STATUS_VALUES, default: 'pending' },
-  attempt: { type: Number, default: 0 },
-  nextAttemptAt: { type: Date, default: Date.now },
-  claimedAt: { type: Date, default: null },
-  lastError: { type: String, default: null }
-}, { timestamps: true });
-
-deferredWorkSchema.index({ kind: 1, dedupeKey: 1 }, { unique: true, name: 'dedupe_idx' });
-deferredWorkSchema.index({ status: 1, nextAttemptAt: 1 }, { name: 'due_idx' });
-deferredWorkSchema.index({ updatedAt: 1 }, { expireAfterSeconds: RETENTION_MS / 1000, name: 'ttl_idx' });
-
-const DeferredWork = mongoose.model('DeferredWork', deferredWorkSchema);
-
-module.exports = { DeferredWork, RETENTION_MS };