@peopl-health/nexus 5.45.0-dev.7136 → 5.45.0-dev.7142

package/README.md

@@ -116,6 +116,38 @@ const bus = nexus.getMessaging().getEventBus();
 bus.on('message:received', (m) => console.log('rx', m.id));
 ```
 
+## Workflows (Optional)
+
+Register a named unit of work, deduplicated by a key you derive from the trigger.
+
+```js
+const runner = nexus.getMessaging().getWorkflowRunner();
+
+await runner.register({
+  kind: 'triage-projection',
+  dedupeKey: (trigger) => String(trigger.submissionId),
+  prepare: async (trigger, checkpoints, save) => { /* ... */ },
+  retrySchedule: [5, 10, 20, 40],   // minutes; omit for fire-and-forget
+});
+
+await runner.enqueue('triage-projection', { submissionId });
+```
+
+With `retrySchedule` the trigger and a checkpoint journal are stored in Mongo and the schedule runs
+on an in-process timer — deferred workflows never touch the queue adapter, so they add no Redis load.
+`prepare` resumes from `checkpoints`; call `save(patch)` before each external call so a retry does
+not repeat completed work. Throw an error carrying `permanent: true` to give up immediately instead
+of walking the schedule; the Airtable helpers tag their own permanent 4xx errors that way, in
+`withAirtableRetry`, which all five of them route through. They rethrow on failure too, so an
+Airtable outage reaches `prepare` as an exception and walks the retry schedule on its own rather
+than reading as success.
+
+Recovery is automatic: registering sweeps for work left behind by a previous process, and the runner
+keeps sweeping so work whose worker died mid-`prepare` is reclaimed once its claim goes stale.
+`enqueue` on an abandoned key revives it, keeping its checkpoints. Work is retained for 24 hours
+after its last update and then expires — the dedupe key expires with it, so idempotency has the
+same horizon as retention.
+
 ## Assistants (Optional)
 
 Register assistant classes and (optionally) a custom resolver. OpenAI is supported via `llm: 'openai'`.

package/lib/core/NexusMessaging.js

@@ -43,6 +43,7 @@ const { isLlmMonitorEnabled } = require('../clinical/flags/llmMonitorConfig');
 const { createQueueAdapter } = require('../queue');
 const { ScheduledMessageJob } = require('../jobs/ScheduledMessageJob');
 const { TemplateApprovalJob } = require('../jobs/TemplateApprovalJob');
+const { WorkflowRunner } = require('./WorkflowRunner');
 
 const { PhiProcessor } = require('./PhiProcessor');
 
@@ -127,6 +128,8 @@ class NexusMessaging {
       queueAdapter: this.queueAdapter,
     });
 
+    this.workflowRunner = new WorkflowRunner({ queueAdapter: this.queueAdapter });
+
     this.phiProcessor = new PhiProcessor({
       encode: config.phi?.encode || false,
       ner: config.phi?.ner || null,
@@ -310,6 +313,7 @@ class NexusMessaging {
   getAssistantProcessor() { return this.assistantProcessor; }
   getLlmMonitor() { return this.llmMonitor; }
   getQueueAdapter() { return this.queueAdapter; }
+  getWorkflowRunner() { return this.workflowRunner; }
   getPhiProcessor() { return this.phiProcessor; }
   isConnected() { return this.provider?.getConnectionStatus() ?? false; }
   isProcessing(chatId) { return this.batchingManager.isProcessing(chatId); }
@@ -1012,6 +1016,7 @@ class NexusMessaging {
   async disconnect() {
     this.queueReconciliation?.stop();
     if (this.provider) await this.provider.disconnect();
+    if (this.workflowRunner) this.workflowRunner.stop();
     if (this.queueAdapter) await this.queueAdapter.shutdown();
     this.events.removeAllListeners();
   }

package/lib/core/WorkflowRunner.js

@@ -0,0 +1,265 @@
+const { describeThrown } = require('../utils/errorUtils');
+const { logger } = require('../utils/logger');
+
+const { DeferredWork, RETENTION_MS } = require('../models/deferredWorkModel');
+
+const STALE_CLAIM_MS = 5 * 60 * 1000;
+const MINUTE_MS = 60 * 1000;
+const INDEX_TIMEOUT_MS = 1000;
+const SWEEP_INTERVAL_MS = 60 * 1000;
+
+let indexesReady = null;
+
+function ensureIndexes() {
+  if (!indexesReady) {
+    indexesReady = DeferredWork.init().catch((error) => {
+      indexesReady = null;
+      logger.error('[WorkflowRunner] Index build failed, dedupe and retention are not guaranteed', { error: error.message });
+    });
+  }
+  return indexesReady;
+}
+
+function withIndexTimeout(promise) {
+  let timer;
+  const timeout = new Promise((resolve) => {
+    timer = setTimeout(resolve, INDEX_TIMEOUT_MS);
+    timer.unref();
+  });
+  return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
+}
+
+class WorkflowRunner {
+  constructor({ queueAdapter, sweepIntervalMs = SWEEP_INTERVAL_MS } = {}) {
+    if (!queueAdapter) throw new Error('WorkflowRunner requires a queueAdapter');
+    this.queueAdapter = queueAdapter;
+    this.sweepIntervalMs = sweepIntervalMs;
+    this.sweepTimer = null;
+    this.timers = new Map();
+    this.workflows = new Map();
+    this.stopped = false;
+  }
+
+  async register(workflow = {}) {
+    if (!workflow.kind || typeof workflow.kind !== 'string') throw new Error('workflow requires a kind');
+    if (workflow.kind.includes('__')) throw new Error(`workflow '${workflow.kind}' must not contain '__' (reserved as the jobId delimiter)`);
+    if (typeof workflow.dedupeKey !== 'function') throw new Error(`workflow '${workflow.kind}' requires a dedupeKey function`);
+    if (typeof workflow.prepare !== 'function') throw new Error(`workflow '${workflow.kind}' requires a prepare function`);
+    if (this.workflows.has(workflow.kind)) throw new Error(`workflow '${workflow.kind}' is already registered`);
+    if (workflow.retrySchedule !== undefined) {
+      if (!Array.isArray(workflow.retrySchedule) || !workflow.retrySchedule.length) {
+        throw new Error(`workflow '${workflow.kind}' requires a non-empty retrySchedule array`);
+      }
+      if (workflow.retrySchedule.some((minutes) => !Number.isFinite(minutes) || minutes <= 0)) {
+        throw new Error(`workflow '${workflow.kind}' requires retrySchedule entries to be positive minutes`);
+      }
+      if (workflow.retrySchedule.some((minutes) => minutes * MINUTE_MS >= RETENTION_MS)) {
+        throw new Error(`workflow '${workflow.kind}' requires retrySchedule entries under the ${RETENTION_MS / MINUTE_MS}m retention, or the record expires before the attempt fires`);
+      }
+    }
+
+    this.workflows.set(workflow.kind, workflow);
+    if (workflow.retrySchedule) {
+      await this.sweep([workflow.kind]).catch((error) => {
+        logger.error('[WorkflowRunner] Sweep on register failed', { kind: workflow.kind, error: error.message });
+      });
+      this._startSweeping();
+      return workflow;
+    }
+
+    try {
+      await this.queueAdapter.process(workflow.kind, (trigger) => workflow.prepare(trigger));
+    } catch (error) {
+      this.workflows.delete(workflow.kind);
+      throw error;
+    }
+    return workflow;
+  }
+
+  enqueue(kind, trigger, options = {}) {
+    const workflow = this.workflows.get(kind);
+    if (!workflow) throw new Error(`no workflow registered for '${kind}'`);
+    const dedupeKey = workflow.dedupeKey(trigger);
+    if (dedupeKey == null || (typeof dedupeKey === 'string' && !dedupeKey.trim())) {
+      throw new Error(`workflow '${kind}' produced an empty dedupe key`);
+    }
+    if (typeof dedupeKey !== 'string') throw new Error(`workflow '${kind}' produced a non-string dedupe key`);
+    if (!workflow.retrySchedule) {
+      return this.queueAdapter.enqueue(kind, trigger, { ...options, jobId: `${kind}__${dedupeKey}` });
+    }
+    return this._armDeferred(workflow, trigger, dedupeKey);
+  }
+
+  async sweep(only = null) {
+    const deferredKinds = [...this.workflows.values()].filter((w) => w.retrySchedule).map((w) => w.kind);
+    const kinds = only ? deferredKinds.filter((kind) => only.includes(kind)) : deferredKinds;
+    if (!kinds.length) return [];
+
+    const orphaned = await DeferredWork.find({
+      kind: { $in: kinds },
+      $or: [
+        { status: 'pending' },
+        { status: 'processing', claimedAt: { $lt: new Date(Date.now() - STALE_CLAIM_MS) } }
+      ]
+    });
+
+    const rearmed = orphaned.map((work) => this._arm(
+      this.workflows.get(work.kind),
+      work._id,
+      this._delayUntilDue(work),
+    ));
+    if (rearmed.length) logger.info('[WorkflowRunner] Swept', { kinds, rearmed: rearmed.length });
+    return rearmed;
+  }
+
+  _startSweeping() {
+    if (this.stopped || this.sweepTimer || !this.sweepIntervalMs) return;
+    this.sweepTimer = setInterval(() => {
+      this.sweep().catch((error) => logger.error('[WorkflowRunner] Periodic sweep failed', { error: error.message }));
+    }, this.sweepIntervalMs);
+    this.sweepTimer.unref();
+  }
+
+  stop() {
+    this.stopped = true;
+    if (this.sweepTimer) clearInterval(this.sweepTimer);
+    this.sweepTimer = null;
+    for (const timer of this.timers.values()) clearTimeout(timer);
+    this.timers.clear();
+  }
+
+  async _armDeferred(workflow, trigger, dedupeKey) {
+    const { kind } = workflow;
+    await withIndexTimeout(ensureIndexes());
+    const work = await DeferredWork.findOne({ kind, dedupeKey })
+      || await DeferredWork.create({ kind, dedupeKey, trigger }).catch(async (error) => {
+        if (error.code !== 11000) throw error;
+        return await DeferredWork.findOne({ kind, dedupeKey });
+      });
+    if (work.status === 'abandoned') {
+      const { modifiedCount } = await DeferredWork.updateOne(
+        { _id: work._id, status: 'abandoned' },
+        { $set: { status: 'pending', attempt: 0, nextAttemptAt: new Date(), lastError: null, claimedAt: null } }
+      );
+      if (!modifiedCount) return null;
+      logger.info('[WorkflowRunner] Re-arming abandoned work', { kind, dedupeKey });
+      return this._arm(workflow, work._id, 0);
+    }
+    if (work.status !== 'pending') {
+      logger.info('[WorkflowRunner] Skipping, work is in flight or complete', { kind, dedupeKey, status: work.status });
+      return null;
+    }
+    return this._arm(workflow, work._id, this._delayUntilDue(work));
+  }
+
+  _delayUntilDue(work) {
+    return Math.max(0, new Date(work.nextAttemptAt).getTime() - Date.now());
+  }
+
+  _arm(workflow, workId, delayMs) {
+    const id = String(workId);
+    // An attempt still in flight when stop() lands must not re-arm after shutdown. The
+    // record stays pending in Mongo, so the next process recovers it on its sweep.
+    if (this.stopped) return id;
+    clearTimeout(this.timers.get(id));
+    const timer = setTimeout(() => {
+      this.timers.delete(id);
+      this._runDeferred(workflow, id).catch((error) => {
+        logger.error('[WorkflowRunner] Deferred run failed', { kind: workflow.kind, deferredWorkId: id, error: error.message });
+      });
+    }, delayMs);
+    timer.unref();
+    this.timers.set(id, timer);
+    return id;
+  }
+
+  async _runDeferred(workflow, deferredWorkId) {
+    const claimedAt = new Date();
+    const prior = await DeferredWork.findOneAndUpdate(
+      {
+        _id: deferredWorkId,
+        $or: [
+          { status: 'pending', nextAttemptAt: { $lte: claimedAt } },
+          { status: 'processing', claimedAt: { $lt: new Date(claimedAt.getTime() - STALE_CLAIM_MS) } }
+        ]
+      },
+      { $set: { status: 'processing', claimedAt } },
+      { new: false }
+    );
+    if (!prior) {
+      logger.info('[WorkflowRunner] Claimed elsewhere or not due', { kind: workflow.kind, deferredWorkId });
+      return { claimed: false };
+    }
+
+    const reclaimed = prior.status === 'processing';
+    const work = { ...prior.toObject(), attempt: prior.attempt + (reclaimed ? 1 : 0) };
+    if (reclaimed) await this._write(work, claimedAt, { attempt: work.attempt });
+
+    const checkpoints = { ...(work.checkpoints || {}) };
+    const save = async (patch) => {
+      Object.assign(checkpoints, patch);
+      const $set = {};
+      for (const [key, value] of Object.entries(patch || {})) $set[`checkpoints.${key}`] = value;
+      if (Object.keys($set).length) await this._write(work, claimedAt, $set);
+    };
+
+    try {
+      await workflow.prepare(work.trigger, checkpoints, save);
+    } catch (error) {
+      return await this._rescheduleOrAbandon(workflow, work, claimedAt, error);
+    }
+    await this._write(work, claimedAt, { status: 'done', claimedAt: null });
+    return { status: 'done' };
+  }
+
+  // matchedCount, not modifiedCount: the fence is "do we still hold the claim", and
+  // modifiedCount is also 0 for a write that matched but set an identical value - a
+  // save() rewriting an unchanged checkpoint is not a lost claim.
+  async _write(work, claimedAt, $set) {
+    const { matchedCount } = await DeferredWork.updateOne({ _id: work._id, claimedAt }, { $set });
+    if (!matchedCount) logger.warn('[WorkflowRunner] Claim lost, write discarded', { deferredWorkId: String(work._id) });
+    return matchedCount > 0;
+  }
+
+  async _rescheduleOrAbandon(workflow, work, claimedAt, error) {
+    // prepare() is consumer code and can throw anything, including null. Dereferencing
+    // it here used to raise a TypeError out of this handler, so neither branch ran and
+    // the record stayed `processing` until stale recovery hit the same failure again.
+    const lastError = describeThrown(error);
+    const delayMinutes = workflow.retrySchedule[work.attempt];
+    if (error?.permanent === true || delayMinutes === undefined) {
+      await this._write(work, claimedAt, { status: 'abandoned', lastError, claimedAt: null });
+      logger.error('[WorkflowRunner] Abandoned', {
+        kind: workflow.kind,
+        dedupeKey: work.dedupeKey,
+        attempt: work.attempt,
+        permanent: error?.permanent === true,
+        error: lastError
+      });
+      return { status: 'abandoned' };
+    }
+
+    const attempt = work.attempt + 1;
+    const delayMs = delayMinutes * MINUTE_MS;
+    const kept = await this._write(work, claimedAt, {
+      status: 'pending',
+      attempt,
+      nextAttemptAt: new Date(Date.now() + delayMs),
+      lastError,
+      claimedAt: null
+    });
+    if (!kept) return { claimed: false };
+
+    logger.warn('[WorkflowRunner] Rescheduled', {
+      kind: workflow.kind,
+      dedupeKey: work.dedupeKey,
+      attempt,
+      delayMinutes,
+      error: lastError
+    });
+    this._arm(workflow, work._id, delayMs);
+    return { status: 'pending', attempt };
+  }
+}
+
+module.exports = { WorkflowRunner };

package/lib/index.d.ts

@@ -243,6 +243,7 @@ declare module '@peopl-health/nexus' {
     getAssistantProcessor(): AssistantProcessor;
     getLlmMonitor(): { start(options?: { scheduleDaily?: boolean }): Promise<{ enabled: boolean }> } | null;
     initializeLlmMonitor(): Promise<{ enabled: boolean; error?: string } | undefined>;
+    getWorkflowRunner(): WorkflowRunner;
     processInstruction(code: string, instruction: string, role?: string, options?: { triggeredBy?: string }): Promise<void>;
     processSystemMessage(code: string, messages: string | string[], role?: string, options?: { triggeredBy?: string; reply?: boolean }): Promise<void>;
     processOutreach(code: string, options?: { brief?: OutreachBrief | null; triggeredBy?: string | null; reason?: string | null; firstName?: string | null; dryRun?: boolean }): Promise<OutreachDecision>;
@@ -520,16 +521,17 @@ declare module '@peopl-health/nexus' {
   export interface Workflow {
     kind: string;
     dedupeKey: (trigger: any) => string;
-    prepare: (trigger: any) => Promise<any>;
+    prepare: (trigger: any, checkpoints?: any, save?: (patch: any) => Promise<void>) => Promise<any>;
+    retrySchedule?: number[];
   }
 
   export interface WorkflowRunner {
     register(workflow: Workflow): Promise<Workflow>;
-    enqueue(kind: string, trigger: any, options?: QueueJobOptions): Promise<string>;
+    enqueue(kind: string, trigger: any, options?: QueueJobOptions): Promise<string | null>;
+    sweep(only?: string[]): Promise<string[]>;
+    stop(): void;
   }
 
-  export function createWorkflowRunner(options: { queueAdapter: QueueAdapter }): WorkflowRunner;
-
   // Memory System
   export interface PatientMemoryDocument {
     _id: any;

package/lib/index.js

@@ -27,7 +27,6 @@ const { BaileysProvider } = require('./adapters/BaileysProvider');
 const { setPreprocessingHandler, hasPreprocessingHandler, invokePreprocessingHandler } = require('./services/preprocessingService');
 const { requestIdMiddleware, getRequestId } = require('./middleware/requestId');
 const { QueueAdapter, LocalQueueAdapter, RedisQueueAdapter, createQueueAdapter, registerQueueAdapter } = require('./queue');
-const { createWorkflowRunner } = require('./core/workflowRunner');
 const routes = require('./routes');
 const { resetAll } = require('./config/lifecycle');
 const { EvalProvider } = require('./eval/EvalProvider');
@@ -229,7 +228,6 @@ class Nexus {
 }
 
 module.exports = {
-  createWorkflowRunner,
   Nexus,
   TwilioProvider,
   BaileysProvider,

package/lib/models/deferredWorkModel.js

@@ -0,0 +1,24 @@
+const mongoose = require('mongoose');
+
+const STATUS_VALUES = ['pending', 'processing', 'done', 'abandoned'];
+const RETENTION_MS = 24 * 60 * 60 * 1000;
+
+const deferredWorkSchema = new mongoose.Schema({
+  kind: { type: String, required: true },
+  dedupeKey: { type: String, required: true },
+  trigger: { type: mongoose.Schema.Types.Mixed, default: null },
+  checkpoints: { type: mongoose.Schema.Types.Mixed, default: () => ({}) },
+  status: { type: String, enum: STATUS_VALUES, default: 'pending' },
+  attempt: { type: Number, default: 0 },
+  nextAttemptAt: { type: Date, default: Date.now },
+  claimedAt: { type: Date, default: null },
+  lastError: { type: String, default: null }
+}, { timestamps: true });
+
+deferredWorkSchema.index({ kind: 1, dedupeKey: 1 }, { unique: true, name: 'dedupe_idx' });
+deferredWorkSchema.index({ status: 1, nextAttemptAt: 1 }, { name: 'due_idx' });
+deferredWorkSchema.index({ updatedAt: 1 }, { expireAfterSeconds: RETENTION_MS / 1000, name: 'ttl_idx' });
+
+const DeferredWork = mongoose.model('DeferredWork', deferredWorkSchema);
+
+module.exports = { DeferredWork, RETENTION_MS };

package/lib/services/airtableService.js

@@ -15,6 +15,11 @@ const AIRTABLE_API_URL = 'https://api.airtable.com/v0';
 const AXIOS_TIMEOUT_CODE = 'ECONNABORTED';
 const RETRY_BASE_DELAY_MS = 1000;
 
+function isPermanentAirtableError(error) {
+  const status = error?.statusCode;
+  return typeof status === 'number' && status >= 400 && status < 500 && status !== 429;
+}
+
 let isEvalMode = false;
 
 function setEvalMode(enabled) {
@@ -42,6 +47,7 @@ async function withAirtableRetry(operation, label, { retryNetworkErrors = true }
       const isTransient = TRANSIENT_STATUS_CODES.includes(error.statusCode)
         || (retryNetworkErrors && isTransientNetworkError(error));
       if (!isTransient || attempt === MAX_ATTEMPTS) {
+        if (isPermanentAirtableError(error)) error.permanent = true;
         throw error;
       }
       const baseDelay = error.statusCode === RATE_LIMITED_STATUS ? RATE_LIMIT_DELAY_MS : RETRY_BASE_DELAY_MS;
@@ -229,6 +235,7 @@ async function upsertRecord(baseID, tableName, fields, { mergeOn } = {}, context
 
 module.exports = {
   setEvalMode,
+  isPermanentAirtableError,
   addRecord,
   getRecords,
   getRecordByFilter,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@peopl-health/nexus",
-  "version": "5.45.0-dev.7136",
+  "version": "5.45.0-dev.7142",
   "description": "Core messaging and assistant library for WhatsApp communication platforms",
   "keywords": [
     "whatsapp",

package/lib/core/workflowRunner.js

@@ -1,35 +0,0 @@
-function createWorkflowRunner({ queueAdapter } = {}) {
-  if (!queueAdapter) throw new Error('createWorkflowRunner requires a queueAdapter');
-  const workflows = new Map();
-
-  async function register(workflow = {}) {
-    if (!workflow.kind || typeof workflow.kind !== 'string') throw new Error('workflow requires a kind');
-    if (workflow.kind.includes('__')) throw new Error(`workflow '${workflow.kind}' must not contain '__' (reserved as the jobId delimiter)`);
-    if (typeof workflow.dedupeKey !== 'function') throw new Error(`workflow '${workflow.kind}' requires a dedupeKey function`);
-    if (typeof workflow.prepare !== 'function') throw new Error(`workflow '${workflow.kind}' requires a prepare function`);
-    if (workflows.has(workflow.kind)) throw new Error(`workflow '${workflow.kind}' is already registered`);
-    workflows.set(workflow.kind, workflow);
-    try {
-      await queueAdapter.process(workflow.kind, (trigger) => workflow.prepare(trigger));
-    } catch (error) {
-      workflows.delete(workflow.kind);
-      throw error;
-    }
-    return workflow;
-  }
-
-  function enqueue(kind, trigger, options = {}) {
-    const workflow = workflows.get(kind);
-    if (!workflow) throw new Error(`no workflow registered for '${kind}'`);
-    const dedupeKey = workflow.dedupeKey(trigger);
-    if (dedupeKey == null || (typeof dedupeKey === 'string' && !dedupeKey.trim())) {
-      throw new Error(`workflow '${kind}' produced an empty dedupe key`);
-    }
-    if (typeof dedupeKey !== 'string') throw new Error(`workflow '${kind}' produced a non-string dedupe key`);
-    return queueAdapter.enqueue(kind, trigger, { ...options, jobId: `${kind}__${dedupeKey}` });
-  }
-
-  return { register, enqueue };
-}
-
-module.exports = { createWorkflowRunner };