@peopl-health/nexus 5.54.0-dev.7893 → 5.54.0-dev.7895

package/README.md

@@ -116,38 +116,6 @@ const bus = nexus.getMessaging().getEventBus();
 bus.on('message:received', (m) => console.log('rx', m.id));
 ```
 
-## Workflows (Optional)
-
-Register a named unit of work, deduplicated by a key you derive from the trigger.
-
-```js
-const runner = nexus.getMessaging().getWorkflowRunner();
-
-await runner.register({
-  kind: 'triage-projection',
-  dedupeKey: (trigger) => String(trigger.submissionId),
-  prepare: async (trigger, checkpoints, save) => { /* ... */ },
-  retrySchedule: [5, 10, 20, 40],   // minutes; omit for fire-and-forget
-});
-
-await runner.enqueue('triage-projection', { submissionId });
-```
-
-With `retrySchedule` the trigger and a checkpoint journal are stored in Mongo and the schedule runs
-on an in-process timer — deferred workflows never touch the queue adapter, so they add no Redis load.
-`prepare` resumes from `checkpoints`; call `save(patch)` before each external call so a retry does
-not repeat completed work. Throw an error carrying `permanent: true` to give up immediately instead
-of walking the schedule; the Airtable helpers tag their own permanent 4xx errors that way, in
-`withAirtableRetry`, which all five of them route through. They rethrow on failure too, so an
-Airtable outage reaches `prepare` as an exception and walks the retry schedule on its own rather
-than reading as success.
-
-Recovery is automatic: registering sweeps for work left behind by a previous process, and the runner
-keeps sweeping so work whose worker died mid-`prepare` is reclaimed once its claim goes stale.
-`enqueue` on an abandoned key revives it, keeping its checkpoints. Work is retained for 24 hours
-after its last update and then expires — the dedupe key expires with it, so idempotency has the
-same horizon as retention.
-
 ## Assistants (Optional)
 
 Register assistant classes and (optionally) a custom resolver. OpenAI is supported via `llm: 'openai'`.

package/lib/core/NexusMessaging.js

@@ -46,7 +46,6 @@ const { isLlmMonitorEnabled } = require('../clinical/flags/llmMonitorConfig');
 const { createQueueAdapter } = require('../queue');
 const { ScheduledMessageJob } = require('../jobs/ScheduledMessageJob');
 const { TemplateApprovalJob } = require('../jobs/TemplateApprovalJob');
-const { WorkflowRunner } = require('./DeferredWorkRunner');
 const { invokeInteractiveRoute } = require('./interactiveRouteService');
 
 const { PhiProcessor } = require('./PhiProcessor');
@@ -136,8 +135,6 @@ class NexusMessaging {
       queueAdapter: this.queueAdapter,
     });
 
-    this.workflowRunner = new WorkflowRunner({ queueAdapter: this.queueAdapter });
-
     this.phiProcessor = new PhiProcessor({
       encode: config.phi?.encode || false,
       ner: config.phi?.ner || null,
@@ -322,7 +319,6 @@ class NexusMessaging {
   getAssistantProcessor() { return this.assistantProcessor; }
   getLlmMonitor() { return this.llmMonitor; }
   getQueueAdapter() { return this.queueAdapter; }
-  getWorkflowRunner() { return this.workflowRunner; }
   getPhiProcessor() { return this.phiProcessor; }
   isConnected() { return this.provider?.getConnectionStatus() ?? false; }
   isProcessing(chatId) { return this.batchingManager.isProcessing(chatId); }
@@ -1081,8 +1077,6 @@ class NexusMessaging {
     await withTimeout(this._reconciliationStopped, teardownMs, 'queueReconciliation.stop')
       .catch(error => logger.warn('[NexusMessaging] Reconciliation sweep did not settle in time', { error: error.message }));
 
-    if (this.workflowRunner) this.workflowRunner.stop();
-
     if (this.queueAdapter) {
       await withTimeout(this.queueAdapter.shutdown(), teardownMs, 'queueAdapter.shutdown')
         .catch(error => logger.warn('[NexusMessaging] Queue shutdown did not finish', { error: error.message }));

package/lib/index.d.ts

@@ -254,7 +254,6 @@ declare module '@peopl-health/nexus' {
     getAssistantProcessor(): AssistantProcessor;
     getLlmMonitor(): { start(options?: { scheduleDaily?: boolean }): Promise<{ enabled: boolean }> } | null;
     initializeLlmMonitor(): Promise<{ enabled: boolean; error?: string } | undefined>;
-    getWorkflowRunner(): WorkflowRunner;
     processInstruction(code: string, instruction: string, role?: string, options?: { triggeredBy?: string }): Promise<void>;
     processSystemMessage(code: string, messages: string | string[], role?: string, options?: { triggeredBy?: string; reply?: boolean }): Promise<void>;
     processOutreach(code: string, options?: { brief?: OutreachBrief | null; triggeredBy?: string | null; reason?: string | null; firstName?: string | null; dryRun?: boolean }): Promise<OutreachDecision>;
@@ -542,17 +541,16 @@ declare module '@peopl-health/nexus' {
   export interface Workflow {
     kind: string;
     dedupeKey: (trigger: any) => string;
-    prepare: (trigger: any, checkpoints?: any, save?: (patch: any) => Promise<void>) => Promise<any>;
-    retrySchedule?: number[];
+    prepare: (trigger: any) => Promise<any>;
   }
 
   export interface WorkflowRunner {
     register(workflow: Workflow): Promise<Workflow>;
-    enqueue(kind: string, trigger: any, options?: QueueJobOptions): Promise<string | null>;
-    sweep(only?: string[]): Promise<string[]>;
-    stop(): void;
+    enqueue(kind: string, trigger: any, options?: QueueJobOptions): Promise<string>;
   }
 
+  export function createWorkflowRunner(options: { queueAdapter: QueueAdapter }): WorkflowRunner;
+
   // Memory System
   export interface PatientMemoryDocument {
     _id: any;

package/lib/index.js

@@ -287,6 +287,7 @@ class Nexus {
 }
 
 module.exports = {
+  createWorkflowRunner,
   Nexus,
   DrainingError,
   TwilioProvider,

package/lib/services/airtableService.js

@@ -15,11 +15,6 @@ const AIRTABLE_API_URL = 'https://api.airtable.com/v0';
 const AXIOS_TIMEOUT_CODE = 'ECONNABORTED';
 const RETRY_BASE_DELAY_MS = 1000;
 
-function isPermanentAirtableError(error) {
-  const status = error?.statusCode;
-  return typeof status === 'number' && status >= 400 && status < 500 && status !== 429;
-}
-
 let isEvalMode = false;
 
 function setEvalMode(enabled) {
@@ -47,7 +42,6 @@ async function withAirtableRetry(operation, label, { retryNetworkErrors = true }
       const isTransient = TRANSIENT_STATUS_CODES.includes(error.statusCode)
         || (retryNetworkErrors && isTransientNetworkError(error));
       if (!isTransient || attempt === MAX_ATTEMPTS) {
-        if (isPermanentAirtableError(error)) error.permanent = true;
         throw error;
       }
       const baseDelay = error.statusCode === RATE_LIMITED_STATUS ? RATE_LIMIT_DELAY_MS : RETRY_BASE_DELAY_MS;
@@ -242,7 +236,6 @@ async function upsertRecord(baseID, tableName, fields, options = {}, context = n
 
 module.exports = {
   setEvalMode,
-  isPermanentAirtableError,
   addRecord,
   getRecords,
   getRecordByFilter,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@peopl-health/nexus",
-  "version": "5.54.0-dev.7893",
+  "version": "5.54.0-dev.7895",
   "description": "Core messaging and assistant library for WhatsApp communication platforms",
   "keywords": [
     "whatsapp",

package/lib/core/DeferredWorkRunner.js

@@ -1,257 +0,0 @@
-const { describeThrown } = require('../utils/errorUtils');
-const { logger } = require('../utils/logger');
-
-const { DeferredWork, RETENTION_MS } = require('../models/deferredWorkModel');
-
-const STALE_CLAIM_MS = 5 * 60 * 1000;
-const MINUTE_MS = 60 * 1000;
-const INDEX_TIMEOUT_MS = 1000;
-const SWEEP_INTERVAL_MS = 60 * 1000;
-
-let indexesReady = null;
-
-function ensureIndexes() {
-  if (!indexesReady) {
-    indexesReady = DeferredWork.init().catch((error) => {
-      indexesReady = null;
-      logger.error('[WorkflowRunner] Index build failed, dedupe and retention are not guaranteed', { error: error.message });
-    });
-  }
-  return indexesReady;
-}
-
-function withIndexTimeout(promise) {
-  let timer;
-  const timeout = new Promise((resolve) => {
-    timer = setTimeout(resolve, INDEX_TIMEOUT_MS);
-    timer.unref();
-  });
-  return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
-}
-
-class WorkflowRunner {
-  constructor({ queueAdapter, sweepIntervalMs = SWEEP_INTERVAL_MS } = {}) {
-    if (!queueAdapter) throw new Error('WorkflowRunner requires a queueAdapter');
-    this.queueAdapter = queueAdapter;
-    this.sweepIntervalMs = sweepIntervalMs;
-    this.sweepTimer = null;
-    this.timers = new Map();
-    this.workflows = new Map();
-    this.stopped = false;
-  }
-
-  async register(workflow = {}) {
-    if (!workflow.kind || typeof workflow.kind !== 'string') throw new Error('workflow requires a kind');
-    if (workflow.kind.includes('__')) throw new Error(`workflow '${workflow.kind}' must not contain '__' (reserved as the jobId delimiter)`);
-    if (typeof workflow.dedupeKey !== 'function') throw new Error(`workflow '${workflow.kind}' requires a dedupeKey function`);
-    if (typeof workflow.prepare !== 'function') throw new Error(`workflow '${workflow.kind}' requires a prepare function`);
-    if (this.workflows.has(workflow.kind)) throw new Error(`workflow '${workflow.kind}' is already registered`);
-    if (workflow.retrySchedule !== undefined) {
-      if (!Array.isArray(workflow.retrySchedule) || !workflow.retrySchedule.length) {
-        throw new Error(`workflow '${workflow.kind}' requires a non-empty retrySchedule array`);
-      }
-      if (workflow.retrySchedule.some((minutes) => !Number.isFinite(minutes) || minutes <= 0)) {
-        throw new Error(`workflow '${workflow.kind}' requires retrySchedule entries to be positive minutes`);
-      }
-      if (workflow.retrySchedule.some((minutes) => minutes * MINUTE_MS >= RETENTION_MS)) {
-        throw new Error(`workflow '${workflow.kind}' requires retrySchedule entries under the ${RETENTION_MS / MINUTE_MS}m retention, or the record expires before the attempt fires`);
-      }
-    }
-
-    this.workflows.set(workflow.kind, workflow);
-    if (workflow.retrySchedule) {
-      await this.sweep([workflow.kind]).catch((error) => {
-        logger.error('[WorkflowRunner] Sweep on register failed', { kind: workflow.kind, error: error.message });
-      });
-      this._startSweeping();
-      return workflow;
-    }
-
-    try {
-      await this.queueAdapter.process(workflow.kind, (trigger) => workflow.prepare(trigger));
-    } catch (error) {
-      this.workflows.delete(workflow.kind);
-      throw error;
-    }
-    return workflow;
-  }
-
-  enqueue(kind, trigger, options = {}) {
-    const workflow = this.workflows.get(kind);
-    if (!workflow) throw new Error(`no workflow registered for '${kind}'`);
-    const dedupeKey = workflow.dedupeKey(trigger);
-    if (dedupeKey == null || (typeof dedupeKey === 'string' && !dedupeKey.trim())) {
-      throw new Error(`workflow '${kind}' produced an empty dedupe key`);
-    }
-    if (typeof dedupeKey !== 'string') throw new Error(`workflow '${kind}' produced a non-string dedupe key`);
-    if (!workflow.retrySchedule) {
-      return this.queueAdapter.enqueue(kind, trigger, { ...options, jobId: `${kind}__${dedupeKey}` });
-    }
-    return this._armDeferred(workflow, trigger, dedupeKey);
-  }
-
-  async sweep(only = null) {
-    const deferredKinds = [...this.workflows.values()].filter((w) => w.retrySchedule).map((w) => w.kind);
-    const kinds = only ? deferredKinds.filter((kind) => only.includes(kind)) : deferredKinds;
-    if (!kinds.length) return [];
-
-    const orphaned = await DeferredWork.find({
-      kind: { $in: kinds },
-      $or: [
-        { status: 'pending' },
-        { status: 'processing', claimedAt: { $lt: new Date(Date.now() - STALE_CLAIM_MS) } }
-      ]
-    });
-
-    const rearmed = orphaned.map((work) => this._arm(
-      this.workflows.get(work.kind),
-      work._id,
-      this._delayUntilDue(work),
-    ));
-    if (rearmed.length) logger.info('[WorkflowRunner] Swept', { kinds, rearmed: rearmed.length });
-    return rearmed;
-  }
-
-  _startSweeping() {
-    if (this.stopped || this.sweepTimer || !this.sweepIntervalMs) return;
-    this.sweepTimer = setInterval(() => {
-      this.sweep().catch((error) => logger.error('[WorkflowRunner] Periodic sweep failed', { error: error.message }));
-    }, this.sweepIntervalMs);
-    this.sweepTimer.unref();
-  }
-
-  stop() {
-    this.stopped = true;
-    if (this.sweepTimer) clearInterval(this.sweepTimer);
-    this.sweepTimer = null;
-    for (const timer of this.timers.values()) clearTimeout(timer);
-    this.timers.clear();
-  }
-
-  async _armDeferred(workflow, trigger, dedupeKey) {
-    const { kind } = workflow;
-    await withIndexTimeout(ensureIndexes());
-    const work = await DeferredWork.findOne({ kind, dedupeKey })
-      || await DeferredWork.create({ kind, dedupeKey, trigger }).catch(async (error) => {
-        if (error.code !== 11000) throw error;
-        return await DeferredWork.findOne({ kind, dedupeKey });
-      });
-    if (work.status === 'abandoned') {
-      const { modifiedCount } = await DeferredWork.updateOne(
-        { _id: work._id, status: 'abandoned' },
-        { $set: { status: 'pending', attempt: 0, nextAttemptAt: new Date(), lastError: null, claimedAt: null } }
-      );
-      if (!modifiedCount) return null;
-      logger.info('[WorkflowRunner] Re-arming abandoned work', { kind, dedupeKey });
-      return this._arm(workflow, work._id, 0);
-    }
-    if (work.status !== 'pending') {
-      logger.info('[WorkflowRunner] Skipping, work is in flight or complete', { kind, dedupeKey, status: work.status });
-      return null;
-    }
-    return this._arm(workflow, work._id, this._delayUntilDue(work));
-  }
-
-  _delayUntilDue(work) {
-    return Math.max(0, new Date(work.nextAttemptAt).getTime() - Date.now());
-  }
-
-  _arm(workflow, workId, delayMs) {
-    const id = String(workId);
-    if (this.stopped) return id;
-    clearTimeout(this.timers.get(id));
-    const timer = setTimeout(() => {
-      this.timers.delete(id);
-      this._runDeferred(workflow, id).catch((error) => {
-        logger.error('[WorkflowRunner] Deferred run failed', { kind: workflow.kind, deferredWorkId: id, error: error.message });
-      });
-    }, delayMs);
-    timer.unref();
-    this.timers.set(id, timer);
-    return id;
-  }
-
-  async _runDeferred(workflow, deferredWorkId) {
-    const claimedAt = new Date();
-    const prior = await DeferredWork.findOneAndUpdate(
-      {
-        _id: deferredWorkId,
-        $or: [
-          { status: 'pending', nextAttemptAt: { $lte: claimedAt } },
-          { status: 'processing', claimedAt: { $lt: new Date(claimedAt.getTime() - STALE_CLAIM_MS) } }
-        ]
-      },
-      { $set: { status: 'processing', claimedAt } },
-      { new: false }
-    );
-    if (!prior) {
-      logger.info('[WorkflowRunner] Claimed elsewhere or not due', { kind: workflow.kind, deferredWorkId });
-      return { claimed: false };
-    }
-
-    const reclaimed = prior.status === 'processing';
-    const work = { ...prior.toObject(), attempt: prior.attempt + (reclaimed ? 1 : 0) };
-    if (reclaimed) await this._write(work, claimedAt, { attempt: work.attempt });
-
-    const checkpoints = { ...(work.checkpoints || {}) };
-    const save = async (patch) => {
-      Object.assign(checkpoints, patch);
-      const $set = {};
-      for (const [key, value] of Object.entries(patch || {})) $set[`checkpoints.${key}`] = value;
-      if (Object.keys($set).length) await this._write(work, claimedAt, $set);
-    };
-
-    try {
-      await workflow.prepare(work.trigger, checkpoints, save);
-    } catch (error) {
-      return await this._rescheduleOrAbandon(workflow, work, claimedAt, error);
-    }
-    await this._write(work, claimedAt, { status: 'done', claimedAt: null });
-    return { status: 'done' };
-  }
-
-  async _write(work, claimedAt, $set) {
-    const { matchedCount } = await DeferredWork.updateOne({ _id: work._id, claimedAt }, { $set });
-    if (!matchedCount) logger.warn('[WorkflowRunner] Claim lost, write discarded', { deferredWorkId: String(work._id) });
-    return matchedCount > 0;
-  }
-
-  async _rescheduleOrAbandon(workflow, work, claimedAt, error) {
-    const lastError = describeThrown(error);
-    const delayMinutes = workflow.retrySchedule[work.attempt];
-    if (error?.permanent === true || delayMinutes === undefined) {
-      await this._write(work, claimedAt, { status: 'abandoned', lastError, claimedAt: null });
-      logger.error('[WorkflowRunner] Abandoned', {
-        kind: workflow.kind,
-        dedupeKey: work.dedupeKey,
-        attempt: work.attempt,
-        permanent: error?.permanent === true,
-        error: lastError
-      });
-      return { status: 'abandoned' };
-    }
-
-    const attempt = work.attempt + 1;
-    const delayMs = delayMinutes * MINUTE_MS;
-    const kept = await this._write(work, claimedAt, {
-      status: 'pending',
-      attempt,
-      nextAttemptAt: new Date(Date.now() + delayMs),
-      lastError,
-      claimedAt: null
-    });
-    if (!kept) return { claimed: false };
-
-    logger.warn('[WorkflowRunner] Rescheduled', {
-      kind: workflow.kind,
-      dedupeKey: work.dedupeKey,
-      attempt,
-      delayMinutes,
-      error: lastError
-    });
-    this._arm(workflow, work._id, delayMs);
-    return { status: 'pending', attempt };
-  }
-}
-
-module.exports = { WorkflowRunner };

package/lib/models/deferredWorkModel.js

@@ -1,24 +0,0 @@
-const mongoose = require('mongoose');
-
-const STATUS_VALUES = ['pending', 'processing', 'done', 'abandoned'];
-const RETENTION_MS = 24 * 60 * 60 * 1000;
-
-const deferredWorkSchema = new mongoose.Schema({
-  kind: { type: String, required: true },
-  dedupeKey: { type: String, required: true },
-  trigger: { type: mongoose.Schema.Types.Mixed, default: null },
-  checkpoints: { type: mongoose.Schema.Types.Mixed, default: () => ({}) },
-  status: { type: String, enum: STATUS_VALUES, default: 'pending' },
-  attempt: { type: Number, default: 0 },
-  nextAttemptAt: { type: Date, default: Date.now },
-  claimedAt: { type: Date, default: null },
-  lastError: { type: String, default: null }
-}, { timestamps: true });
-
-deferredWorkSchema.index({ kind: 1, dedupeKey: 1 }, { unique: true, name: 'dedupe_idx' });
-deferredWorkSchema.index({ status: 1, nextAttemptAt: 1 }, { name: 'due_idx' });
-deferredWorkSchema.index({ updatedAt: 1 }, { expireAfterSeconds: RETENTION_MS / 1000, name: 'ttl_idx' });
-
-const DeferredWork = mongoose.model('DeferredWork', deferredWorkSchema);
-
-module.exports = { DeferredWork, RETENTION_MS };