@peopl-health/nexus 5.45.0-dev.7144 → 5.45.0-dev.7159

package/README.md

@@ -116,6 +116,38 @@ const bus = nexus.getMessaging().getEventBus();
 bus.on('message:received', (m) => console.log('rx', m.id));
 ```
 
+## Workflows (Optional)
+
+Register a named unit of work, deduplicated by a key you derive from the trigger.
+
+```js
+const runner = nexus.getMessaging().getWorkflowRunner();
+
+await runner.register({
+  kind: 'triage-projection',
+  dedupeKey: (trigger) => String(trigger.submissionId),
+  prepare: async (trigger, checkpoints, save) => { /* ... */ },
+  retrySchedule: [5, 10, 20, 40],   // minutes; omit for fire-and-forget
+});
+
+await runner.enqueue('triage-projection', { submissionId });
+```
+
+With `retrySchedule` the trigger and a checkpoint journal are stored in Mongo and the schedule runs
+on an in-process timer — deferred workflows never touch the queue adapter, so they add no Redis load.
+`prepare` resumes from `checkpoints`; call `save(patch)` before each external call so a retry does
+not repeat completed work. Throw an error carrying `permanent: true` to give up immediately instead
+of walking the schedule; the Airtable helpers tag their own permanent 4xx errors that way, in
+`withAirtableRetry`, which all five of them route through. They rethrow on failure too, so an
+Airtable outage reaches `prepare` as an exception and walks the retry schedule on its own rather
+than reading as success.
+
+Recovery is automatic: registering sweeps for work left behind by a previous process, and the runner
+keeps sweeping so work whose worker died mid-`prepare` is reclaimed once its claim goes stale.
+`enqueue` on an abandoned key revives it, keeping its checkpoints. Work is retained for 24 hours
+after its last update and then expires — the dedupe key expires with it, so idempotency has the
+same horizon as retention.
+
 ## Assistants (Optional)
 
 Register assistant classes and (optionally) a custom resolver. OpenAI is supported via `llm: 'openai'`.

package/lib/clinical/config/divergenceConfig.js

@@ -2,6 +2,8 @@ const { Config_ID } = require('../../config/airtableConfig');
 
 const { getRecordByFilter } = require('../../services/airtableService');
 
+const { ESCALATION_ROUTES } = require('../../shared/dtos/RecommendationSafetyGate');
+
 const { MapCache } = require('../../utils/MapCache');
 const { safeParse, safeParseArray, isPlainObject } = require('../../utils/jsonUtils');
 
@@ -14,6 +16,70 @@ const STOCHASTIC_DEFAULTS = { enabled: false, presetId: '' };
 
 const cache = new MapCache({ maxSize: 1, ttl: CACHE_TTL });
 
+function parseSafetyFlagPatterns(value) {
+  if (!value) return [];
+  const parsedPatterns = safeParseArray(value);
+  if (!parsedPatterns) throw new Error('divergence config SAFETY_FLAG_PATTERNS is not a JSON array');
+  return parsedPatterns.map((row) => {
+    if (!isPlainObject(row)) throw new Error('divergence config SAFETY_FLAG_PATTERNS row is not an object');
+    const { flag, pattern } = row;
+    if (!flag || !pattern) throw new Error('divergence config SAFETY_FLAG_PATTERNS row missing flag or pattern');
+    try {
+      return { flag, re: new RegExp(pattern, 'i') };
+    } catch (err) {
+      throw new Error(`divergence config SAFETY_FLAG_PATTERNS pattern for ${flag} is not a valid regex: ${err.message}`);
+    }
+  });
+}
+
+function parseRedFlagMinRoute(row) {
+  if (!isPlainObject(row)) throw new Error('divergence config RED_FLAG_MIN_ROUTES row is not an object');
+  const {
+    flag,
+    ctcae_terms: ctcaeTerms,
+    min_route: minRoute,
+    max_action_time_hours: maxActionTimeHours,
+    requires_risk: requiresRisk = true,
+  } = row;
+  if (typeof flag !== 'string' || !flag.trim()) throw new Error('divergence config RED_FLAG_MIN_ROUTES row missing flag');
+  if (!Array.isArray(ctcaeTerms) || !ctcaeTerms.length || !ctcaeTerms.every((term) => typeof term === 'string' && term.trim())) {
+    throw new Error(`divergence config RED_FLAG_MIN_ROUTES ctcae_terms for ${flag} is not a non-empty list of terms`);
+  }
+  if (!ESCALATION_ROUTES.includes(minRoute)) {
+    throw new Error(`divergence config RED_FLAG_MIN_ROUTES min_route for ${flag} is not one of ${ESCALATION_ROUTES.join(', ')}`);
+  }
+  if (!Number.isInteger(maxActionTimeHours) || maxActionTimeHours < 1 || maxActionTimeHours > 24) {
+    throw new Error(`divergence config RED_FLAG_MIN_ROUTES max_action_time_hours for ${flag} is not an integer from 1 to 24`);
+  }
+  if (typeof requiresRisk !== 'boolean') throw new Error(`divergence config RED_FLAG_MIN_ROUTES requires_risk for ${flag} is not a boolean`);
+  return {
+    flag, ctcaeTerms, minRoute, maxActionTimeHours, requiresRisk,
+  };
+}
+
+function parseRedFlagMinRoutes(value) {
+  const routes = { rows: [], invalidRows: [] };
+  if (!value) return routes;
+  const parsedRoutes = safeParseArray(value);
+  if (!parsedRoutes) throw new Error('divergence config RED_FLAG_MIN_ROUTES is not a JSON array');
+  for (const row of parsedRoutes) {
+    try {
+      routes.rows.push(parseRedFlagMinRoute(row));
+    } catch (err) {
+      routes.invalidRows.push(err.message);
+    }
+  }
+  return routes;
+}
+
+function parseIsolated(parse) {
+  try {
+    return { value: parse(), error: null };
+  } catch (error) {
+    return { value: null, error };
+  }
+}
+
 async function load() {
   const cached = cache.get(CACHE_KEY);
   if (cached) return cached;
@@ -46,23 +112,13 @@ async function load() {
   const stochastic = { ...STOCHASTIC_DEFAULTS, ...(parsedStochastic || {}) };
   if (stochastic.enabled && !stochastic.presetId) throw new Error('divergence config STOCHASTIC.presetId is required when enabled');
 
-  let safetyFlagPatterns = [];
-  if (byKey.SAFETY_FLAG_PATTERNS) {
-    const parsedPatterns = safeParseArray(byKey.SAFETY_FLAG_PATTERNS);
-    if (!parsedPatterns) throw new Error('divergence config SAFETY_FLAG_PATTERNS is not a JSON array');
-    safetyFlagPatterns = parsedPatterns.map((row) => {
-      if (!isPlainObject(row)) throw new Error('divergence config SAFETY_FLAG_PATTERNS row is not an object');
-      const { flag, pattern } = row;
-      if (!flag || !pattern) throw new Error('divergence config SAFETY_FLAG_PATTERNS row missing flag or pattern');
-      try {
-        return { flag, re: new RegExp(pattern, 'i') };
-      } catch (err) {
-        throw new Error(`divergence config SAFETY_FLAG_PATTERNS pattern for ${flag} is not a valid regex: ${err.message}`);
-      }
-    });
-  }
-
-  const config = { FIELDS: fields, ORACLE_CONNECTION: oracleConnection, STOCHASTIC: stochastic, SAFETY_FLAG_PATTERNS: safetyFlagPatterns };
+  const config = {
+    FIELDS: fields,
+    ORACLE_CONNECTION: oracleConnection,
+    STOCHASTIC: stochastic,
+    SAFETY_FLAG_PATTERNS: parseIsolated(() => parseSafetyFlagPatterns(byKey.SAFETY_FLAG_PATTERNS)),
+    RED_FLAG_MIN_ROUTES: parseIsolated(() => parseRedFlagMinRoutes(byKey.RED_FLAG_MIN_ROUTES)),
+  };
   cache.set(CACHE_KEY, config);
   return config;
 }
@@ -72,9 +128,16 @@ async function get(key) {
   return config[key];
 }
 
+async function getIsolated(key) {
+  const { value, error } = await get(key);
+  if (error) throw error;
+  return value;
+}
+
 module.exports = {
   getFields: () => get('FIELDS'),
   getOracleConnection: () => get('ORACLE_CONNECTION'),
   getStochastic: () => get('STOCHASTIC'),
-  getSafetyFlagPatterns: () => get('SAFETY_FLAG_PATTERNS'),
+  getSafetyFlagPatterns: () => getIsolated('SAFETY_FLAG_PATTERNS'),
+  getRedFlagMinRoutes: () => getIsolated('RED_FLAG_MIN_ROUTES'),
 };

package/lib/clinical/helpers/recommendationGateHelper.js

@@ -1,8 +1,10 @@
 const { ESCALATION_ROUTES, RED_FLAG_STATUSES } = require('../../shared/dtos/RecommendationSafetyGate');
+const { aliasClosure } = require('../../fhir/helpers/symptomAliasHelper');
 const { str, arr, strList } = require('../../utils/coercionUtils');
 const { foldAccents } = require('../../utils/formatUtils');
 
 const ECHO_SHARE = 0.5;
+const ROUTE_FLOOR_UNVERIFIABLE_CODE = 'route_floor_unverifiable';
 
 const tokensOf = (text) => foldAccents(str(text)).toLowerCase().match(/[a-z0-9]{4,}/g) || [];
 
@@ -107,6 +109,93 @@ function normalizeGateEscalation(raw, repairs) {
   return { urgent, maxActionTimeHours, escalationRoute, reasonForEscalation };
 }
 
+const routeUrgency = (route) => ESCALATION_ROUTES.indexOf(route);
+
+const aliasKeySet = (catalog, terms) => new Set(terms.flatMap((term) => aliasClosure(catalog, term)));
+
+function riskSymptomKeys(catalog, riskAssessment) {
+  return aliasKeySet(catalog, [
+    ...arr(riskAssessment?.predictedAes)
+      .filter((ae) => ae?.severityIfConfirmed === 'emergency')
+      .flatMap((ae) => arr(ae.relatedSymptomTerms)),
+    ...arr(riskAssessment?.diseaseRisks).flatMap((risk) => arr(risk?.relatedSymptomTerms)),
+  ]);
+}
+
+const unverifiableFloor = (subject, reason) => ({
+  path: 'escalation_route',
+  code: ROUTE_FLOOR_UNVERIFIABLE_CODE,
+  warning: true,
+  problem: `${subject}: ${reason}; el gate no se elevó`,
+});
+
+function applyRouteFloor(escalation, {
+  catalog,
+  rows = [],
+  invalidRows = [],
+  affirmedKeys = new Set(),
+  riskAssessment = null,
+  riskReadFailed = false,
+  freeTextPending = false,
+  configError = null,
+} = {}, repairs = [], warnings = []) {
+  if (configError) {
+    warnings.push(unverifiableFloor('rutas mínimas no verificables', `no se pudo leer RED_FLAG_MIN_ROUTES (${configError})`));
+    return escalation;
+  }
+  if (invalidRows.length) {
+    warnings.push({
+      path: 'escalation_route',
+      code: ROUTE_FLOOR_UNVERIFIABLE_CODE,
+      warning: true,
+      problem: `rutas mínimas incompletas: se ignoraron filas inválidas de RED_FLAG_MIN_ROUTES (${invalidRows.join('; ')}); esas filas no pueden elevar el gate`,
+    });
+  }
+  const riskKeys = riskSymptomKeys(catalog, riskAssessment);
+  const fired = [];
+  for (const row of rows) {
+    const termKeys = [...aliasKeySet(catalog, row.ctcaeTerms)];
+    const eligibleKeys = termKeys.filter((key) => !row.requiresRisk || riskKeys.has(key));
+    if (eligibleKeys.some((key) => affirmedKeys.has(key))) {
+      fired.push(row);
+    } else if (row.requiresRisk && !riskAssessment && termKeys.some((key) => affirmedKeys.has(key))) {
+      const reason = riskReadFailed ? 'no se pudo leer el perfil de riesgo' : 'no hay perfil de riesgo';
+      warnings.push(unverifiableFloor(`ruta mínima ${row.minRoute} de ${row.flag} no verificable`, reason));
+    } else if (freeTextPending && eligibleKeys.length) {
+      warnings.push(unverifiableFloor(`ruta mínima ${row.minRoute} de ${row.flag} no verificable`, 'el clasificador del texto libre no ha terminado'));
+    }
+  }
+  if (!fired.length) return escalation;
+  const currentRoute = escalation.urgent ? escalation.escalationRoute : null;
+  const currentHours = escalation.urgent ? escalation.maxActionTimeHours : null;
+  const floorRoute = fired.map((row) => row.minRoute).sort((left, right) => routeUrgency(left) - routeUrgency(right))[0];
+  const floorHours = Math.min(...fired.map((row) => row.maxActionTimeHours));
+  const route = currentRoute && routeUrgency(currentRoute) <= routeUrgency(floorRoute) ? currentRoute : floorRoute;
+  const hours = Number.isInteger(currentHours) ? Math.min(currentHours, floorHours) : floorHours;
+  const routeRaised = route !== currentRoute;
+  const windowTightened = hours !== currentHours;
+  if (!routeRaised && !windowTightened) return escalation;
+  const flags = [...new Set(fired
+    .filter((row) => (routeRaised && row.minRoute === route) || (windowTightened && row.maxActionTimeHours === hours))
+    .map((row) => row.flag))].join(', ');
+  const changes = [
+    routeRaised ? `escalation_route ${currentRoute ? `'${currentRoute}'` : 'null'} → '${route}'` : null,
+    windowTightened ? `max_action_time_hours ${currentHours ?? 'null'} → ${hours}` : null,
+    escalation.urgent ? null : 'urgent_under_24h false → true',
+  ].filter(Boolean);
+  repairs.push(`ruta mínima por término reportado (${flags}): ${changes.join(', ')}`);
+  return {
+    ...escalation,
+    urgent: true,
+    maxActionTimeHours: hours,
+    escalationRoute: route,
+    reasonForEscalation: escalation.reasonForEscalation || `Término reportado en este triaje (${flags}): ruta mínima ${route}, máximo ${hours} h.`,
+    summary: [str(escalation.summary).trim(), `Ruta mínima aplicada por un término reportado en este triaje (${flags}): ${route}, máximo ${hours} h.`]
+      .filter(Boolean)
+      .join(' '),
+  };
+}
+
 module.exports = {
   str,
   arr,
@@ -115,4 +204,5 @@ module.exports = {
   normalizeStatus,
   buildRedFlag,
   normalizeGateEscalation,
+  applyRouteFloor,
 };

package/lib/clinical/services/triageRecommendationService.js

@@ -1,10 +1,12 @@
 const { getTriageGate, getTriagePlan } = require('../config/subAgentsConfig');
-const { getSafetyFlagPatterns } = require('../config/divergenceConfig');
+const { getSafetyFlagPatterns, getRedFlagMinRoutes } = require('../config/divergenceConfig');
 const { requireGatewayProvider } = require('../config/llmConfig');
 const { getTriageRecommendationEventModel } = require('../models/triageRecommendationEventModel');
 const { isShadowRun } = require('../helpers/shadowHelper');
 const { readOutputText } = require('../helpers/clinicalMentionHelper');
-const { str, arr, buildRedFlag, normalizeGateEscalation } = require('../helpers/recommendationGateHelper');
+const {
+  str, arr, buildRedFlag, normalizeGateEscalation, applyRouteFloor,
+} = require('../helpers/recommendationGateHelper');
 const { buildElement, nullableStr } = require('../helpers/recommendationPlanHelper');
 const { readClinicalProfile } = require('../helpers/clinicalProfileHelper');
 const {
@@ -265,7 +267,13 @@ async function generateTriageRecommendations({ payload, provider, runId = null }
   if (profile === undefined) return { ...settle, status: 'failed', stage: 'assemble', reason: 'profile_read_failed' };
   if (!profile) return { ...settle, status: 'failed', stage: 'assemble', reason: 'no_clinical_profile' };
 
-  const [riskAssessment] = await readPatientRiskAssessments({ patientId: code, limit: 1 }).catch(() => []);
+  const riskRead = await readPatientRiskAssessments({ patientId: code, limit: 1 })
+    .then((assessments) => ({ assessments, failed: false }))
+    .catch((err) => {
+      logger.warn('[triageRecommendation] risk assessment unavailable; section degraded', { code, error: err?.message });
+      return { assessments: [], failed: true };
+    });
+  const [riskAssessment] = riskRead.assessments;
   // A failed read degrades the section; it must never fail the run.
   const [openCases, escalations] = await Promise.all([
     readSymptomCases({ patientId: code })
@@ -315,16 +323,40 @@ async function generateTriageRecommendations({ payload, provider, runId = null }
     ...freeTextDerivedSymptoms,
   ];
   const reportedKeys = symptomKeySet(catalog, reportedTerms);
+  const floorConfig = await getRedFlagMinRoutes()
+    .then(({ rows, invalidRows }) => ({ rows, invalidRows, configError: null }))
+    .catch((err) => {
+      logger.warn('[triageRecommendation] red-flag minimum routes unavailable; no route floor', { code, error: err?.message });
+      return { rows: [], invalidRows: [], configError: err?.message || 'desconocido' };
+    });
+  const routeFloor = {
+    catalog,
+    ...floorConfig,
+    affirmedKeys: reportedKeys,
+    riskAssessment,
+    riskReadFailed: riskRead.failed,
+    freeTextPending: settle.classifierPending === true && freeTextBits.length > 0,
+  };
 
   const gateStart = Date.now();
-  const gateOutcome = await runGate({ provider, presetId: gatePresetId, casoBase: casoBaseFor('gate'), rid, code, payload });
+  const gateOutcome = await runGate({
+    provider, presetId: gatePresetId, casoBase: casoBaseFor('gate'), routeFloor, rid, code, payload,
+  });
   settle.gateLatencyMs = Date.now() - gateStart;
   settle.gateAttempts = gateOutcome.attempts;
-  if (gateOutcome.failure) return { ...settle, ...gateOutcome.failure, autoRepairs: gateOutcome.gateRepairs || [] };
-  const { gate, gateRepairs } = gateOutcome;
+  if (gateOutcome.failure) {
+    return {
+      ...settle,
+      ...gateOutcome.failure,
+      autoRepairs: gateOutcome.gateRepairs || [],
+      qualityWarnings: gateOutcome.gateWarnings || settle.qualityWarnings,
+    };
+  }
+  const { gate, gateRepairs, gateWarnings } = gateOutcome;
   settle.gateVerdict = gate.urgentUnder24h ? 'urgent' : 'clear';
   settle.urgentUnder24h = gate.urgentUnder24h;
   settle.escalationRoute = gate.escalationRoute;
+  settle.qualityWarnings = gateWarnings;
 
   const planStart = Date.now();
   const boundary = {
@@ -344,7 +376,7 @@ async function generateTriageRecommendations({ payload, provider, runId = null }
   const { plan, planRepairs, qualityWarnings } = planOutcome;
   settle.elementCount = plan.elements.length;
   settle.autoRepairs = [...gateRepairs, ...planRepairs];
-  settle.qualityWarnings = qualityWarnings;
+  settle.qualityWarnings = [...gateWarnings, ...qualityWarnings];
   settle.rankReordered = planOutcome.rankReordered;
   settle.deniedCardsDropped = planOutcome.deniedCardsDropped;
   settle.deniedTargetsStripped = planOutcome.deniedTargetsStripped;
@@ -404,7 +436,9 @@ function rejectedGateShape(rawGate, redFlags) {
   };
 }
 
-async function runGate({ provider, presetId, casoBase, rid, code, payload }) {
+async function runGate({
+  provider, presetId, casoBase, routeFloor, rid, code, payload,
+}) {
   let attempts = 0;
   let gateRepairs = [];
   let accepted = null;
@@ -468,7 +502,10 @@ async function runGate({ provider, presetId, casoBase, rid, code, payload }) {
     break;
   }
   if (!accepted) return { attempts, gateRepairs, failure: { status: 'failed', stage: 'safety_gate', reason: 'gate_malformed' } };
-  const { summary, redFlags, urgent, maxActionTimeHours, escalationRoute, reasonForEscalation } = accepted;
+  const gateWarnings = [];
+  const {
+    summary, redFlags, urgent, maxActionTimeHours, escalationRoute, reasonForEscalation,
+  } = applyRouteFloor(accepted, routeFloor, gateRepairs, gateWarnings);
 
   let gate;
   try {
@@ -487,15 +524,21 @@ async function runGate({ provider, presetId, casoBase, rid, code, payload }) {
     });
   } catch (err) {
     logger.warn('[triageRecommendation] gate failed DTO validation', { code, runId: rid, error: err?.message });
-    return { attempts, gateRepairs, failure: { status: 'failed', stage: 'safety_gate', reason: 'gate_malformed' } };
+    return {
+      attempts, gateRepairs, gateWarnings, failure: { status: 'failed', stage: 'safety_gate', reason: 'gate_malformed' },
+    };
   }
   try {
     await storeSafetyGate({ patientId: code, safetyGate: gate });
   } catch (err) {
     logger.error('[triageRecommendation] gate store failed', { code, runId: rid, error: err?.message });
-    return { attempts, gateRepairs, failure: { status: 'failed', stage: 'fhir', reason: 'gate_store_failed' } };
+    return {
+      attempts, gateRepairs, gateWarnings, failure: { status: 'failed', stage: 'fhir', reason: 'gate_store_failed' },
+    };
   }
-  return { attempts, gate, gateRepairs };
+  return {
+    attempts, gate, gateRepairs, gateWarnings,
+  };
 }
 
 async function runPlan({ provider, presetId, casoBase, gate, boundary = null, rid, code, payload }) {

package/lib/controllers/patientController.js

@@ -6,7 +6,6 @@ const { isAllowedUrl, redactDirectIdentifiers } = require('../utils/sanitizerUti
 const { getPatientInformation, updatePatientInformation, logPatientChange } = require('../services/patientService');
 const { getTriageHistory } = require('../services/triageHistoryService');
 const { getCarePlanHistory } = require('../services/carePlanHistoryService');
-const { getEscalationContext } = require('../services/escalationContextService');
 const { getPatientTasks, updatePatientTask } = require('../services/patientTaskService');
 const {
   getRecommendationCarousels,
@@ -242,25 +241,10 @@ const logPatientChangeController = async (req, res) => {
   }
 };
 
-const getPatientEscalationContextController = async (req, res) => {
-  try {
-    const context = await getEscalationContext({ code: req.params.code });
-    res.status(200).json({ success: true, context });
-  } catch (error) {
-    logger.error('Error fetching patient escalation context:', {
-      error: redactDirectIdentifiers(String(error.message || '')),
-      stack: redactDirectIdentifiers(String(error.stack || '')),
-      code: redactDirectIdentifiers(String(req.params?.code || '')),
-    });
-    res.status(error.statusCode || 500).json({ success: false, error: error.message });
-  }
-};
-
 module.exports = {
   getPatientInfoController,
   getPatientTriageInfoController,
   getPatientCarePlanController,
-  getPatientEscalationContextController,
   getPatientTasksController,
   getPatientRecommendationCarouselsController,
   getPatientRecommendationCarouselImageController,

package/lib/core/NexusMessaging.js

@@ -43,6 +43,7 @@ const { isLlmMonitorEnabled } = require('../clinical/flags/llmMonitorConfig');
 const { createQueueAdapter } = require('../queue');
 const { ScheduledMessageJob } = require('../jobs/ScheduledMessageJob');
 const { TemplateApprovalJob } = require('../jobs/TemplateApprovalJob');
+const { WorkflowRunner } = require('./WorkflowRunner');
 
 const { PhiProcessor } = require('./PhiProcessor');
 
@@ -127,6 +128,8 @@ class NexusMessaging {
       queueAdapter: this.queueAdapter,
     });
 
+    this.workflowRunner = new WorkflowRunner({ queueAdapter: this.queueAdapter });
+
     this.phiProcessor = new PhiProcessor({
       encode: config.phi?.encode || false,
       ner: config.phi?.ner || null,
@@ -310,6 +313,7 @@ class NexusMessaging {
   getAssistantProcessor() { return this.assistantProcessor; }
   getLlmMonitor() { return this.llmMonitor; }
   getQueueAdapter() { return this.queueAdapter; }
+  getWorkflowRunner() { return this.workflowRunner; }
   getPhiProcessor() { return this.phiProcessor; }
   isConnected() { return this.provider?.getConnectionStatus() ?? false; }
   isProcessing(chatId) { return this.batchingManager.isProcessing(chatId); }
@@ -1012,6 +1016,7 @@ class NexusMessaging {
   async disconnect() {
     this.queueReconciliation?.stop();
     if (this.provider) await this.provider.disconnect();
+    if (this.workflowRunner) this.workflowRunner.stop();
     if (this.queueAdapter) await this.queueAdapter.shutdown();
     this.events.removeAllListeners();
   }

package/lib/core/WorkflowRunner.js

@@ -0,0 +1,265 @@
+const { describeThrown } = require('../utils/errorUtils');
+const { logger } = require('../utils/logger');
+
+const { DeferredWork, RETENTION_MS } = require('../models/deferredWorkModel');
+
+const STALE_CLAIM_MS = 5 * 60 * 1000;
+const MINUTE_MS = 60 * 1000;
+const INDEX_TIMEOUT_MS = 1000;
+const SWEEP_INTERVAL_MS = 60 * 1000;
+
+let indexesReady = null;
+
+function ensureIndexes() {
+  if (!indexesReady) {
+    indexesReady = DeferredWork.init().catch((error) => {
+      indexesReady = null;
+      logger.error('[WorkflowRunner] Index build failed, dedupe and retention are not guaranteed', { error: error.message });
+    });
+  }
+  return indexesReady;
+}
+
+function withIndexTimeout(promise) {
+  let timer;
+  const timeout = new Promise((resolve) => {
+    timer = setTimeout(resolve, INDEX_TIMEOUT_MS);
+    timer.unref();
+  });
+  return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
+}
+
+class WorkflowRunner {
+  constructor({ queueAdapter, sweepIntervalMs = SWEEP_INTERVAL_MS } = {}) {
+    if (!queueAdapter) throw new Error('WorkflowRunner requires a queueAdapter');
+    this.queueAdapter = queueAdapter;
+    this.sweepIntervalMs = sweepIntervalMs;
+    this.sweepTimer = null;
+    this.timers = new Map();
+    this.workflows = new Map();
+    this.stopped = false;
+  }
+
+  async register(workflow = {}) {
+    if (!workflow.kind || typeof workflow.kind !== 'string') throw new Error('workflow requires a kind');
+    if (workflow.kind.includes('__')) throw new Error(`workflow '${workflow.kind}' must not contain '__' (reserved as the jobId delimiter)`);
+    if (typeof workflow.dedupeKey !== 'function') throw new Error(`workflow '${workflow.kind}' requires a dedupeKey function`);
+    if (typeof workflow.prepare !== 'function') throw new Error(`workflow '${workflow.kind}' requires a prepare function`);
+    if (this.workflows.has(workflow.kind)) throw new Error(`workflow '${workflow.kind}' is already registered`);
+    if (workflow.retrySchedule !== undefined) {
+      if (!Array.isArray(workflow.retrySchedule) || !workflow.retrySchedule.length) {
+        throw new Error(`workflow '${workflow.kind}' requires a non-empty retrySchedule array`);
+      }
+      if (workflow.retrySchedule.some((minutes) => !Number.isFinite(minutes) || minutes <= 0)) {
+        throw new Error(`workflow '${workflow.kind}' requires retrySchedule entries to be positive minutes`);
+      }
+      if (workflow.retrySchedule.some((minutes) => minutes * MINUTE_MS >= RETENTION_MS)) {
+        throw new Error(`workflow '${workflow.kind}' requires retrySchedule entries under the ${RETENTION_MS / MINUTE_MS}m retention, or the record expires before the attempt fires`);
+      }
+    }
+
+    this.workflows.set(workflow.kind, workflow);
+    if (workflow.retrySchedule) {
+      await this.sweep([workflow.kind]).catch((error) => {
+        logger.error('[WorkflowRunner] Sweep on register failed', { kind: workflow.kind, error: error.message });
+      });
+      this._startSweeping();
+      return workflow;
+    }
+
+    try {
+      await this.queueAdapter.process(workflow.kind, (trigger) => workflow.prepare(trigger));
+    } catch (error) {
+      this.workflows.delete(workflow.kind);
+      throw error;
+    }
+    return workflow;
+  }
+
+  enqueue(kind, trigger, options = {}) {
+    const workflow = this.workflows.get(kind);
+    if (!workflow) throw new Error(`no workflow registered for '${kind}'`);
+    const dedupeKey = workflow.dedupeKey(trigger);
+    if (dedupeKey == null || (typeof dedupeKey === 'string' && !dedupeKey.trim())) {
+      throw new Error(`workflow '${kind}' produced an empty dedupe key`);
+    }
+    if (typeof dedupeKey !== 'string') throw new Error(`workflow '${kind}' produced a non-string dedupe key`);
+    if (!workflow.retrySchedule) {
+      return this.queueAdapter.enqueue(kind, trigger, { ...options, jobId: `${kind}__${dedupeKey}` });
+    }
+    return this._armDeferred(workflow, trigger, dedupeKey);
+  }
+
+  async sweep(only = null) {
+    const deferredKinds = [...this.workflows.values()].filter((w) => w.retrySchedule).map((w) => w.kind);
+    const kinds = only ? deferredKinds.filter((kind) => only.includes(kind)) : deferredKinds;
+    if (!kinds.length) return [];
+
+    const orphaned = await DeferredWork.find({
+      kind: { $in: kinds },
+      $or: [
+        { status: 'pending' },
+        { status: 'processing', claimedAt: { $lt: new Date(Date.now() - STALE_CLAIM_MS) } }
+      ]
+    });
+
+    const rearmed = orphaned.map((work) => this._arm(
+      this.workflows.get(work.kind),
+      work._id,
+      this._delayUntilDue(work),
+    ));
+    if (rearmed.length) logger.info('[WorkflowRunner] Swept', { kinds, rearmed: rearmed.length });
+    return rearmed;
+  }
+
+  _startSweeping() {
+    if (this.stopped || this.sweepTimer || !this.sweepIntervalMs) return;
+    this.sweepTimer = setInterval(() => {
+      this.sweep().catch((error) => logger.error('[WorkflowRunner] Periodic sweep failed', { error: error.message }));
+    }, this.sweepIntervalMs);
+    this.sweepTimer.unref();
+  }
+
+  stop() {
+    this.stopped = true;
+    if (this.sweepTimer) clearInterval(this.sweepTimer);
+    this.sweepTimer = null;
+    for (const timer of this.timers.values()) clearTimeout(timer);
+    this.timers.clear();
+  }
+
+  async _armDeferred(workflow, trigger, dedupeKey) {
+    const { kind } = workflow;
+    await withIndexTimeout(ensureIndexes());
+    const work = await DeferredWork.findOne({ kind, dedupeKey })
+      || await DeferredWork.create({ kind, dedupeKey, trigger }).catch(async (error) => {
+        if (error.code !== 11000) throw error;
+        return await DeferredWork.findOne({ kind, dedupeKey });
+      });
+    if (work.status === 'abandoned') {
+      const { modifiedCount } = await DeferredWork.updateOne(
+        { _id: work._id, status: 'abandoned' },
+        { $set: { status: 'pending', attempt: 0, nextAttemptAt: new Date(), lastError: null, claimedAt: null } }
+      );
+      if (!modifiedCount) return null;
+      logger.info('[WorkflowRunner] Re-arming abandoned work', { kind, dedupeKey });
+      return this._arm(workflow, work._id, 0);
+    }
+    if (work.status !== 'pending') {
+      logger.info('[WorkflowRunner] Skipping, work is in flight or complete', { kind, dedupeKey, status: work.status });
+      return null;
+    }
+    return this._arm(workflow, work._id, this._delayUntilDue(work));
+  }
+
+  _delayUntilDue(work) {
+    return Math.max(0, new Date(work.nextAttemptAt).getTime() - Date.now());
+  }
+
+  _arm(workflow, workId, delayMs) {
+    const id = String(workId);
+    // An attempt still in flight when stop() lands must not re-arm after shutdown. The
+    // record stays pending in Mongo, so the next process recovers it on its sweep.
+    if (this.stopped) return id;
+    clearTimeout(this.timers.get(id));
+    const timer = setTimeout(() => {
+      this.timers.delete(id);
+      this._runDeferred(workflow, id).catch((error) => {
+        logger.error('[WorkflowRunner] Deferred run failed', { kind: workflow.kind, deferredWorkId: id, error: error.message });
+      });
+    }, delayMs);
+    timer.unref();
+    this.timers.set(id, timer);
+    return id;
+  }
+
+  async _runDeferred(workflow, deferredWorkId) {
+    const claimedAt = new Date();
+    const prior = await DeferredWork.findOneAndUpdate(
+      {
+        _id: deferredWorkId,
+        $or: [
+          { status: 'pending', nextAttemptAt: { $lte: claimedAt } },
+          { status: 'processing', claimedAt: { $lt: new Date(claimedAt.getTime() - STALE_CLAIM_MS) } }
+        ]
+      },
+      { $set: { status: 'processing', claimedAt } },
+      { new: false }
+    );
+    if (!prior) {
+      logger.info('[WorkflowRunner] Claimed elsewhere or not due', { kind: workflow.kind, deferredWorkId });
+      return { claimed: false };
+    }
+
+    const reclaimed = prior.status === 'processing';
+    const work = { ...prior.toObject(), attempt: prior.attempt + (reclaimed ? 1 : 0) };
+    if (reclaimed) await this._write(work, claimedAt, { attempt: work.attempt });
+
+    const checkpoints = { ...(work.checkpoints || {}) };
+    const save = async (patch) => {
+      Object.assign(checkpoints, patch);
+      const $set = {};
+      for (const [key, value] of Object.entries(patch || {})) $set[`checkpoints.${key}`] = value;
+      if (Object.keys($set).length) await this._write(work, claimedAt, $set);
+    };
+
+    try {
+      await workflow.prepare(work.trigger, checkpoints, save);
+    } catch (error) {
+      return await this._rescheduleOrAbandon(workflow, work, claimedAt, error);
+    }
+    await this._write(work, claimedAt, { status: 'done', claimedAt: null });
+    return { status: 'done' };
+  }
+
+  // matchedCount, not modifiedCount: the fence is "do we still hold the claim", and
+  // modifiedCount is also 0 for a write that matched but set an identical value - a
+  // save() rewriting an unchanged checkpoint is not a lost claim.
+  async _write(work, claimedAt, $set) {
+    const { matchedCount } = await DeferredWork.updateOne({ _id: work._id, claimedAt }, { $set });
+    if (!matchedCount) logger.warn('[WorkflowRunner] Claim lost, write discarded', { deferredWorkId: String(work._id) });
+    return matchedCount > 0;
+  }
+
+  async _rescheduleOrAbandon(workflow, work, claimedAt, error) {
+    // prepare() is consumer code and can throw anything, including null. Dereferencing
+    // it here used to raise a TypeError out of this handler, so neither branch ran and
+    // the record stayed `processing` until stale recovery hit the same failure again.
+    const lastError = describeThrown(error);
+    const delayMinutes = workflow.retrySchedule[work.attempt];
+    if (error?.permanent === true || delayMinutes === undefined) {
+      await this._write(work, claimedAt, { status: 'abandoned', lastError, claimedAt: null });
+      logger.error('[WorkflowRunner] Abandoned', {
+        kind: workflow.kind,
+        dedupeKey: work.dedupeKey,
+        attempt: work.attempt,
+        permanent: error?.permanent === true,
+        error: lastError
+      });
+      return { status: 'abandoned' };
+    }
+
+    const attempt = work.attempt + 1;
+    const delayMs = delayMinutes * MINUTE_MS;
+    const kept = await this._write(work, claimedAt, {
+      status: 'pending',
+      attempt,
+      nextAttemptAt: new Date(Date.now() + delayMs),
+      lastError,
+      claimedAt: null
+    });
+    if (!kept) return { claimed: false };
+
+    logger.warn('[WorkflowRunner] Rescheduled', {
+      kind: workflow.kind,
+      dedupeKey: work.dedupeKey,
+      attempt,
+      delayMinutes,
+      error: lastError
+    });
+    this._arm(workflow, work._id, delayMs);
+    return { status: 'pending', attempt };
+  }
+}
+
+module.exports = { WorkflowRunner };

package/lib/index.d.ts

@@ -243,6 +243,7 @@ declare module '@peopl-health/nexus' {
     getAssistantProcessor(): AssistantProcessor;
     getLlmMonitor(): { start(options?: { scheduleDaily?: boolean }): Promise<{ enabled: boolean }> } | null;
     initializeLlmMonitor(): Promise<{ enabled: boolean; error?: string } | undefined>;
+    getWorkflowRunner(): WorkflowRunner;
     processInstruction(code: string, instruction: string, role?: string, options?: { triggeredBy?: string }): Promise<void>;
     processSystemMessage(code: string, messages: string | string[], role?: string, options?: { triggeredBy?: string; reply?: boolean }): Promise<void>;
     processOutreach(code: string, options?: { brief?: OutreachBrief | null; triggeredBy?: string | null; reason?: string | null; firstName?: string | null; dryRun?: boolean }): Promise<OutreachDecision>;
@@ -520,16 +521,17 @@ declare module '@peopl-health/nexus' {
   export interface Workflow {
     kind: string;
     dedupeKey: (trigger: any) => string;
-    prepare: (trigger: any) => Promise<any>;
+    prepare: (trigger: any, checkpoints?: any, save?: (patch: any) => Promise<void>) => Promise<any>;
+    retrySchedule?: number[];
   }
 
   export interface WorkflowRunner {
     register(workflow: Workflow): Promise<Workflow>;
-    enqueue(kind: string, trigger: any, options?: QueueJobOptions): Promise<string>;
+    enqueue(kind: string, trigger: any, options?: QueueJobOptions): Promise<string | null>;
+    sweep(only?: string[]): Promise<string[]>;
+    stop(): void;
   }
 
-  export function createWorkflowRunner(options: { queueAdapter: QueueAdapter }): WorkflowRunner;
-
   // Memory System
   export interface PatientMemoryDocument {
     _id: any;

package/lib/index.js

@@ -27,7 +27,6 @@ const { BaileysProvider } = require('./adapters/BaileysProvider');
 const { setPreprocessingHandler, hasPreprocessingHandler, invokePreprocessingHandler } = require('./services/preprocessingService');
 const { requestIdMiddleware, getRequestId } = require('./middleware/requestId');
 const { QueueAdapter, LocalQueueAdapter, RedisQueueAdapter, createQueueAdapter, registerQueueAdapter } = require('./queue');
-const { createWorkflowRunner } = require('./core/workflowRunner');
 const routes = require('./routes');
 const { resetAll } = require('./config/lifecycle');
 const { EvalProvider } = require('./eval/EvalProvider');
@@ -229,7 +228,6 @@ class Nexus {
 }
 
 module.exports = {
-  createWorkflowRunner,
   Nexus,
   TwilioProvider,
   BaileysProvider,

package/lib/models/deferredWorkModel.js

@@ -0,0 +1,24 @@
+const mongoose = require('mongoose');
+
+const STATUS_VALUES = ['pending', 'processing', 'done', 'abandoned'];
+const RETENTION_MS = 24 * 60 * 60 * 1000;
+
+const deferredWorkSchema = new mongoose.Schema({
+  kind: { type: String, required: true },
+  dedupeKey: { type: String, required: true },
+  trigger: { type: mongoose.Schema.Types.Mixed, default: null },
+  checkpoints: { type: mongoose.Schema.Types.Mixed, default: () => ({}) },
+  status: { type: String, enum: STATUS_VALUES, default: 'pending' },
+  attempt: { type: Number, default: 0 },
+  nextAttemptAt: { type: Date, default: Date.now },
+  claimedAt: { type: Date, default: null },
+  lastError: { type: String, default: null }
+}, { timestamps: true });
+
+deferredWorkSchema.index({ kind: 1, dedupeKey: 1 }, { unique: true, name: 'dedupe_idx' });
+deferredWorkSchema.index({ status: 1, nextAttemptAt: 1 }, { name: 'due_idx' });
+deferredWorkSchema.index({ updatedAt: 1 }, { expireAfterSeconds: RETENTION_MS / 1000, name: 'ttl_idx' });
+
+const DeferredWork = mongoose.model('DeferredWork', deferredWorkSchema);
+
+module.exports = { DeferredWork, RETENTION_MS };

package/lib/routes/index.js

@@ -70,7 +70,6 @@ const patientRouteDefinitions = {
   'GET /:code': 'getPatientInfoController',
   'GET /triage/:code': 'getPatientTriageInfoController',
   'GET /careplan/:code': 'getPatientCarePlanController',
-  'GET /escalation-context/:code': 'getPatientEscalationContextController',
   'GET /tasks/:code': 'getPatientTasksController',
   'PATCH /:code': 'updatePatientController',
   'PATCH /tasks/:code/:kind/:recordId': 'updatePatientTaskController',
@@ -215,7 +214,6 @@ const builtInControllers = {
   getPatientInfoController: patientController.getPatientInfoController,
   getPatientTriageInfoController: patientController.getPatientTriageInfoController,
   getPatientCarePlanController: patientController.getPatientCarePlanController,
-  getPatientEscalationContextController: patientController.getPatientEscalationContextController,
   getPatientTasksController: patientController.getPatientTasksController,
   getPatientRecommendationCarouselsController: patientController.getPatientRecommendationCarouselsController,
   getPatientRecommendationCarouselImageController: patientController.getPatientRecommendationCarouselImageController,

package/lib/services/airtableService.js

@@ -15,6 +15,11 @@ const AIRTABLE_API_URL = 'https://api.airtable.com/v0';
 const AXIOS_TIMEOUT_CODE = 'ECONNABORTED';
 const RETRY_BASE_DELAY_MS = 1000;
 
+function isPermanentAirtableError(error) {
+  const status = error?.statusCode;
+  return typeof status === 'number' && status >= 400 && status < 500 && status !== 429;
+}
+
 let isEvalMode = false;
 
 function setEvalMode(enabled) {
@@ -42,6 +47,7 @@ async function withAirtableRetry(operation, label, { retryNetworkErrors = true }
       const isTransient = TRANSIENT_STATUS_CODES.includes(error.statusCode)
         || (retryNetworkErrors && isTransientNetworkError(error));
       if (!isTransient || attempt === MAX_ATTEMPTS) {
+        if (isPermanentAirtableError(error)) error.permanent = true;
         throw error;
       }
       const baseDelay = error.statusCode === RATE_LIMITED_STATUS ? RATE_LIMIT_DELAY_MS : RETRY_BASE_DELAY_MS;
@@ -229,6 +235,7 @@ async function upsertRecord(baseID, tableName, fields, { mergeOn } = {}, context
 
 module.exports = {
   setEvalMode,
+  isPermanentAirtableError,
   addRecord,
   getRecords,
   getRecordByFilter,

package/lib/utils/sanitizerUtils.js

@@ -149,7 +149,6 @@ function redactDirectIdentifiers(value) {
 }
 
 module.exports = {
-  PATIENT_CODE,
   decodeUnicodeEscapes,
   decodeUnicodeEscapesDeep,
   collectStringsDeep,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@peopl-health/nexus",
-  "version": "5.45.0-dev.7144",
+  "version": "5.45.0-dev.7159",
   "description": "Core messaging and assistant library for WhatsApp communication platforms",
   "keywords": [
     "whatsapp",

package/lib/core/workflowRunner.js

@@ -1,35 +0,0 @@
-function createWorkflowRunner({ queueAdapter } = {}) {
-  if (!queueAdapter) throw new Error('createWorkflowRunner requires a queueAdapter');
-  const workflows = new Map();
-
-  async function register(workflow = {}) {
-    if (!workflow.kind || typeof workflow.kind !== 'string') throw new Error('workflow requires a kind');
-    if (workflow.kind.includes('__')) throw new Error(`workflow '${workflow.kind}' must not contain '__' (reserved as the jobId delimiter)`);
-    if (typeof workflow.dedupeKey !== 'function') throw new Error(`workflow '${workflow.kind}' requires a dedupeKey function`);
-    if (typeof workflow.prepare !== 'function') throw new Error(`workflow '${workflow.kind}' requires a prepare function`);
-    if (workflows.has(workflow.kind)) throw new Error(`workflow '${workflow.kind}' is already registered`);
-    workflows.set(workflow.kind, workflow);
-    try {
-      await queueAdapter.process(workflow.kind, (trigger) => workflow.prepare(trigger));
-    } catch (error) {
-      workflows.delete(workflow.kind);
-      throw error;
-    }
-    return workflow;
-  }
-
-  function enqueue(kind, trigger, options = {}) {
-    const workflow = workflows.get(kind);
-    if (!workflow) throw new Error(`no workflow registered for '${kind}'`);
-    const dedupeKey = workflow.dedupeKey(trigger);
-    if (dedupeKey == null || (typeof dedupeKey === 'string' && !dedupeKey.trim())) {
-      throw new Error(`workflow '${kind}' produced an empty dedupe key`);
-    }
-    if (typeof dedupeKey !== 'string') throw new Error(`workflow '${kind}' produced a non-string dedupe key`);
-    return queueAdapter.enqueue(kind, trigger, { ...options, jobId: `${kind}__${dedupeKey}` });
-  }
-
-  return { register, enqueue };
-}
-
-module.exports = { createWorkflowRunner };

package/lib/services/escalationContextService.js

@@ -1,81 +0,0 @@
-const { isoDay } = require('../utils/dateUtils');
-const { logger } = require('../utils/logger');
-const { PATIENT_CODE, redactDirectIdentifiers } = require('../utils/sanitizerUtils');
-const { withTimeout } = require('../utils/timeoutUtils');
-const { ensureWhatsAppFormat } = require('../helpers/twilioHelper');
-const { assembleHistory } = require('../clinical/services/patientHistoryService');
-const { DefaultMemoryManager } = require('../clinical/memory/DefaultMemoryManager');
-
-const { getCarePlanHistory } = require('./carePlanHistoryService');
-const { getPatientInformation } = require('./patientService');
-const { getTriageHistory } = require('./triageHistoryService');
-
-const SECTION_TIMEOUT_MS = 10000;
-const TRIAGE_WINDOW_DAYS = 30;
-const CARE_PLAN_LIMIT = 5;
-const HISTORY_SCOPE = ['closed_cases_90d', 'recurrence_patterns', 'recent_medications', 'patient_reported_procedures'];
-
-const httpError = (statusCode, message) => Object.assign(new Error(message), { statusCode });
-
-function sectionReaders(code, now) {
-  const memoryManager = new DefaultMemoryManager();
-  const triageStart = isoDay(new Date(now - TRIAGE_WINDOW_DAYS * 86400000).toISOString());
-  return {
-    agentContext: () => memoryManager.getClinicalData(code),
-    conversation: () => memoryManager.buildContext({ thread: { code } }),
-    triages: () => getTriageHistory({ code, startDate: triageStart }),
-    carePlan: () => getCarePlanHistory({ code, limit: CARE_PLAN_LIMIT }),
-    history: () => assembleHistory(code, { scope: HISTORY_SCOPE }),
-  };
-}
-
-async function getEscalationContext({ code, now = Date.now(), timeoutMs = SECTION_TIMEOUT_MS } = {}) {
-  const patientCode = ensureWhatsAppFormat(String(code ?? '').trim());
-  if (!PATIENT_CODE.test(patientCode)) throw httpError(400, 'A valid WhatsApp ID is required');
-
-  const patient = await withTimeout(getPatientInformation(patientCode), timeoutMs, 'patient lookup');
-  if (!patient) throw httpError(404, 'Patient not found');
-
-  const readers = sectionReaders(patientCode, now);
-  const names = Object.keys(readers);
-  const outcomes = await Promise.allSettled(names.map((name) => withTimeout(readers[name](), timeoutMs, name)));
-
-  const context = { patientCode, asOf: new Date(now).toISOString(), degradedSections: [] };
-  outcomes.forEach((outcome, index) => {
-    const name = names[index];
-    if (outcome.status === 'fulfilled' && outcome.value != null) {
-      context[name] = outcome.value;
-      return;
-    }
-    context[name] = null;
-    context.degradedSections.push(name);
-    if (outcome.status === 'rejected') {
-      logger.error('[escalationContextService] Section reader failed', {
-        code: patientCode,
-        section: name,
-        error: redactDirectIdentifiers(String(outcome.reason?.message ?? outcome.reason)),
-        stack: redactDirectIdentifiers(String(outcome.reason?.stack ?? '')),
-      });
-    }
-  });
-
-  if (context.conversation) {
-    context.conversation = context.conversation
-      .filter((item) => item.type === 'message')
-      .map(({ role, content }) => ({ role, content }));
-  }
-  if (context.triages) {
-    if (context.triages.pagination?.hasNext) context.degradedSections.push('triages');
-    context.triages = context.triages.records;
-  }
-  if (context.carePlan) {
-    const { records, pagination = null, currentState = null } = context.carePlan;
-    const { degradedSections: carePlanDegraded = [], ...state } = currentState || {};
-    context.carePlan = { records, pagination, currentState: currentState ? state : null };
-    for (const section of carePlanDegraded) context.degradedSections.push(`carePlan.${section}`);
-  }
-
-  return context;
-}
-
-module.exports = { getEscalationContext };