@peopl-health/nexus 5.44.0-dev.6216 → 5.44.0-dev.6227

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -116,38 +116,6 @@ const bus = nexus.getMessaging().getEventBus();
116
116
  bus.on('message:received', (m) => console.log('rx', m.id));
117
117
  ```
118
118
 
119
- ## Workflows (Optional)
120
-
121
- Register a named unit of work, deduplicated by a key you derive from the trigger.
122
-
123
- ```js
124
- const runner = nexus.getMessaging().getWorkflowRunner();
125
-
126
- await runner.register({
127
- kind: 'triage-projection',
128
- dedupeKey: (trigger) => String(trigger.submissionId),
129
- prepare: async (trigger, checkpoints, save) => { /* ... */ },
130
- retrySchedule: [5, 10, 20, 40], // minutes; omit for fire-and-forget
131
- });
132
-
133
- await runner.enqueue('triage-projection', { submissionId });
134
- ```
135
-
136
- With `retrySchedule` the trigger and a checkpoint journal are stored in Mongo and the schedule runs
137
- on an in-process timer — deferred workflows never touch the queue adapter, so they add no Redis load.
138
- `prepare` resumes from `checkpoints`; call `save(patch)` before each external call so a retry does
139
- not repeat completed work. Throw an error carrying `permanent: true` to give up immediately instead
140
- of walking the schedule; the Airtable helpers tag their own permanent 4xx errors that way, in
141
- `withAirtableRetry`, which all five of them route through. They rethrow on failure too, so an
142
- Airtable outage reaches `prepare` as an exception and walks the retry schedule on its own rather
143
- than reading as success.
144
-
145
- Recovery is automatic: registering sweeps for work left behind by a previous process, and the runner
146
- keeps sweeping so work whose worker died mid-`prepare` is reclaimed once its claim goes stale.
147
- `enqueue` on an abandoned key revives it, keeping its checkpoints. Work is retained for 24 hours
148
- after its last update and then expires — the dedupe key expires with it, so idempotency has the
149
- same horizon as retention.
150
-
151
119
  ## Assistants (Optional)
152
120
 
153
121
  Register assistant classes and (optionally) a custom resolver. OpenAI is supported via `llm: 'openai'`.
@@ -32,7 +32,6 @@ const { isLlmMonitorEnabled } = require('../clinical/flags/llmMonitorConfig');
32
32
  const { createQueueAdapter } = require('../queue');
33
33
  const { ScheduledMessageJob } = require('../jobs/ScheduledMessageJob');
34
34
  const { TemplateApprovalJob } = require('../jobs/TemplateApprovalJob');
35
- const { WorkflowRunner } = require('./WorkflowRunner');
36
35
 
37
36
  const { PhiProcessor } = require('./PhiProcessor');
38
37
 
@@ -117,8 +116,6 @@ class NexusMessaging {
117
116
  queueAdapter: this.queueAdapter,
118
117
  });
119
118
 
120
- this.workflowRunner = new WorkflowRunner({ queueAdapter: this.queueAdapter });
121
-
122
119
  this.phiProcessor = new PhiProcessor({
123
120
  encode: config.phi?.encode || false,
124
121
  ner: config.phi?.ner || null,
@@ -301,7 +298,6 @@ class NexusMessaging {
301
298
  getAssistantProcessor() { return this.assistantProcessor; }
302
299
  getLlmMonitor() { return this.llmMonitor; }
303
300
  getQueueAdapter() { return this.queueAdapter; }
304
- getWorkflowRunner() { return this.workflowRunner; }
305
301
  getPhiProcessor() { return this.phiProcessor; }
306
302
  isConnected() { return this.provider?.getConnectionStatus() ?? false; }
307
303
  isProcessing(chatId) { return this.batchingManager.isProcessing(chatId); }
@@ -737,7 +733,6 @@ class NexusMessaging {
737
733
  async disconnect() {
738
734
  this.queueReconciliation?.stop();
739
735
  if (this.provider) await this.provider.disconnect();
740
- if (this.workflowRunner) this.workflowRunner.stop();
741
736
  if (this.queueAdapter) await this.queueAdapter.shutdown();
742
737
  this.events.removeAllListeners();
743
738
  }
@@ -0,0 +1,35 @@
1
+ function createWorkflowRunner({ queueAdapter } = {}) {
2
+ if (!queueAdapter) throw new Error('createWorkflowRunner requires a queueAdapter');
3
+ const workflows = new Map();
4
+
5
+ async function register(workflow = {}) {
6
+ if (!workflow.kind || typeof workflow.kind !== 'string') throw new Error('workflow requires a kind');
7
+ if (workflow.kind.includes('__')) throw new Error(`workflow '${workflow.kind}' must not contain '__' (reserved as the jobId delimiter)`);
8
+ if (typeof workflow.dedupeKey !== 'function') throw new Error(`workflow '${workflow.kind}' requires a dedupeKey function`);
9
+ if (typeof workflow.prepare !== 'function') throw new Error(`workflow '${workflow.kind}' requires a prepare function`);
10
+ if (workflows.has(workflow.kind)) throw new Error(`workflow '${workflow.kind}' is already registered`);
11
+ workflows.set(workflow.kind, workflow);
12
+ try {
13
+ await queueAdapter.process(workflow.kind, (trigger) => workflow.prepare(trigger));
14
+ } catch (error) {
15
+ workflows.delete(workflow.kind);
16
+ throw error;
17
+ }
18
+ return workflow;
19
+ }
20
+
21
+ function enqueue(kind, trigger, options = {}) {
22
+ const workflow = workflows.get(kind);
23
+ if (!workflow) throw new Error(`no workflow registered for '${kind}'`);
24
+ const dedupeKey = workflow.dedupeKey(trigger);
25
+ if (dedupeKey == null || (typeof dedupeKey === 'string' && !dedupeKey.trim())) {
26
+ throw new Error(`workflow '${kind}' produced an empty dedupe key`);
27
+ }
28
+ if (typeof dedupeKey !== 'string') throw new Error(`workflow '${kind}' produced a non-string dedupe key`);
29
+ return queueAdapter.enqueue(kind, trigger, { ...options, jobId: `${kind}__${dedupeKey}` });
30
+ }
31
+
32
+ return { register, enqueue };
33
+ }
34
+
35
+ module.exports = { createWorkflowRunner };
package/lib/index.d.ts CHANGED
@@ -242,7 +242,6 @@ declare module '@peopl-health/nexus' {
242
242
  getAssistantProcessor(): AssistantProcessor;
243
243
  getLlmMonitor(): { start(options?: { scheduleDaily?: boolean }): Promise<{ enabled: boolean }> } | null;
244
244
  initializeLlmMonitor(): Promise<{ enabled: boolean; error?: string } | undefined>;
245
- getWorkflowRunner(): WorkflowRunner;
246
245
  processInstruction(code: string, instruction: string, role?: string, options?: { triggeredBy?: string }): Promise<void>;
247
246
  processSystemMessage(code: string, messages: string | string[], role?: string, options?: { triggeredBy?: string; reply?: boolean }): Promise<void>;
248
247
  isConnected(): boolean;
@@ -455,17 +454,16 @@ declare module '@peopl-health/nexus' {
455
454
  export interface Workflow {
456
455
  kind: string;
457
456
  dedupeKey: (trigger: any) => string;
458
- prepare: (trigger: any, checkpoints?: any, save?: (patch: any) => Promise<void>) => Promise<any>;
459
- retrySchedule?: number[];
457
+ prepare: (trigger: any) => Promise<any>;
460
458
  }
461
459
 
462
460
  export interface WorkflowRunner {
463
461
  register(workflow: Workflow): Promise<Workflow>;
464
- enqueue(kind: string, trigger: any, options?: QueueJobOptions): Promise<string | null>;
465
- sweep(only?: string[]): Promise<string[]>;
466
- stop(): void;
462
+ enqueue(kind: string, trigger: any, options?: QueueJobOptions): Promise<string>;
467
463
  }
468
464
 
465
+ export function createWorkflowRunner(options: { queueAdapter: QueueAdapter }): WorkflowRunner;
466
+
469
467
  // Memory System
470
468
  export interface PatientMemoryDocument {
471
469
  _id: any;
package/lib/index.js CHANGED
@@ -27,6 +27,7 @@ const { BaileysProvider } = require('./adapters/BaileysProvider');
27
27
  const { setPreprocessingHandler, hasPreprocessingHandler, invokePreprocessingHandler } = require('./services/preprocessingService');
28
28
  const { requestIdMiddleware, getRequestId } = require('./middleware/requestId');
29
29
  const { QueueAdapter, LocalQueueAdapter, RedisQueueAdapter, createQueueAdapter, registerQueueAdapter } = require('./queue');
30
+ const { createWorkflowRunner } = require('./core/workflowRunner');
30
31
  const routes = require('./routes');
31
32
  const { resetAll } = require('./config/lifecycle');
32
33
  const { EvalProvider } = require('./eval/EvalProvider');
@@ -220,6 +221,7 @@ class Nexus {
220
221
  }
221
222
 
222
223
  module.exports = {
224
+ createWorkflowRunner,
223
225
  Nexus,
224
226
  TwilioProvider,
225
227
  BaileysProvider,
@@ -9,7 +9,8 @@ const PHI_CATEGORIES = Object.freeze(['PERSONA', 'TELEFONO', 'EMAIL', 'ID']);
9
9
  const tokenMapSchema = new mongoose.Schema({
10
10
  code: { type: String, required: true },
11
11
  encode: { type: Map, of: String, default: () => new Map() },
12
- decode: { type: Map, of: String, default: () => new Map() }
12
+ decode: { type: Map, of: String, default: () => new Map() },
13
+ seq: { type: Number }
13
14
  }, { timestamps: true });
14
15
 
15
16
  tokenMapSchema.index({ code: 1 }, { unique: true });
@@ -11,11 +11,6 @@ const TRANSIENT_ERROR_CODES = ['ETIMEDOUT', 'ECONNRESET', 'ECONNREFUSED', 'ENOTF
11
11
  const MAX_ATTEMPTS = 3;
12
12
  const RETRY_BASE_DELAY_MS = 1000;
13
13
 
14
- function isPermanentAirtableError(error) {
15
- const status = error?.statusCode;
16
- return typeof status === 'number' && status >= 400 && status < 500 && status !== 429;
17
- }
18
-
19
14
  let isEvalMode = false;
20
15
 
21
16
  function setEvalMode(enabled) {
@@ -43,7 +38,6 @@ async function withAirtableRetry(operation, label, { retryNetworkErrors = true }
43
38
  const isTransient = TRANSIENT_STATUS_CODES.includes(error.statusCode)
44
39
  || (retryNetworkErrors && isTransientNetworkError(error));
45
40
  if (!isTransient || attempt === MAX_ATTEMPTS) {
46
- if (isPermanentAirtableError(error)) error.permanent = true;
47
41
  throw error;
48
42
  }
49
43
  const baseDelay = error.statusCode === RATE_LIMITED_STATUS ? RATE_LIMIT_DELAY_MS : RETRY_BASE_DELAY_MS;
@@ -175,7 +169,6 @@ async function addLinkedRecord(baseID, targetTable, fields, linkConfig, context
175
169
 
176
170
  module.exports = {
177
171
  setEvalMode,
178
- isPermanentAirtableError,
179
172
  addRecord,
180
173
  getRecords,
181
174
  getRecordByFilter,
@@ -24,12 +24,23 @@ const preferredSurface = (current, candidate) => {
24
24
  return richer > 0 ? candidate : current;
25
25
  };
26
26
 
27
- const tokenCache = new MapCache({ maxSize: 500, ttl: 60 * 1000 });
27
+ const CLAIM_ATTEMPTS = 3;
28
+
29
+ const NEXT_INDEX_PIPELINE = [{
30
+ $set: {
31
+ seq: {
32
+ $add: [
33
+ { $ifNull: ['$seq', { $size: { $objectToArray: { $ifNull: ['$encode', {}] } } }] },
34
+ 1,
35
+ ],
36
+ },
37
+ },
38
+ }];
28
39
 
29
- const getSubmap = async (code) => {
30
- const cached = tokenCache.get(code);
31
- if (cached) return cached;
40
+ const tokenCache = new MapCache({ maxSize: 500, ttl: 60 * 1000 });
41
+ const inFlight = new Map();
32
42
 
43
+ const readSubmap = async (code) => {
33
44
  const doc = await getTokenMap().findOne({ code }).lean();
34
45
  const submap = {
35
46
  encode: new Map(Object.entries(doc?.encode || {})),
@@ -39,32 +50,34 @@ const getSubmap = async (code) => {
39
50
  return submap;
40
51
  };
41
52
 
42
- const allocateToken = async (code, plaintext, category, normalized, submap) => {
43
- const counter = submap.encode.size + 1;
44
- const token = `<<${category}_${counter}>>`;
53
+ const getSubmap = async (code) => {
54
+ const cached = tokenCache.get(code);
55
+ if (cached) return cached;
56
+ if (!inFlight.has(code)) {
57
+ inFlight.set(code, readSubmap(code).finally(() => inFlight.delete(code)));
58
+ }
59
+ return inFlight.get(code);
60
+ };
45
61
 
46
- submap.encode.set(normalized, token);
47
- submap.decode.set(token, plaintext);
62
+ const nextIndex = async (code) => {
63
+ const doc = await getTokenMap().findOneAndUpdate({ code }, NEXT_INDEX_PIPELINE, {
64
+ upsert: true, new: true, projection: { seq: 1 }, lean: true,
65
+ });
66
+ return doc.seq;
67
+ };
48
68
 
49
- const res = await getTokenMap().updateOne(
50
- { code },
51
- {
52
- $set: {
53
- encode: Object.fromEntries(submap.encode),
54
- decode: Object.fromEntries(submap.decode),
55
- }
56
- },
57
- { upsert: true }
58
- );
59
- if (!res || (res.modifiedCount === 0 && !res.upsertedCount)) {
60
- logger.error('[allocateToken] Mapping write modified 0 docs', { code, category, token });
61
- throw new Error(`[allocateToken] Mapping write modified 0 docs for token ${token}`);
62
- }
63
- return token;
69
+ const rememberSurface = (code, token, plaintext) => {
70
+ const live = tokenCache.get(code);
71
+ if (live && plaintext !== undefined) live.decode.set(token, plaintext);
72
+ };
73
+
74
+ const rememberMapping = (code, normalized, token, plaintext) => {
75
+ const live = tokenCache.get(code);
76
+ if (live) live.encode.set(normalized, token);
77
+ rememberSurface(code, token, plaintext);
64
78
  };
65
79
 
66
- const upgradeSurface = async (code, token, plaintext, submap) => {
67
- const current = submap.decode.get(token);
80
+ const upgradeSurface = async (code, token, current, plaintext) => {
68
81
  const preferred = preferredSurface(current, plaintext);
69
82
  if (preferred === current) return;
70
83
 
@@ -75,13 +88,55 @@ const upgradeSurface = async (code, token, plaintext, submap) => {
75
88
  );
76
89
 
77
90
  if (res?.modifiedCount) {
78
- submap.decode.set(token, preferred);
91
+ rememberSurface(code, token, preferred);
79
92
  logger.debug('[upgradeSurface] Decode surface upgraded', { code, token });
80
93
  return;
81
94
  }
82
95
  tokenCache.delete(code);
83
96
  };
84
97
 
98
+ const claimToken = async (code, plaintext, category, normalized) => {
99
+ const token = `<<${category}_${await nextIndex(code)}>>`;
100
+ const res = await getTokenMap().updateOne(
101
+ {
102
+ code,
103
+ [`encode.${normalized}`]: { $exists: false },
104
+ [`decode.${token}`]: { $exists: false },
105
+ },
106
+ { $set: { [`encode.${normalized}`]: token, [`decode.${token}`]: plaintext } },
107
+ );
108
+ return res?.modifiedCount ? token : null;
109
+ };
110
+
111
+ const adoptClaimedToken = async (code, plaintext, normalized) => {
112
+ const doc = await getTokenMap().findOne({ code }).lean();
113
+ const winner = doc?.encode?.[normalized];
114
+ if (!winner) return null;
115
+
116
+ rememberMapping(code, normalized, winner, doc.decode?.[winner]);
117
+ await upgradeSurface(code, winner, doc.decode?.[winner], plaintext);
118
+ return winner;
119
+ };
120
+
121
+ const allocateToken = async (code, plaintext, category, normalized) => {
122
+ for (let attempt = 1; attempt <= CLAIM_ATTEMPTS; attempt += 1) {
123
+ let token = null;
124
+ try {
125
+ token = await claimToken(code, plaintext, category, normalized);
126
+ } catch (err) {
127
+ if (err?.code !== 11000 || attempt === CLAIM_ATTEMPTS) throw err;
128
+ }
129
+ if (token) {
130
+ rememberMapping(code, normalized, token, plaintext);
131
+ return token;
132
+ }
133
+ const winner = await adoptClaimedToken(code, plaintext, normalized);
134
+ if (winner) return winner;
135
+ logger.error('[allocateToken] Token claim failed, retrying', { code, category, attempt });
136
+ }
137
+ throw new Error(`[allocateToken] Could not claim a ${category} token for ${code}`);
138
+ };
139
+
85
140
  const encode = async (code, plaintext, category) => {
86
141
  if (!PHI_CATEGORIES.includes(category)) {
87
142
  throw new Error(`[encode] Unknown category: ${category}`);
@@ -96,11 +151,11 @@ const encode = async (code, plaintext, category) => {
96
151
  const submap = await getSubmap(code);
97
152
  const existing = submap.encode.get(normalized);
98
153
  if (existing) {
99
- await upgradeSurface(code, existing, String(plaintext), submap);
154
+ await upgradeSurface(code, existing, submap.decode.get(existing), String(plaintext));
100
155
  return existing;
101
156
  }
102
157
 
103
- const token = await allocateToken(code, String(plaintext), category, normalized, submap);
158
+ const token = await allocateToken(code, String(plaintext), category, normalized);
104
159
  logger.debug('[encode] Allocated', { code, category, token });
105
160
  return token;
106
161
  };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@peopl-health/nexus",
3
- "version": "5.44.0-dev.6216",
3
+ "version": "5.44.0-dev.6227",
4
4
  "description": "Core messaging and assistant library for WhatsApp communication platforms",
5
5
  "keywords": [
6
6
  "whatsapp",
@@ -1,265 +0,0 @@
1
- const { describeThrown } = require('../utils/errorUtils');
2
- const { logger } = require('../utils/logger');
3
-
4
- const { DeferredWork, RETENTION_MS } = require('../models/deferredWorkModel');
5
-
6
- const STALE_CLAIM_MS = 5 * 60 * 1000;
7
- const MINUTE_MS = 60 * 1000;
8
- const INDEX_TIMEOUT_MS = 1000;
9
- const SWEEP_INTERVAL_MS = 60 * 1000;
10
-
11
- let indexesReady = null;
12
-
13
- function ensureIndexes() {
14
- if (!indexesReady) {
15
- indexesReady = DeferredWork.init().catch((error) => {
16
- indexesReady = null;
17
- logger.error('[WorkflowRunner] Index build failed, dedupe and retention are not guaranteed', { error: error.message });
18
- });
19
- }
20
- return indexesReady;
21
- }
22
-
23
- function withIndexTimeout(promise) {
24
- let timer;
25
- const timeout = new Promise((resolve) => {
26
- timer = setTimeout(resolve, INDEX_TIMEOUT_MS);
27
- timer.unref();
28
- });
29
- return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
30
- }
31
-
32
- class WorkflowRunner {
33
- constructor({ queueAdapter, sweepIntervalMs = SWEEP_INTERVAL_MS } = {}) {
34
- if (!queueAdapter) throw new Error('WorkflowRunner requires a queueAdapter');
35
- this.queueAdapter = queueAdapter;
36
- this.sweepIntervalMs = sweepIntervalMs;
37
- this.sweepTimer = null;
38
- this.timers = new Map();
39
- this.workflows = new Map();
40
- this.stopped = false;
41
- }
42
-
43
- async register(workflow = {}) {
44
- if (!workflow.kind || typeof workflow.kind !== 'string') throw new Error('workflow requires a kind');
45
- if (workflow.kind.includes('__')) throw new Error(`workflow '${workflow.kind}' must not contain '__' (reserved as the jobId delimiter)`);
46
- if (typeof workflow.dedupeKey !== 'function') throw new Error(`workflow '${workflow.kind}' requires a dedupeKey function`);
47
- if (typeof workflow.prepare !== 'function') throw new Error(`workflow '${workflow.kind}' requires a prepare function`);
48
- if (this.workflows.has(workflow.kind)) throw new Error(`workflow '${workflow.kind}' is already registered`);
49
- if (workflow.retrySchedule !== undefined) {
50
- if (!Array.isArray(workflow.retrySchedule) || !workflow.retrySchedule.length) {
51
- throw new Error(`workflow '${workflow.kind}' requires a non-empty retrySchedule array`);
52
- }
53
- if (workflow.retrySchedule.some((minutes) => !Number.isFinite(minutes) || minutes <= 0)) {
54
- throw new Error(`workflow '${workflow.kind}' requires retrySchedule entries to be positive minutes`);
55
- }
56
- if (workflow.retrySchedule.some((minutes) => minutes * MINUTE_MS >= RETENTION_MS)) {
57
- throw new Error(`workflow '${workflow.kind}' requires retrySchedule entries under the ${RETENTION_MS / MINUTE_MS}m retention, or the record expires before the attempt fires`);
58
- }
59
- }
60
-
61
- this.workflows.set(workflow.kind, workflow);
62
- if (workflow.retrySchedule) {
63
- await this.sweep([workflow.kind]).catch((error) => {
64
- logger.error('[WorkflowRunner] Sweep on register failed', { kind: workflow.kind, error: error.message });
65
- });
66
- this._startSweeping();
67
- return workflow;
68
- }
69
-
70
- try {
71
- await this.queueAdapter.process(workflow.kind, (trigger) => workflow.prepare(trigger));
72
- } catch (error) {
73
- this.workflows.delete(workflow.kind);
74
- throw error;
75
- }
76
- return workflow;
77
- }
78
-
79
- enqueue(kind, trigger, options = {}) {
80
- const workflow = this.workflows.get(kind);
81
- if (!workflow) throw new Error(`no workflow registered for '${kind}'`);
82
- const dedupeKey = workflow.dedupeKey(trigger);
83
- if (dedupeKey == null || (typeof dedupeKey === 'string' && !dedupeKey.trim())) {
84
- throw new Error(`workflow '${kind}' produced an empty dedupe key`);
85
- }
86
- if (typeof dedupeKey !== 'string') throw new Error(`workflow '${kind}' produced a non-string dedupe key`);
87
- if (!workflow.retrySchedule) {
88
- return this.queueAdapter.enqueue(kind, trigger, { ...options, jobId: `${kind}__${dedupeKey}` });
89
- }
90
- return this._armDeferred(workflow, trigger, dedupeKey);
91
- }
92
-
93
- async sweep(only = null) {
94
- const deferredKinds = [...this.workflows.values()].filter((w) => w.retrySchedule).map((w) => w.kind);
95
- const kinds = only ? deferredKinds.filter((kind) => only.includes(kind)) : deferredKinds;
96
- if (!kinds.length) return [];
97
-
98
- const orphaned = await DeferredWork.find({
99
- kind: { $in: kinds },
100
- $or: [
101
- { status: 'pending' },
102
- { status: 'processing', claimedAt: { $lt: new Date(Date.now() - STALE_CLAIM_MS) } }
103
- ]
104
- });
105
-
106
- const rearmed = orphaned.map((work) => this._arm(
107
- this.workflows.get(work.kind),
108
- work._id,
109
- this._delayUntilDue(work),
110
- ));
111
- if (rearmed.length) logger.info('[WorkflowRunner] Swept', { kinds, rearmed: rearmed.length });
112
- return rearmed;
113
- }
114
-
115
- _startSweeping() {
116
- if (this.stopped || this.sweepTimer || !this.sweepIntervalMs) return;
117
- this.sweepTimer = setInterval(() => {
118
- this.sweep().catch((error) => logger.error('[WorkflowRunner] Periodic sweep failed', { error: error.message }));
119
- }, this.sweepIntervalMs);
120
- this.sweepTimer.unref();
121
- }
122
-
123
- stop() {
124
- this.stopped = true;
125
- if (this.sweepTimer) clearInterval(this.sweepTimer);
126
- this.sweepTimer = null;
127
- for (const timer of this.timers.values()) clearTimeout(timer);
128
- this.timers.clear();
129
- }
130
-
131
- async _armDeferred(workflow, trigger, dedupeKey) {
132
- const { kind } = workflow;
133
- await withIndexTimeout(ensureIndexes());
134
- const work = await DeferredWork.findOne({ kind, dedupeKey })
135
- || await DeferredWork.create({ kind, dedupeKey, trigger }).catch(async (error) => {
136
- if (error.code !== 11000) throw error;
137
- return await DeferredWork.findOne({ kind, dedupeKey });
138
- });
139
- if (work.status === 'abandoned') {
140
- const { modifiedCount } = await DeferredWork.updateOne(
141
- { _id: work._id, status: 'abandoned' },
142
- { $set: { status: 'pending', attempt: 0, nextAttemptAt: new Date(), lastError: null, claimedAt: null } }
143
- );
144
- if (!modifiedCount) return null;
145
- logger.info('[WorkflowRunner] Re-arming abandoned work', { kind, dedupeKey });
146
- return this._arm(workflow, work._id, 0);
147
- }
148
- if (work.status !== 'pending') {
149
- logger.info('[WorkflowRunner] Skipping, work is in flight or complete', { kind, dedupeKey, status: work.status });
150
- return null;
151
- }
152
- return this._arm(workflow, work._id, this._delayUntilDue(work));
153
- }
154
-
155
- _delayUntilDue(work) {
156
- return Math.max(0, new Date(work.nextAttemptAt).getTime() - Date.now());
157
- }
158
-
159
- _arm(workflow, workId, delayMs) {
160
- const id = String(workId);
161
- // An attempt still in flight when stop() lands must not re-arm after shutdown. The
162
- // record stays pending in Mongo, so the next process recovers it on its sweep.
163
- if (this.stopped) return id;
164
- clearTimeout(this.timers.get(id));
165
- const timer = setTimeout(() => {
166
- this.timers.delete(id);
167
- this._runDeferred(workflow, id).catch((error) => {
168
- logger.error('[WorkflowRunner] Deferred run failed', { kind: workflow.kind, deferredWorkId: id, error: error.message });
169
- });
170
- }, delayMs);
171
- timer.unref();
172
- this.timers.set(id, timer);
173
- return id;
174
- }
175
-
176
- async _runDeferred(workflow, deferredWorkId) {
177
- const claimedAt = new Date();
178
- const prior = await DeferredWork.findOneAndUpdate(
179
- {
180
- _id: deferredWorkId,
181
- $or: [
182
- { status: 'pending', nextAttemptAt: { $lte: claimedAt } },
183
- { status: 'processing', claimedAt: { $lt: new Date(claimedAt.getTime() - STALE_CLAIM_MS) } }
184
- ]
185
- },
186
- { $set: { status: 'processing', claimedAt } },
187
- { new: false }
188
- );
189
- if (!prior) {
190
- logger.info('[WorkflowRunner] Claimed elsewhere or not due', { kind: workflow.kind, deferredWorkId });
191
- return { claimed: false };
192
- }
193
-
194
- const reclaimed = prior.status === 'processing';
195
- const work = { ...prior.toObject(), attempt: prior.attempt + (reclaimed ? 1 : 0) };
196
- if (reclaimed) await this._write(work, claimedAt, { attempt: work.attempt });
197
-
198
- const checkpoints = { ...(work.checkpoints || {}) };
199
- const save = async (patch) => {
200
- Object.assign(checkpoints, patch);
201
- const $set = {};
202
- for (const [key, value] of Object.entries(patch || {})) $set[`checkpoints.${key}`] = value;
203
- if (Object.keys($set).length) await this._write(work, claimedAt, $set);
204
- };
205
-
206
- try {
207
- await workflow.prepare(work.trigger, checkpoints, save);
208
- } catch (error) {
209
- return await this._rescheduleOrAbandon(workflow, work, claimedAt, error);
210
- }
211
- await this._write(work, claimedAt, { status: 'done', claimedAt: null });
212
- return { status: 'done' };
213
- }
214
-
215
- // matchedCount, not modifiedCount: the fence is "do we still hold the claim", and
216
- // modifiedCount is also 0 for a write that matched but set an identical value - a
217
- // save() rewriting an unchanged checkpoint is not a lost claim.
218
- async _write(work, claimedAt, $set) {
219
- const { matchedCount } = await DeferredWork.updateOne({ _id: work._id, claimedAt }, { $set });
220
- if (!matchedCount) logger.warn('[WorkflowRunner] Claim lost, write discarded', { deferredWorkId: String(work._id) });
221
- return matchedCount > 0;
222
- }
223
-
224
- async _rescheduleOrAbandon(workflow, work, claimedAt, error) {
225
- // prepare() is consumer code and can throw anything, including null. Dereferencing
226
- // it here used to raise a TypeError out of this handler, so neither branch ran and
227
- // the record stayed `processing` until stale recovery hit the same failure again.
228
- const lastError = describeThrown(error);
229
- const delayMinutes = workflow.retrySchedule[work.attempt];
230
- if (error?.permanent === true || delayMinutes === undefined) {
231
- await this._write(work, claimedAt, { status: 'abandoned', lastError, claimedAt: null });
232
- logger.error('[WorkflowRunner] Abandoned', {
233
- kind: workflow.kind,
234
- dedupeKey: work.dedupeKey,
235
- attempt: work.attempt,
236
- permanent: error?.permanent === true,
237
- error: lastError
238
- });
239
- return { status: 'abandoned' };
240
- }
241
-
242
- const attempt = work.attempt + 1;
243
- const delayMs = delayMinutes * MINUTE_MS;
244
- const kept = await this._write(work, claimedAt, {
245
- status: 'pending',
246
- attempt,
247
- nextAttemptAt: new Date(Date.now() + delayMs),
248
- lastError,
249
- claimedAt: null
250
- });
251
- if (!kept) return { claimed: false };
252
-
253
- logger.warn('[WorkflowRunner] Rescheduled', {
254
- kind: workflow.kind,
255
- dedupeKey: work.dedupeKey,
256
- attempt,
257
- delayMinutes,
258
- error: lastError
259
- });
260
- this._arm(workflow, work._id, delayMs);
261
- return { status: 'pending', attempt };
262
- }
263
- }
264
-
265
- module.exports = { WorkflowRunner };
@@ -1,24 +0,0 @@
1
- const mongoose = require('mongoose');
2
-
3
- const STATUS_VALUES = ['pending', 'processing', 'done', 'abandoned'];
4
- const RETENTION_MS = 24 * 60 * 60 * 1000;
5
-
6
- const deferredWorkSchema = new mongoose.Schema({
7
- kind: { type: String, required: true },
8
- dedupeKey: { type: String, required: true },
9
- trigger: { type: mongoose.Schema.Types.Mixed, default: null },
10
- checkpoints: { type: mongoose.Schema.Types.Mixed, default: () => ({}) },
11
- status: { type: String, enum: STATUS_VALUES, default: 'pending' },
12
- attempt: { type: Number, default: 0 },
13
- nextAttemptAt: { type: Date, default: Date.now },
14
- claimedAt: { type: Date, default: null },
15
- lastError: { type: String, default: null }
16
- }, { timestamps: true });
17
-
18
- deferredWorkSchema.index({ kind: 1, dedupeKey: 1 }, { unique: true, name: 'dedupe_idx' });
19
- deferredWorkSchema.index({ status: 1, nextAttemptAt: 1 }, { name: 'due_idx' });
20
- deferredWorkSchema.index({ updatedAt: 1 }, { expireAfterSeconds: RETENTION_MS / 1000, name: 'ttl_idx' });
21
-
22
- const DeferredWork = mongoose.model('DeferredWork', deferredWorkSchema);
23
-
24
- module.exports = { DeferredWork, RETENTION_MS };