@peopl-health/nexus 5.45.0-dev.7142 → 5.45.0-dev.7144
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -32
- package/lib/controllers/patientController.js +16 -0
- package/lib/core/NexusMessaging.js +0 -5
- package/lib/core/workflowRunner.js +35 -0
- package/lib/index.d.ts +4 -6
- package/lib/index.js +2 -0
- package/lib/routes/index.js +2 -0
- package/lib/services/airtableService.js +0 -7
- package/lib/services/escalationContextService.js +81 -0
- package/lib/utils/sanitizerUtils.js +1 -0
- package/package.json +1 -1
- package/lib/core/WorkflowRunner.js +0 -265
- package/lib/models/deferredWorkModel.js +0 -24
package/README.md
CHANGED
|
@@ -116,38 +116,6 @@ const bus = nexus.getMessaging().getEventBus();
|
|
|
116
116
|
bus.on('message:received', (m) => console.log('rx', m.id));
|
|
117
117
|
```
|
|
118
118
|
|
|
119
|
-
## Workflows (Optional)
|
|
120
|
-
|
|
121
|
-
Register a named unit of work, deduplicated by a key you derive from the trigger.
|
|
122
|
-
|
|
123
|
-
```js
|
|
124
|
-
const runner = nexus.getMessaging().getWorkflowRunner();
|
|
125
|
-
|
|
126
|
-
await runner.register({
|
|
127
|
-
kind: 'triage-projection',
|
|
128
|
-
dedupeKey: (trigger) => String(trigger.submissionId),
|
|
129
|
-
prepare: async (trigger, checkpoints, save) => { /* ... */ },
|
|
130
|
-
retrySchedule: [5, 10, 20, 40], // minutes; omit for fire-and-forget
|
|
131
|
-
});
|
|
132
|
-
|
|
133
|
-
await runner.enqueue('triage-projection', { submissionId });
|
|
134
|
-
```
|
|
135
|
-
|
|
136
|
-
With `retrySchedule` the trigger and a checkpoint journal are stored in Mongo and the schedule runs
|
|
137
|
-
on an in-process timer — deferred workflows never touch the queue adapter, so they add no Redis load.
|
|
138
|
-
`prepare` resumes from `checkpoints`; call `save(patch)` before each external call so a retry does
|
|
139
|
-
not repeat completed work. Throw an error carrying `permanent: true` to give up immediately instead
|
|
140
|
-
of walking the schedule; the Airtable helpers tag their own permanent 4xx errors that way, in
|
|
141
|
-
`withAirtableRetry`, which all five of them route through. They rethrow on failure too, so an
|
|
142
|
-
Airtable outage reaches `prepare` as an exception and walks the retry schedule on its own rather
|
|
143
|
-
than reading as success.
|
|
144
|
-
|
|
145
|
-
Recovery is automatic: registering sweeps for work left behind by a previous process, and the runner
|
|
146
|
-
keeps sweeping so work whose worker died mid-`prepare` is reclaimed once its claim goes stale.
|
|
147
|
-
`enqueue` on an abandoned key revives it, keeping its checkpoints. Work is retained for 24 hours
|
|
148
|
-
after its last update and then expires — the dedupe key expires with it, so idempotency has the
|
|
149
|
-
same horizon as retention.
|
|
150
|
-
|
|
151
119
|
## Assistants (Optional)
|
|
152
120
|
|
|
153
121
|
Register assistant classes and (optionally) a custom resolver. OpenAI is supported via `llm: 'openai'`.
|
|
@@ -6,6 +6,7 @@ const { isAllowedUrl, redactDirectIdentifiers } = require('../utils/sanitizerUti
|
|
|
6
6
|
const { getPatientInformation, updatePatientInformation, logPatientChange } = require('../services/patientService');
|
|
7
7
|
const { getTriageHistory } = require('../services/triageHistoryService');
|
|
8
8
|
const { getCarePlanHistory } = require('../services/carePlanHistoryService');
|
|
9
|
+
const { getEscalationContext } = require('../services/escalationContextService');
|
|
9
10
|
const { getPatientTasks, updatePatientTask } = require('../services/patientTaskService');
|
|
10
11
|
const {
|
|
11
12
|
getRecommendationCarousels,
|
|
@@ -241,10 +242,25 @@ const logPatientChangeController = async (req, res) => {
|
|
|
241
242
|
}
|
|
242
243
|
};
|
|
243
244
|
|
|
245
|
+
const getPatientEscalationContextController = async (req, res) => {
|
|
246
|
+
try {
|
|
247
|
+
const context = await getEscalationContext({ code: req.params.code });
|
|
248
|
+
res.status(200).json({ success: true, context });
|
|
249
|
+
} catch (error) {
|
|
250
|
+
logger.error('Error fetching patient escalation context:', {
|
|
251
|
+
error: redactDirectIdentifiers(String(error.message || '')),
|
|
252
|
+
stack: redactDirectIdentifiers(String(error.stack || '')),
|
|
253
|
+
code: redactDirectIdentifiers(String(req.params?.code || '')),
|
|
254
|
+
});
|
|
255
|
+
res.status(error.statusCode || 500).json({ success: false, error: error.message });
|
|
256
|
+
}
|
|
257
|
+
};
|
|
258
|
+
|
|
244
259
|
module.exports = {
|
|
245
260
|
getPatientInfoController,
|
|
246
261
|
getPatientTriageInfoController,
|
|
247
262
|
getPatientCarePlanController,
|
|
263
|
+
getPatientEscalationContextController,
|
|
248
264
|
getPatientTasksController,
|
|
249
265
|
getPatientRecommendationCarouselsController,
|
|
250
266
|
getPatientRecommendationCarouselImageController,
|
|
@@ -43,7 +43,6 @@ const { isLlmMonitorEnabled } = require('../clinical/flags/llmMonitorConfig');
|
|
|
43
43
|
const { createQueueAdapter } = require('../queue');
|
|
44
44
|
const { ScheduledMessageJob } = require('../jobs/ScheduledMessageJob');
|
|
45
45
|
const { TemplateApprovalJob } = require('../jobs/TemplateApprovalJob');
|
|
46
|
-
const { WorkflowRunner } = require('./WorkflowRunner');
|
|
47
46
|
|
|
48
47
|
const { PhiProcessor } = require('./PhiProcessor');
|
|
49
48
|
|
|
@@ -128,8 +127,6 @@ class NexusMessaging {
|
|
|
128
127
|
queueAdapter: this.queueAdapter,
|
|
129
128
|
});
|
|
130
129
|
|
|
131
|
-
this.workflowRunner = new WorkflowRunner({ queueAdapter: this.queueAdapter });
|
|
132
|
-
|
|
133
130
|
this.phiProcessor = new PhiProcessor({
|
|
134
131
|
encode: config.phi?.encode || false,
|
|
135
132
|
ner: config.phi?.ner || null,
|
|
@@ -313,7 +310,6 @@ class NexusMessaging {
|
|
|
313
310
|
getAssistantProcessor() { return this.assistantProcessor; }
|
|
314
311
|
getLlmMonitor() { return this.llmMonitor; }
|
|
315
312
|
getQueueAdapter() { return this.queueAdapter; }
|
|
316
|
-
getWorkflowRunner() { return this.workflowRunner; }
|
|
317
313
|
getPhiProcessor() { return this.phiProcessor; }
|
|
318
314
|
isConnected() { return this.provider?.getConnectionStatus() ?? false; }
|
|
319
315
|
isProcessing(chatId) { return this.batchingManager.isProcessing(chatId); }
|
|
@@ -1016,7 +1012,6 @@ class NexusMessaging {
|
|
|
1016
1012
|
async disconnect() {
|
|
1017
1013
|
this.queueReconciliation?.stop();
|
|
1018
1014
|
if (this.provider) await this.provider.disconnect();
|
|
1019
|
-
if (this.workflowRunner) this.workflowRunner.stop();
|
|
1020
1015
|
if (this.queueAdapter) await this.queueAdapter.shutdown();
|
|
1021
1016
|
this.events.removeAllListeners();
|
|
1022
1017
|
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
function createWorkflowRunner({ queueAdapter } = {}) {
|
|
2
|
+
if (!queueAdapter) throw new Error('createWorkflowRunner requires a queueAdapter');
|
|
3
|
+
const workflows = new Map();
|
|
4
|
+
|
|
5
|
+
async function register(workflow = {}) {
|
|
6
|
+
if (!workflow.kind || typeof workflow.kind !== 'string') throw new Error('workflow requires a kind');
|
|
7
|
+
if (workflow.kind.includes('__')) throw new Error(`workflow '${workflow.kind}' must not contain '__' (reserved as the jobId delimiter)`);
|
|
8
|
+
if (typeof workflow.dedupeKey !== 'function') throw new Error(`workflow '${workflow.kind}' requires a dedupeKey function`);
|
|
9
|
+
if (typeof workflow.prepare !== 'function') throw new Error(`workflow '${workflow.kind}' requires a prepare function`);
|
|
10
|
+
if (workflows.has(workflow.kind)) throw new Error(`workflow '${workflow.kind}' is already registered`);
|
|
11
|
+
workflows.set(workflow.kind, workflow);
|
|
12
|
+
try {
|
|
13
|
+
await queueAdapter.process(workflow.kind, (trigger) => workflow.prepare(trigger));
|
|
14
|
+
} catch (error) {
|
|
15
|
+
workflows.delete(workflow.kind);
|
|
16
|
+
throw error;
|
|
17
|
+
}
|
|
18
|
+
return workflow;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function enqueue(kind, trigger, options = {}) {
|
|
22
|
+
const workflow = workflows.get(kind);
|
|
23
|
+
if (!workflow) throw new Error(`no workflow registered for '${kind}'`);
|
|
24
|
+
const dedupeKey = workflow.dedupeKey(trigger);
|
|
25
|
+
if (dedupeKey == null || (typeof dedupeKey === 'string' && !dedupeKey.trim())) {
|
|
26
|
+
throw new Error(`workflow '${kind}' produced an empty dedupe key`);
|
|
27
|
+
}
|
|
28
|
+
if (typeof dedupeKey !== 'string') throw new Error(`workflow '${kind}' produced a non-string dedupe key`);
|
|
29
|
+
return queueAdapter.enqueue(kind, trigger, { ...options, jobId: `${kind}__${dedupeKey}` });
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
return { register, enqueue };
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
module.exports = { createWorkflowRunner };
|
package/lib/index.d.ts
CHANGED
|
@@ -243,7 +243,6 @@ declare module '@peopl-health/nexus' {
|
|
|
243
243
|
getAssistantProcessor(): AssistantProcessor;
|
|
244
244
|
getLlmMonitor(): { start(options?: { scheduleDaily?: boolean }): Promise<{ enabled: boolean }> } | null;
|
|
245
245
|
initializeLlmMonitor(): Promise<{ enabled: boolean; error?: string } | undefined>;
|
|
246
|
-
getWorkflowRunner(): WorkflowRunner;
|
|
247
246
|
processInstruction(code: string, instruction: string, role?: string, options?: { triggeredBy?: string }): Promise<void>;
|
|
248
247
|
processSystemMessage(code: string, messages: string | string[], role?: string, options?: { triggeredBy?: string; reply?: boolean }): Promise<void>;
|
|
249
248
|
processOutreach(code: string, options?: { brief?: OutreachBrief | null; triggeredBy?: string | null; reason?: string | null; firstName?: string | null; dryRun?: boolean }): Promise<OutreachDecision>;
|
|
@@ -521,17 +520,16 @@ declare module '@peopl-health/nexus' {
|
|
|
521
520
|
export interface Workflow {
|
|
522
521
|
kind: string;
|
|
523
522
|
dedupeKey: (trigger: any) => string;
|
|
524
|
-
prepare: (trigger: any
|
|
525
|
-
retrySchedule?: number[];
|
|
523
|
+
prepare: (trigger: any) => Promise<any>;
|
|
526
524
|
}
|
|
527
525
|
|
|
528
526
|
export interface WorkflowRunner {
|
|
529
527
|
register(workflow: Workflow): Promise<Workflow>;
|
|
530
|
-
enqueue(kind: string, trigger: any, options?: QueueJobOptions): Promise<string
|
|
531
|
-
sweep(only?: string[]): Promise<string[]>;
|
|
532
|
-
stop(): void;
|
|
528
|
+
enqueue(kind: string, trigger: any, options?: QueueJobOptions): Promise<string>;
|
|
533
529
|
}
|
|
534
530
|
|
|
531
|
+
export function createWorkflowRunner(options: { queueAdapter: QueueAdapter }): WorkflowRunner;
|
|
532
|
+
|
|
535
533
|
// Memory System
|
|
536
534
|
export interface PatientMemoryDocument {
|
|
537
535
|
_id: any;
|
package/lib/index.js
CHANGED
|
@@ -27,6 +27,7 @@ const { BaileysProvider } = require('./adapters/BaileysProvider');
|
|
|
27
27
|
const { setPreprocessingHandler, hasPreprocessingHandler, invokePreprocessingHandler } = require('./services/preprocessingService');
|
|
28
28
|
const { requestIdMiddleware, getRequestId } = require('./middleware/requestId');
|
|
29
29
|
const { QueueAdapter, LocalQueueAdapter, RedisQueueAdapter, createQueueAdapter, registerQueueAdapter } = require('./queue');
|
|
30
|
+
const { createWorkflowRunner } = require('./core/workflowRunner');
|
|
30
31
|
const routes = require('./routes');
|
|
31
32
|
const { resetAll } = require('./config/lifecycle');
|
|
32
33
|
const { EvalProvider } = require('./eval/EvalProvider');
|
|
@@ -228,6 +229,7 @@ class Nexus {
|
|
|
228
229
|
}
|
|
229
230
|
|
|
230
231
|
module.exports = {
|
|
232
|
+
createWorkflowRunner,
|
|
231
233
|
Nexus,
|
|
232
234
|
TwilioProvider,
|
|
233
235
|
BaileysProvider,
|
package/lib/routes/index.js
CHANGED
|
@@ -70,6 +70,7 @@ const patientRouteDefinitions = {
|
|
|
70
70
|
'GET /:code': 'getPatientInfoController',
|
|
71
71
|
'GET /triage/:code': 'getPatientTriageInfoController',
|
|
72
72
|
'GET /careplan/:code': 'getPatientCarePlanController',
|
|
73
|
+
'GET /escalation-context/:code': 'getPatientEscalationContextController',
|
|
73
74
|
'GET /tasks/:code': 'getPatientTasksController',
|
|
74
75
|
'PATCH /:code': 'updatePatientController',
|
|
75
76
|
'PATCH /tasks/:code/:kind/:recordId': 'updatePatientTaskController',
|
|
@@ -214,6 +215,7 @@ const builtInControllers = {
|
|
|
214
215
|
getPatientInfoController: patientController.getPatientInfoController,
|
|
215
216
|
getPatientTriageInfoController: patientController.getPatientTriageInfoController,
|
|
216
217
|
getPatientCarePlanController: patientController.getPatientCarePlanController,
|
|
218
|
+
getPatientEscalationContextController: patientController.getPatientEscalationContextController,
|
|
217
219
|
getPatientTasksController: patientController.getPatientTasksController,
|
|
218
220
|
getPatientRecommendationCarouselsController: patientController.getPatientRecommendationCarouselsController,
|
|
219
221
|
getPatientRecommendationCarouselImageController: patientController.getPatientRecommendationCarouselImageController,
|
|
@@ -15,11 +15,6 @@ const AIRTABLE_API_URL = 'https://api.airtable.com/v0';
|
|
|
15
15
|
const AXIOS_TIMEOUT_CODE = 'ECONNABORTED';
|
|
16
16
|
const RETRY_BASE_DELAY_MS = 1000;
|
|
17
17
|
|
|
18
|
-
function isPermanentAirtableError(error) {
|
|
19
|
-
const status = error?.statusCode;
|
|
20
|
-
return typeof status === 'number' && status >= 400 && status < 500 && status !== 429;
|
|
21
|
-
}
|
|
22
|
-
|
|
23
18
|
let isEvalMode = false;
|
|
24
19
|
|
|
25
20
|
function setEvalMode(enabled) {
|
|
@@ -47,7 +42,6 @@ async function withAirtableRetry(operation, label, { retryNetworkErrors = true }
|
|
|
47
42
|
const isTransient = TRANSIENT_STATUS_CODES.includes(error.statusCode)
|
|
48
43
|
|| (retryNetworkErrors && isTransientNetworkError(error));
|
|
49
44
|
if (!isTransient || attempt === MAX_ATTEMPTS) {
|
|
50
|
-
if (isPermanentAirtableError(error)) error.permanent = true;
|
|
51
45
|
throw error;
|
|
52
46
|
}
|
|
53
47
|
const baseDelay = error.statusCode === RATE_LIMITED_STATUS ? RATE_LIMIT_DELAY_MS : RETRY_BASE_DELAY_MS;
|
|
@@ -235,7 +229,6 @@ async function upsertRecord(baseID, tableName, fields, { mergeOn } = {}, context
|
|
|
235
229
|
|
|
236
230
|
module.exports = {
|
|
237
231
|
setEvalMode,
|
|
238
|
-
isPermanentAirtableError,
|
|
239
232
|
addRecord,
|
|
240
233
|
getRecords,
|
|
241
234
|
getRecordByFilter,
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
const { isoDay } = require('../utils/dateUtils');
|
|
2
|
+
const { logger } = require('../utils/logger');
|
|
3
|
+
const { PATIENT_CODE, redactDirectIdentifiers } = require('../utils/sanitizerUtils');
|
|
4
|
+
const { withTimeout } = require('../utils/timeoutUtils');
|
|
5
|
+
const { ensureWhatsAppFormat } = require('../helpers/twilioHelper');
|
|
6
|
+
const { assembleHistory } = require('../clinical/services/patientHistoryService');
|
|
7
|
+
const { DefaultMemoryManager } = require('../clinical/memory/DefaultMemoryManager');
|
|
8
|
+
|
|
9
|
+
const { getCarePlanHistory } = require('./carePlanHistoryService');
|
|
10
|
+
const { getPatientInformation } = require('./patientService');
|
|
11
|
+
const { getTriageHistory } = require('./triageHistoryService');
|
|
12
|
+
|
|
13
|
+
const SECTION_TIMEOUT_MS = 10000;
|
|
14
|
+
const TRIAGE_WINDOW_DAYS = 30;
|
|
15
|
+
const CARE_PLAN_LIMIT = 5;
|
|
16
|
+
const HISTORY_SCOPE = ['closed_cases_90d', 'recurrence_patterns', 'recent_medications', 'patient_reported_procedures'];
|
|
17
|
+
|
|
18
|
+
const httpError = (statusCode, message) => Object.assign(new Error(message), { statusCode });
|
|
19
|
+
|
|
20
|
+
function sectionReaders(code, now) {
|
|
21
|
+
const memoryManager = new DefaultMemoryManager();
|
|
22
|
+
const triageStart = isoDay(new Date(now - TRIAGE_WINDOW_DAYS * 86400000).toISOString());
|
|
23
|
+
return {
|
|
24
|
+
agentContext: () => memoryManager.getClinicalData(code),
|
|
25
|
+
conversation: () => memoryManager.buildContext({ thread: { code } }),
|
|
26
|
+
triages: () => getTriageHistory({ code, startDate: triageStart }),
|
|
27
|
+
carePlan: () => getCarePlanHistory({ code, limit: CARE_PLAN_LIMIT }),
|
|
28
|
+
history: () => assembleHistory(code, { scope: HISTORY_SCOPE }),
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
async function getEscalationContext({ code, now = Date.now(), timeoutMs = SECTION_TIMEOUT_MS } = {}) {
|
|
33
|
+
const patientCode = ensureWhatsAppFormat(String(code ?? '').trim());
|
|
34
|
+
if (!PATIENT_CODE.test(patientCode)) throw httpError(400, 'A valid WhatsApp ID is required');
|
|
35
|
+
|
|
36
|
+
const patient = await withTimeout(getPatientInformation(patientCode), timeoutMs, 'patient lookup');
|
|
37
|
+
if (!patient) throw httpError(404, 'Patient not found');
|
|
38
|
+
|
|
39
|
+
const readers = sectionReaders(patientCode, now);
|
|
40
|
+
const names = Object.keys(readers);
|
|
41
|
+
const outcomes = await Promise.allSettled(names.map((name) => withTimeout(readers[name](), timeoutMs, name)));
|
|
42
|
+
|
|
43
|
+
const context = { patientCode, asOf: new Date(now).toISOString(), degradedSections: [] };
|
|
44
|
+
outcomes.forEach((outcome, index) => {
|
|
45
|
+
const name = names[index];
|
|
46
|
+
if (outcome.status === 'fulfilled' && outcome.value != null) {
|
|
47
|
+
context[name] = outcome.value;
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
context[name] = null;
|
|
51
|
+
context.degradedSections.push(name);
|
|
52
|
+
if (outcome.status === 'rejected') {
|
|
53
|
+
logger.error('[escalationContextService] Section reader failed', {
|
|
54
|
+
code: patientCode,
|
|
55
|
+
section: name,
|
|
56
|
+
error: redactDirectIdentifiers(String(outcome.reason?.message ?? outcome.reason)),
|
|
57
|
+
stack: redactDirectIdentifiers(String(outcome.reason?.stack ?? '')),
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
if (context.conversation) {
|
|
63
|
+
context.conversation = context.conversation
|
|
64
|
+
.filter((item) => item.type === 'message')
|
|
65
|
+
.map(({ role, content }) => ({ role, content }));
|
|
66
|
+
}
|
|
67
|
+
if (context.triages) {
|
|
68
|
+
if (context.triages.pagination?.hasNext) context.degradedSections.push('triages');
|
|
69
|
+
context.triages = context.triages.records;
|
|
70
|
+
}
|
|
71
|
+
if (context.carePlan) {
|
|
72
|
+
const { records, pagination = null, currentState = null } = context.carePlan;
|
|
73
|
+
const { degradedSections: carePlanDegraded = [], ...state } = currentState || {};
|
|
74
|
+
context.carePlan = { records, pagination, currentState: currentState ? state : null };
|
|
75
|
+
for (const section of carePlanDegraded) context.degradedSections.push(`carePlan.${section}`);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
return context;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
module.exports = { getEscalationContext };
|
package/package.json
CHANGED
|
@@ -1,265 +0,0 @@
|
|
|
1
|
-
const { describeThrown } = require('../utils/errorUtils');
|
|
2
|
-
const { logger } = require('../utils/logger');
|
|
3
|
-
|
|
4
|
-
const { DeferredWork, RETENTION_MS } = require('../models/deferredWorkModel');
|
|
5
|
-
|
|
6
|
-
const STALE_CLAIM_MS = 5 * 60 * 1000;
|
|
7
|
-
const MINUTE_MS = 60 * 1000;
|
|
8
|
-
const INDEX_TIMEOUT_MS = 1000;
|
|
9
|
-
const SWEEP_INTERVAL_MS = 60 * 1000;
|
|
10
|
-
|
|
11
|
-
let indexesReady = null;
|
|
12
|
-
|
|
13
|
-
function ensureIndexes() {
|
|
14
|
-
if (!indexesReady) {
|
|
15
|
-
indexesReady = DeferredWork.init().catch((error) => {
|
|
16
|
-
indexesReady = null;
|
|
17
|
-
logger.error('[WorkflowRunner] Index build failed, dedupe and retention are not guaranteed', { error: error.message });
|
|
18
|
-
});
|
|
19
|
-
}
|
|
20
|
-
return indexesReady;
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
function withIndexTimeout(promise) {
|
|
24
|
-
let timer;
|
|
25
|
-
const timeout = new Promise((resolve) => {
|
|
26
|
-
timer = setTimeout(resolve, INDEX_TIMEOUT_MS);
|
|
27
|
-
timer.unref();
|
|
28
|
-
});
|
|
29
|
-
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
class WorkflowRunner {
|
|
33
|
-
constructor({ queueAdapter, sweepIntervalMs = SWEEP_INTERVAL_MS } = {}) {
|
|
34
|
-
if (!queueAdapter) throw new Error('WorkflowRunner requires a queueAdapter');
|
|
35
|
-
this.queueAdapter = queueAdapter;
|
|
36
|
-
this.sweepIntervalMs = sweepIntervalMs;
|
|
37
|
-
this.sweepTimer = null;
|
|
38
|
-
this.timers = new Map();
|
|
39
|
-
this.workflows = new Map();
|
|
40
|
-
this.stopped = false;
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
async register(workflow = {}) {
|
|
44
|
-
if (!workflow.kind || typeof workflow.kind !== 'string') throw new Error('workflow requires a kind');
|
|
45
|
-
if (workflow.kind.includes('__')) throw new Error(`workflow '${workflow.kind}' must not contain '__' (reserved as the jobId delimiter)`);
|
|
46
|
-
if (typeof workflow.dedupeKey !== 'function') throw new Error(`workflow '${workflow.kind}' requires a dedupeKey function`);
|
|
47
|
-
if (typeof workflow.prepare !== 'function') throw new Error(`workflow '${workflow.kind}' requires a prepare function`);
|
|
48
|
-
if (this.workflows.has(workflow.kind)) throw new Error(`workflow '${workflow.kind}' is already registered`);
|
|
49
|
-
if (workflow.retrySchedule !== undefined) {
|
|
50
|
-
if (!Array.isArray(workflow.retrySchedule) || !workflow.retrySchedule.length) {
|
|
51
|
-
throw new Error(`workflow '${workflow.kind}' requires a non-empty retrySchedule array`);
|
|
52
|
-
}
|
|
53
|
-
if (workflow.retrySchedule.some((minutes) => !Number.isFinite(minutes) || minutes <= 0)) {
|
|
54
|
-
throw new Error(`workflow '${workflow.kind}' requires retrySchedule entries to be positive minutes`);
|
|
55
|
-
}
|
|
56
|
-
if (workflow.retrySchedule.some((minutes) => minutes * MINUTE_MS >= RETENTION_MS)) {
|
|
57
|
-
throw new Error(`workflow '${workflow.kind}' requires retrySchedule entries under the ${RETENTION_MS / MINUTE_MS}m retention, or the record expires before the attempt fires`);
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
this.workflows.set(workflow.kind, workflow);
|
|
62
|
-
if (workflow.retrySchedule) {
|
|
63
|
-
await this.sweep([workflow.kind]).catch((error) => {
|
|
64
|
-
logger.error('[WorkflowRunner] Sweep on register failed', { kind: workflow.kind, error: error.message });
|
|
65
|
-
});
|
|
66
|
-
this._startSweeping();
|
|
67
|
-
return workflow;
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
try {
|
|
71
|
-
await this.queueAdapter.process(workflow.kind, (trigger) => workflow.prepare(trigger));
|
|
72
|
-
} catch (error) {
|
|
73
|
-
this.workflows.delete(workflow.kind);
|
|
74
|
-
throw error;
|
|
75
|
-
}
|
|
76
|
-
return workflow;
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
enqueue(kind, trigger, options = {}) {
|
|
80
|
-
const workflow = this.workflows.get(kind);
|
|
81
|
-
if (!workflow) throw new Error(`no workflow registered for '${kind}'`);
|
|
82
|
-
const dedupeKey = workflow.dedupeKey(trigger);
|
|
83
|
-
if (dedupeKey == null || (typeof dedupeKey === 'string' && !dedupeKey.trim())) {
|
|
84
|
-
throw new Error(`workflow '${kind}' produced an empty dedupe key`);
|
|
85
|
-
}
|
|
86
|
-
if (typeof dedupeKey !== 'string') throw new Error(`workflow '${kind}' produced a non-string dedupe key`);
|
|
87
|
-
if (!workflow.retrySchedule) {
|
|
88
|
-
return this.queueAdapter.enqueue(kind, trigger, { ...options, jobId: `${kind}__${dedupeKey}` });
|
|
89
|
-
}
|
|
90
|
-
return this._armDeferred(workflow, trigger, dedupeKey);
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
async sweep(only = null) {
|
|
94
|
-
const deferredKinds = [...this.workflows.values()].filter((w) => w.retrySchedule).map((w) => w.kind);
|
|
95
|
-
const kinds = only ? deferredKinds.filter((kind) => only.includes(kind)) : deferredKinds;
|
|
96
|
-
if (!kinds.length) return [];
|
|
97
|
-
|
|
98
|
-
const orphaned = await DeferredWork.find({
|
|
99
|
-
kind: { $in: kinds },
|
|
100
|
-
$or: [
|
|
101
|
-
{ status: 'pending' },
|
|
102
|
-
{ status: 'processing', claimedAt: { $lt: new Date(Date.now() - STALE_CLAIM_MS) } }
|
|
103
|
-
]
|
|
104
|
-
});
|
|
105
|
-
|
|
106
|
-
const rearmed = orphaned.map((work) => this._arm(
|
|
107
|
-
this.workflows.get(work.kind),
|
|
108
|
-
work._id,
|
|
109
|
-
this._delayUntilDue(work),
|
|
110
|
-
));
|
|
111
|
-
if (rearmed.length) logger.info('[WorkflowRunner] Swept', { kinds, rearmed: rearmed.length });
|
|
112
|
-
return rearmed;
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
_startSweeping() {
|
|
116
|
-
if (this.stopped || this.sweepTimer || !this.sweepIntervalMs) return;
|
|
117
|
-
this.sweepTimer = setInterval(() => {
|
|
118
|
-
this.sweep().catch((error) => logger.error('[WorkflowRunner] Periodic sweep failed', { error: error.message }));
|
|
119
|
-
}, this.sweepIntervalMs);
|
|
120
|
-
this.sweepTimer.unref();
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
stop() {
|
|
124
|
-
this.stopped = true;
|
|
125
|
-
if (this.sweepTimer) clearInterval(this.sweepTimer);
|
|
126
|
-
this.sweepTimer = null;
|
|
127
|
-
for (const timer of this.timers.values()) clearTimeout(timer);
|
|
128
|
-
this.timers.clear();
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
async _armDeferred(workflow, trigger, dedupeKey) {
|
|
132
|
-
const { kind } = workflow;
|
|
133
|
-
await withIndexTimeout(ensureIndexes());
|
|
134
|
-
const work = await DeferredWork.findOne({ kind, dedupeKey })
|
|
135
|
-
|| await DeferredWork.create({ kind, dedupeKey, trigger }).catch(async (error) => {
|
|
136
|
-
if (error.code !== 11000) throw error;
|
|
137
|
-
return await DeferredWork.findOne({ kind, dedupeKey });
|
|
138
|
-
});
|
|
139
|
-
if (work.status === 'abandoned') {
|
|
140
|
-
const { modifiedCount } = await DeferredWork.updateOne(
|
|
141
|
-
{ _id: work._id, status: 'abandoned' },
|
|
142
|
-
{ $set: { status: 'pending', attempt: 0, nextAttemptAt: new Date(), lastError: null, claimedAt: null } }
|
|
143
|
-
);
|
|
144
|
-
if (!modifiedCount) return null;
|
|
145
|
-
logger.info('[WorkflowRunner] Re-arming abandoned work', { kind, dedupeKey });
|
|
146
|
-
return this._arm(workflow, work._id, 0);
|
|
147
|
-
}
|
|
148
|
-
if (work.status !== 'pending') {
|
|
149
|
-
logger.info('[WorkflowRunner] Skipping, work is in flight or complete', { kind, dedupeKey, status: work.status });
|
|
150
|
-
return null;
|
|
151
|
-
}
|
|
152
|
-
return this._arm(workflow, work._id, this._delayUntilDue(work));
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
_delayUntilDue(work) {
|
|
156
|
-
return Math.max(0, new Date(work.nextAttemptAt).getTime() - Date.now());
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
_arm(workflow, workId, delayMs) {
|
|
160
|
-
const id = String(workId);
|
|
161
|
-
// An attempt still in flight when stop() lands must not re-arm after shutdown. The
|
|
162
|
-
// record stays pending in Mongo, so the next process recovers it on its sweep.
|
|
163
|
-
if (this.stopped) return id;
|
|
164
|
-
clearTimeout(this.timers.get(id));
|
|
165
|
-
const timer = setTimeout(() => {
|
|
166
|
-
this.timers.delete(id);
|
|
167
|
-
this._runDeferred(workflow, id).catch((error) => {
|
|
168
|
-
logger.error('[WorkflowRunner] Deferred run failed', { kind: workflow.kind, deferredWorkId: id, error: error.message });
|
|
169
|
-
});
|
|
170
|
-
}, delayMs);
|
|
171
|
-
timer.unref();
|
|
172
|
-
this.timers.set(id, timer);
|
|
173
|
-
return id;
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
async _runDeferred(workflow, deferredWorkId) {
|
|
177
|
-
const claimedAt = new Date();
|
|
178
|
-
const prior = await DeferredWork.findOneAndUpdate(
|
|
179
|
-
{
|
|
180
|
-
_id: deferredWorkId,
|
|
181
|
-
$or: [
|
|
182
|
-
{ status: 'pending', nextAttemptAt: { $lte: claimedAt } },
|
|
183
|
-
{ status: 'processing', claimedAt: { $lt: new Date(claimedAt.getTime() - STALE_CLAIM_MS) } }
|
|
184
|
-
]
|
|
185
|
-
},
|
|
186
|
-
{ $set: { status: 'processing', claimedAt } },
|
|
187
|
-
{ new: false }
|
|
188
|
-
);
|
|
189
|
-
if (!prior) {
|
|
190
|
-
logger.info('[WorkflowRunner] Claimed elsewhere or not due', { kind: workflow.kind, deferredWorkId });
|
|
191
|
-
return { claimed: false };
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
const reclaimed = prior.status === 'processing';
|
|
195
|
-
const work = { ...prior.toObject(), attempt: prior.attempt + (reclaimed ? 1 : 0) };
|
|
196
|
-
if (reclaimed) await this._write(work, claimedAt, { attempt: work.attempt });
|
|
197
|
-
|
|
198
|
-
const checkpoints = { ...(work.checkpoints || {}) };
|
|
199
|
-
const save = async (patch) => {
|
|
200
|
-
Object.assign(checkpoints, patch);
|
|
201
|
-
const $set = {};
|
|
202
|
-
for (const [key, value] of Object.entries(patch || {})) $set[`checkpoints.${key}`] = value;
|
|
203
|
-
if (Object.keys($set).length) await this._write(work, claimedAt, $set);
|
|
204
|
-
};
|
|
205
|
-
|
|
206
|
-
try {
|
|
207
|
-
await workflow.prepare(work.trigger, checkpoints, save);
|
|
208
|
-
} catch (error) {
|
|
209
|
-
return await this._rescheduleOrAbandon(workflow, work, claimedAt, error);
|
|
210
|
-
}
|
|
211
|
-
await this._write(work, claimedAt, { status: 'done', claimedAt: null });
|
|
212
|
-
return { status: 'done' };
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
// matchedCount, not modifiedCount: the fence is "do we still hold the claim", and
|
|
216
|
-
// modifiedCount is also 0 for a write that matched but set an identical value - a
|
|
217
|
-
// save() rewriting an unchanged checkpoint is not a lost claim.
|
|
218
|
-
async _write(work, claimedAt, $set) {
|
|
219
|
-
const { matchedCount } = await DeferredWork.updateOne({ _id: work._id, claimedAt }, { $set });
|
|
220
|
-
if (!matchedCount) logger.warn('[WorkflowRunner] Claim lost, write discarded', { deferredWorkId: String(work._id) });
|
|
221
|
-
return matchedCount > 0;
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
async _rescheduleOrAbandon(workflow, work, claimedAt, error) {
|
|
225
|
-
// prepare() is consumer code and can throw anything, including null. Dereferencing
|
|
226
|
-
// it here used to raise a TypeError out of this handler, so neither branch ran and
|
|
227
|
-
// the record stayed `processing` until stale recovery hit the same failure again.
|
|
228
|
-
const lastError = describeThrown(error);
|
|
229
|
-
const delayMinutes = workflow.retrySchedule[work.attempt];
|
|
230
|
-
if (error?.permanent === true || delayMinutes === undefined) {
|
|
231
|
-
await this._write(work, claimedAt, { status: 'abandoned', lastError, claimedAt: null });
|
|
232
|
-
logger.error('[WorkflowRunner] Abandoned', {
|
|
233
|
-
kind: workflow.kind,
|
|
234
|
-
dedupeKey: work.dedupeKey,
|
|
235
|
-
attempt: work.attempt,
|
|
236
|
-
permanent: error?.permanent === true,
|
|
237
|
-
error: lastError
|
|
238
|
-
});
|
|
239
|
-
return { status: 'abandoned' };
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
const attempt = work.attempt + 1;
|
|
243
|
-
const delayMs = delayMinutes * MINUTE_MS;
|
|
244
|
-
const kept = await this._write(work, claimedAt, {
|
|
245
|
-
status: 'pending',
|
|
246
|
-
attempt,
|
|
247
|
-
nextAttemptAt: new Date(Date.now() + delayMs),
|
|
248
|
-
lastError,
|
|
249
|
-
claimedAt: null
|
|
250
|
-
});
|
|
251
|
-
if (!kept) return { claimed: false };
|
|
252
|
-
|
|
253
|
-
logger.warn('[WorkflowRunner] Rescheduled', {
|
|
254
|
-
kind: workflow.kind,
|
|
255
|
-
dedupeKey: work.dedupeKey,
|
|
256
|
-
attempt,
|
|
257
|
-
delayMinutes,
|
|
258
|
-
error: lastError
|
|
259
|
-
});
|
|
260
|
-
this._arm(workflow, work._id, delayMs);
|
|
261
|
-
return { status: 'pending', attempt };
|
|
262
|
-
}
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
module.exports = { WorkflowRunner };
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
const mongoose = require('mongoose');
|
|
2
|
-
|
|
3
|
-
const STATUS_VALUES = ['pending', 'processing', 'done', 'abandoned'];
|
|
4
|
-
const RETENTION_MS = 24 * 60 * 60 * 1000;
|
|
5
|
-
|
|
6
|
-
const deferredWorkSchema = new mongoose.Schema({
|
|
7
|
-
kind: { type: String, required: true },
|
|
8
|
-
dedupeKey: { type: String, required: true },
|
|
9
|
-
trigger: { type: mongoose.Schema.Types.Mixed, default: null },
|
|
10
|
-
checkpoints: { type: mongoose.Schema.Types.Mixed, default: () => ({}) },
|
|
11
|
-
status: { type: String, enum: STATUS_VALUES, default: 'pending' },
|
|
12
|
-
attempt: { type: Number, default: 0 },
|
|
13
|
-
nextAttemptAt: { type: Date, default: Date.now },
|
|
14
|
-
claimedAt: { type: Date, default: null },
|
|
15
|
-
lastError: { type: String, default: null }
|
|
16
|
-
}, { timestamps: true });
|
|
17
|
-
|
|
18
|
-
deferredWorkSchema.index({ kind: 1, dedupeKey: 1 }, { unique: true, name: 'dedupe_idx' });
|
|
19
|
-
deferredWorkSchema.index({ status: 1, nextAttemptAt: 1 }, { name: 'due_idx' });
|
|
20
|
-
deferredWorkSchema.index({ updatedAt: 1 }, { expireAfterSeconds: RETENTION_MS / 1000, name: 'ttl_idx' });
|
|
21
|
-
|
|
22
|
-
const DeferredWork = mongoose.model('DeferredWork', deferredWorkSchema);
|
|
23
|
-
|
|
24
|
-
module.exports = { DeferredWork, RETENTION_MS };
|