@peopl-health/nexus 5.54.0-dev.7872 → 5.54.0-dev.7891
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -32
- package/lib/clinical/services/assistantService.js +2 -17
- package/lib/clinical/services/pausedSafetyNetService.js +26 -1
- package/lib/core/NexusMessaging.js +9 -8
- package/lib/index.d.ts +4 -6
- package/lib/index.js +1 -0
- package/lib/services/airtableService.js +0 -7
- package/package.json +1 -1
- package/lib/core/DeferredWorkRunner.js +0 -257
- package/lib/models/deferredWorkModel.js +0 -24
package/README.md
CHANGED
|
@@ -116,38 +116,6 @@ const bus = nexus.getMessaging().getEventBus();
|
|
|
116
116
|
bus.on('message:received', (m) => console.log('rx', m.id));
|
|
117
117
|
```
|
|
118
118
|
|
|
119
|
-
## Workflows (Optional)
|
|
120
|
-
|
|
121
|
-
Register a named unit of work, deduplicated by a key you derive from the trigger.
|
|
122
|
-
|
|
123
|
-
```js
|
|
124
|
-
const runner = nexus.getMessaging().getWorkflowRunner();
|
|
125
|
-
|
|
126
|
-
await runner.register({
|
|
127
|
-
kind: 'triage-projection',
|
|
128
|
-
dedupeKey: (trigger) => String(trigger.submissionId),
|
|
129
|
-
prepare: async (trigger, checkpoints, save) => { /* ... */ },
|
|
130
|
-
retrySchedule: [5, 10, 20, 40], // minutes; omit for fire-and-forget
|
|
131
|
-
});
|
|
132
|
-
|
|
133
|
-
await runner.enqueue('triage-projection', { submissionId });
|
|
134
|
-
```
|
|
135
|
-
|
|
136
|
-
With `retrySchedule` the trigger and a checkpoint journal are stored in Mongo and the schedule runs
|
|
137
|
-
on an in-process timer — deferred workflows never touch the queue adapter, so they add no Redis load.
|
|
138
|
-
`prepare` resumes from `checkpoints`; call `save(patch)` before each external call so a retry does
|
|
139
|
-
not repeat completed work. Throw an error carrying `permanent: true` to give up immediately instead
|
|
140
|
-
of walking the schedule; the Airtable helpers tag their own permanent 4xx errors that way, in
|
|
141
|
-
`withAirtableRetry`, which all five of them route through. They rethrow on failure too, so an
|
|
142
|
-
Airtable outage reaches `prepare` as an exception and walks the retry schedule on its own rather
|
|
143
|
-
than reading as success.
|
|
144
|
-
|
|
145
|
-
Recovery is automatic: registering sweeps for work left behind by a previous process, and the runner
|
|
146
|
-
keeps sweeping so work whose worker died mid-`prepare` is reclaimed once its claim goes stale.
|
|
147
|
-
`enqueue` on an abandoned key revives it, keeping its checkpoints. Work is retained for 24 hours
|
|
148
|
-
after its last update and then expires — the dedupe key expires with it, so idempotency has the
|
|
149
|
-
same horizon as retention.
|
|
150
|
-
|
|
151
119
|
## Assistants (Optional)
|
|
152
120
|
|
|
153
121
|
Register assistant classes and (optionally) a custom resolver. OpenAI is supported via `llm: 'openai'`.
|
|
@@ -15,20 +15,10 @@ const { cleanupFiles } = require('../../helpers/filesHelper.js');
|
|
|
15
15
|
const { getMessages, countMessages } = require('../../services/messageService');
|
|
16
16
|
|
|
17
17
|
const { getAssistantById } = require('./assistantResolver');
|
|
18
|
-
const {
|
|
19
|
-
const { offerSafetyNet } = require('./pausedSafetyNetService');
|
|
18
|
+
const { coverUnansweredPatient } = require('./pausedSafetyNetService');
|
|
20
19
|
|
|
21
20
|
const MEDIA_BACKLOG_LIMIT = 10;
|
|
22
21
|
|
|
23
|
-
const pausedAlertText = async (code, newest) => {
|
|
24
|
-
if (!newest.interactive_type) return newest.plainBody || newest.body;
|
|
25
|
-
const [written] = await getMessages(
|
|
26
|
-
{ numero: code, from_me: false, interactive_type: null, createdAt: { $lte: newest.createdAt } },
|
|
27
|
-
{ sort: { createdAt: -1 }, limit: 1 }
|
|
28
|
-
);
|
|
29
|
-
return written ? written.plainBody || written.body : null;
|
|
30
|
-
};
|
|
31
|
-
|
|
32
22
|
const fetchMediaBacklog = async (code, trigger) => {
|
|
33
23
|
const [lastOutbound] = await getMessages(
|
|
34
24
|
{ numero: code, from_me: true, createdAt: { $lt: trigger.createdAt } },
|
|
@@ -122,12 +112,7 @@ const preProcessMessagesCore = async (code, message_ = null, thread) => {
|
|
|
122
112
|
return { shouldProcess: false, messages: null, timings, pendingMediaUpdates: [] };
|
|
123
113
|
}
|
|
124
114
|
|
|
125
|
-
if (thread.stopped) {
|
|
126
|
-
await Promise.allSettled([
|
|
127
|
-
recordPausedThreadMessage({ code, text: await pausedAlertText(code, lastMessage[0]) }),
|
|
128
|
-
offerSafetyNet({ code })
|
|
129
|
-
]);
|
|
130
|
-
}
|
|
115
|
+
if (thread.stopped) await coverUnansweredPatient({ code });
|
|
131
116
|
|
|
132
117
|
const backlog = await fetchMediaBacklog(code, lastMessage[0]);
|
|
133
118
|
const replies = [...backlog, ...lastMessage];
|
|
@@ -18,7 +18,7 @@ const { getMessages } = require('../../services/messageService');
|
|
|
18
18
|
const { getPatientTasks, updatePatientTask } = require('../../services/patientTaskService');
|
|
19
19
|
|
|
20
20
|
const { WriteClaimStore } = require('./writeClaimStore');
|
|
21
|
-
const { PAUSED_THREAD_DETAILS_PREFIX, recordUnresolvedRequest } = require('./clinicalAirtableService');
|
|
21
|
+
const { PAUSED_THREAD_DETAILS_PREFIX, recordPausedThreadMessage, recordUnresolvedRequest } = require('./clinicalAirtableService');
|
|
22
22
|
const { humanRepliedSince, openPledge, stopSafetyNetSweep } = require('./safetyNetPledgeService');
|
|
23
23
|
|
|
24
24
|
const getMessaging = () => require('../../core/NexusMessaging');
|
|
@@ -132,6 +132,30 @@ async function offerIsLive(code) {
|
|
|
132
132
|
}
|
|
133
133
|
}
|
|
134
134
|
|
|
135
|
+
async function unansweredText(code) {
|
|
136
|
+
const [written] = await getMessages(
|
|
137
|
+
{ numero: code, from_me: false, interactive_type: null },
|
|
138
|
+
{ sort: { createdAt: -1 }, limit: 1 }
|
|
139
|
+
);
|
|
140
|
+
return written ? written.plainBody || written.body : null;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
async function coverUnansweredPatient({ code }) {
|
|
144
|
+
if (!code) return;
|
|
145
|
+
|
|
146
|
+
const claim = await acquireSendClaim({ idempotencyKey: `safetynet-cover:${code}` });
|
|
147
|
+
if (!claim.acquired) return;
|
|
148
|
+
|
|
149
|
+
const results = await Promise.allSettled([
|
|
150
|
+
unansweredText(code).then((text) => recordPausedThreadMessage({ code, text })),
|
|
151
|
+
offerSafetyNet({ code }),
|
|
152
|
+
]);
|
|
153
|
+
|
|
154
|
+
const covered = results.every((r) => r.status === 'fulfilled');
|
|
155
|
+
if (covered) await completeSendClaim(claim.claimId);
|
|
156
|
+
else await releaseSendClaim(claim.claimId);
|
|
157
|
+
}
|
|
158
|
+
|
|
135
159
|
async function fileUrgentRequest(code, choice) {
|
|
136
160
|
const copy = REQUEST_COPY[choice];
|
|
137
161
|
return recordUnresolvedRequest({
|
|
@@ -283,6 +307,7 @@ function registerSafetyNetRoutes() {
|
|
|
283
307
|
}
|
|
284
308
|
|
|
285
309
|
module.exports = {
|
|
310
|
+
coverUnansweredPatient,
|
|
286
311
|
disarmSafetyNet,
|
|
287
312
|
offerSafetyNet,
|
|
288
313
|
handleSafetyNetTap,
|
|
@@ -31,6 +31,7 @@ const { releaseHeldMessages, sweepHeldMessages } = require('../services/heldMess
|
|
|
31
31
|
const { BatchingManager } = require('../core/BatchingManager');
|
|
32
32
|
const { ProcessingPipeline } = require('../core/ProcessingPipeline');
|
|
33
33
|
const { preProcessMessages, configureOpenAIProvider, AssistantProcessor, TURN_KIND } = require('../clinical');
|
|
34
|
+
const { coverUnansweredPatient } = require('../clinical/services/pausedSafetyNetService');
|
|
34
35
|
const { buildOutreachCarrier } = require('../clinical/services/outreachCarrierService');
|
|
35
36
|
const { routineOutreachCollisionReason } = require('../clinical/services/outreachSweepService');
|
|
36
37
|
const {
|
|
@@ -45,7 +46,6 @@ const { isLlmMonitorEnabled } = require('../clinical/flags/llmMonitorConfig');
|
|
|
45
46
|
const { createQueueAdapter } = require('../queue');
|
|
46
47
|
const { ScheduledMessageJob } = require('../jobs/ScheduledMessageJob');
|
|
47
48
|
const { TemplateApprovalJob } = require('../jobs/TemplateApprovalJob');
|
|
48
|
-
const { WorkflowRunner } = require('./DeferredWorkRunner');
|
|
49
49
|
const { invokeInteractiveRoute } = require('./interactiveRouteService');
|
|
50
50
|
|
|
51
51
|
const { PhiProcessor } = require('./PhiProcessor');
|
|
@@ -135,8 +135,6 @@ class NexusMessaging {
|
|
|
135
135
|
queueAdapter: this.queueAdapter,
|
|
136
136
|
});
|
|
137
137
|
|
|
138
|
-
this.workflowRunner = new WorkflowRunner({ queueAdapter: this.queueAdapter });
|
|
139
|
-
|
|
140
138
|
this.phiProcessor = new PhiProcessor({
|
|
141
139
|
encode: config.phi?.encode || false,
|
|
142
140
|
ner: config.phi?.ner || null,
|
|
@@ -321,7 +319,6 @@ class NexusMessaging {
|
|
|
321
319
|
getAssistantProcessor() { return this.assistantProcessor; }
|
|
322
320
|
getLlmMonitor() { return this.llmMonitor; }
|
|
323
321
|
getQueueAdapter() { return this.queueAdapter; }
|
|
324
|
-
getWorkflowRunner() { return this.workflowRunner; }
|
|
325
322
|
getPhiProcessor() { return this.phiProcessor; }
|
|
326
323
|
isConnected() { return this.provider?.getConnectionStatus() ?? false; }
|
|
327
324
|
isProcessing(chatId) { return this.batchingManager.isProcessing(chatId); }
|
|
@@ -679,8 +676,14 @@ class NexusMessaging {
|
|
|
679
676
|
return await this._executeWithPipeline(chatId, 'message', 'preempt',
|
|
680
677
|
async (preProcessResult, shouldContinue, runId) => {
|
|
681
678
|
return await this._processMessages(chatId, async () => {
|
|
682
|
-
const resolved = await this.assistantProcessor.resolveThread(chatId)
|
|
683
|
-
|
|
679
|
+
const resolved = await this.assistantProcessor.resolveThread(chatId).catch((error) => {
|
|
680
|
+
logger.warn('[NexusMessaging] Could not resolve an assistant for this patient', { code: chatId, error: error.message });
|
|
681
|
+
return null;
|
|
682
|
+
});
|
|
683
|
+
if (!resolved) {
|
|
684
|
+
await coverUnansweredPatient({ code: chatId });
|
|
685
|
+
return null;
|
|
686
|
+
}
|
|
684
687
|
|
|
685
688
|
const preProcessed = await preProcessMessages(chatId, null, resolved.thread);
|
|
686
689
|
|
|
@@ -1074,8 +1077,6 @@ class NexusMessaging {
|
|
|
1074
1077
|
await withTimeout(this._reconciliationStopped, teardownMs, 'queueReconciliation.stop')
|
|
1075
1078
|
.catch(error => logger.warn('[NexusMessaging] Reconciliation sweep did not settle in time', { error: error.message }));
|
|
1076
1079
|
|
|
1077
|
-
if (this.workflowRunner) this.workflowRunner.stop();
|
|
1078
|
-
|
|
1079
1080
|
if (this.queueAdapter) {
|
|
1080
1081
|
await withTimeout(this.queueAdapter.shutdown(), teardownMs, 'queueAdapter.shutdown')
|
|
1081
1082
|
.catch(error => logger.warn('[NexusMessaging] Queue shutdown did not finish', { error: error.message }));
|
package/lib/index.d.ts
CHANGED
|
@@ -254,7 +254,6 @@ declare module '@peopl-health/nexus' {
|
|
|
254
254
|
getAssistantProcessor(): AssistantProcessor;
|
|
255
255
|
getLlmMonitor(): { start(options?: { scheduleDaily?: boolean }): Promise<{ enabled: boolean }> } | null;
|
|
256
256
|
initializeLlmMonitor(): Promise<{ enabled: boolean; error?: string } | undefined>;
|
|
257
|
-
getWorkflowRunner(): WorkflowRunner;
|
|
258
257
|
processInstruction(code: string, instruction: string, role?: string, options?: { triggeredBy?: string }): Promise<void>;
|
|
259
258
|
processSystemMessage(code: string, messages: string | string[], role?: string, options?: { triggeredBy?: string; reply?: boolean }): Promise<void>;
|
|
260
259
|
processOutreach(code: string, options?: { brief?: OutreachBrief | null; triggeredBy?: string | null; reason?: string | null; firstName?: string | null; dryRun?: boolean }): Promise<OutreachDecision>;
|
|
@@ -542,17 +541,16 @@ declare module '@peopl-health/nexus' {
|
|
|
542
541
|
export interface Workflow {
|
|
543
542
|
kind: string;
|
|
544
543
|
dedupeKey: (trigger: any) => string;
|
|
545
|
-
prepare: (trigger: any
|
|
546
|
-
retrySchedule?: number[];
|
|
544
|
+
prepare: (trigger: any) => Promise<any>;
|
|
547
545
|
}
|
|
548
546
|
|
|
549
547
|
export interface WorkflowRunner {
|
|
550
548
|
register(workflow: Workflow): Promise<Workflow>;
|
|
551
|
-
enqueue(kind: string, trigger: any, options?: QueueJobOptions): Promise<string
|
|
552
|
-
sweep(only?: string[]): Promise<string[]>;
|
|
553
|
-
stop(): void;
|
|
549
|
+
enqueue(kind: string, trigger: any, options?: QueueJobOptions): Promise<string>;
|
|
554
550
|
}
|
|
555
551
|
|
|
552
|
+
export function createWorkflowRunner(options: { queueAdapter: QueueAdapter }): WorkflowRunner;
|
|
553
|
+
|
|
556
554
|
// Memory System
|
|
557
555
|
export interface PatientMemoryDocument {
|
|
558
556
|
_id: any;
|
package/lib/index.js
CHANGED
|
@@ -15,11 +15,6 @@ const AIRTABLE_API_URL = 'https://api.airtable.com/v0';
|
|
|
15
15
|
const AXIOS_TIMEOUT_CODE = 'ECONNABORTED';
|
|
16
16
|
const RETRY_BASE_DELAY_MS = 1000;
|
|
17
17
|
|
|
18
|
-
function isPermanentAirtableError(error) {
|
|
19
|
-
const status = error?.statusCode;
|
|
20
|
-
return typeof status === 'number' && status >= 400 && status < 500 && status !== 429;
|
|
21
|
-
}
|
|
22
|
-
|
|
23
18
|
let isEvalMode = false;
|
|
24
19
|
|
|
25
20
|
function setEvalMode(enabled) {
|
|
@@ -47,7 +42,6 @@ async function withAirtableRetry(operation, label, { retryNetworkErrors = true }
|
|
|
47
42
|
const isTransient = TRANSIENT_STATUS_CODES.includes(error.statusCode)
|
|
48
43
|
|| (retryNetworkErrors && isTransientNetworkError(error));
|
|
49
44
|
if (!isTransient || attempt === MAX_ATTEMPTS) {
|
|
50
|
-
if (isPermanentAirtableError(error)) error.permanent = true;
|
|
51
45
|
throw error;
|
|
52
46
|
}
|
|
53
47
|
const baseDelay = error.statusCode === RATE_LIMITED_STATUS ? RATE_LIMIT_DELAY_MS : RETRY_BASE_DELAY_MS;
|
|
@@ -242,7 +236,6 @@ async function upsertRecord(baseID, tableName, fields, options = {}, context = n
|
|
|
242
236
|
|
|
243
237
|
module.exports = {
|
|
244
238
|
setEvalMode,
|
|
245
|
-
isPermanentAirtableError,
|
|
246
239
|
addRecord,
|
|
247
240
|
getRecords,
|
|
248
241
|
getRecordByFilter,
|
package/package.json
CHANGED
|
@@ -1,257 +0,0 @@
|
|
|
1
|
-
const { describeThrown } = require('../utils/errorUtils');
|
|
2
|
-
const { logger } = require('../utils/logger');
|
|
3
|
-
|
|
4
|
-
const { DeferredWork, RETENTION_MS } = require('../models/deferredWorkModel');
|
|
5
|
-
|
|
6
|
-
const STALE_CLAIM_MS = 5 * 60 * 1000;
|
|
7
|
-
const MINUTE_MS = 60 * 1000;
|
|
8
|
-
const INDEX_TIMEOUT_MS = 1000;
|
|
9
|
-
const SWEEP_INTERVAL_MS = 60 * 1000;
|
|
10
|
-
|
|
11
|
-
let indexesReady = null;
|
|
12
|
-
|
|
13
|
-
function ensureIndexes() {
|
|
14
|
-
if (!indexesReady) {
|
|
15
|
-
indexesReady = DeferredWork.init().catch((error) => {
|
|
16
|
-
indexesReady = null;
|
|
17
|
-
logger.error('[WorkflowRunner] Index build failed, dedupe and retention are not guaranteed', { error: error.message });
|
|
18
|
-
});
|
|
19
|
-
}
|
|
20
|
-
return indexesReady;
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
function withIndexTimeout(promise) {
|
|
24
|
-
let timer;
|
|
25
|
-
const timeout = new Promise((resolve) => {
|
|
26
|
-
timer = setTimeout(resolve, INDEX_TIMEOUT_MS);
|
|
27
|
-
timer.unref();
|
|
28
|
-
});
|
|
29
|
-
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
class WorkflowRunner {
|
|
33
|
-
constructor({ queueAdapter, sweepIntervalMs = SWEEP_INTERVAL_MS } = {}) {
|
|
34
|
-
if (!queueAdapter) throw new Error('WorkflowRunner requires a queueAdapter');
|
|
35
|
-
this.queueAdapter = queueAdapter;
|
|
36
|
-
this.sweepIntervalMs = sweepIntervalMs;
|
|
37
|
-
this.sweepTimer = null;
|
|
38
|
-
this.timers = new Map();
|
|
39
|
-
this.workflows = new Map();
|
|
40
|
-
this.stopped = false;
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
async register(workflow = {}) {
|
|
44
|
-
if (!workflow.kind || typeof workflow.kind !== 'string') throw new Error('workflow requires a kind');
|
|
45
|
-
if (workflow.kind.includes('__')) throw new Error(`workflow '${workflow.kind}' must not contain '__' (reserved as the jobId delimiter)`);
|
|
46
|
-
if (typeof workflow.dedupeKey !== 'function') throw new Error(`workflow '${workflow.kind}' requires a dedupeKey function`);
|
|
47
|
-
if (typeof workflow.prepare !== 'function') throw new Error(`workflow '${workflow.kind}' requires a prepare function`);
|
|
48
|
-
if (this.workflows.has(workflow.kind)) throw new Error(`workflow '${workflow.kind}' is already registered`);
|
|
49
|
-
if (workflow.retrySchedule !== undefined) {
|
|
50
|
-
if (!Array.isArray(workflow.retrySchedule) || !workflow.retrySchedule.length) {
|
|
51
|
-
throw new Error(`workflow '${workflow.kind}' requires a non-empty retrySchedule array`);
|
|
52
|
-
}
|
|
53
|
-
if (workflow.retrySchedule.some((minutes) => !Number.isFinite(minutes) || minutes <= 0)) {
|
|
54
|
-
throw new Error(`workflow '${workflow.kind}' requires retrySchedule entries to be positive minutes`);
|
|
55
|
-
}
|
|
56
|
-
if (workflow.retrySchedule.some((minutes) => minutes * MINUTE_MS >= RETENTION_MS)) {
|
|
57
|
-
throw new Error(`workflow '${workflow.kind}' requires retrySchedule entries under the ${RETENTION_MS / MINUTE_MS}m retention, or the record expires before the attempt fires`);
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
this.workflows.set(workflow.kind, workflow);
|
|
62
|
-
if (workflow.retrySchedule) {
|
|
63
|
-
await this.sweep([workflow.kind]).catch((error) => {
|
|
64
|
-
logger.error('[WorkflowRunner] Sweep on register failed', { kind: workflow.kind, error: error.message });
|
|
65
|
-
});
|
|
66
|
-
this._startSweeping();
|
|
67
|
-
return workflow;
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
try {
|
|
71
|
-
await this.queueAdapter.process(workflow.kind, (trigger) => workflow.prepare(trigger));
|
|
72
|
-
} catch (error) {
|
|
73
|
-
this.workflows.delete(workflow.kind);
|
|
74
|
-
throw error;
|
|
75
|
-
}
|
|
76
|
-
return workflow;
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
enqueue(kind, trigger, options = {}) {
|
|
80
|
-
const workflow = this.workflows.get(kind);
|
|
81
|
-
if (!workflow) throw new Error(`no workflow registered for '${kind}'`);
|
|
82
|
-
const dedupeKey = workflow.dedupeKey(trigger);
|
|
83
|
-
if (dedupeKey == null || (typeof dedupeKey === 'string' && !dedupeKey.trim())) {
|
|
84
|
-
throw new Error(`workflow '${kind}' produced an empty dedupe key`);
|
|
85
|
-
}
|
|
86
|
-
if (typeof dedupeKey !== 'string') throw new Error(`workflow '${kind}' produced a non-string dedupe key`);
|
|
87
|
-
if (!workflow.retrySchedule) {
|
|
88
|
-
return this.queueAdapter.enqueue(kind, trigger, { ...options, jobId: `${kind}__${dedupeKey}` });
|
|
89
|
-
}
|
|
90
|
-
return this._armDeferred(workflow, trigger, dedupeKey);
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
async sweep(only = null) {
|
|
94
|
-
const deferredKinds = [...this.workflows.values()].filter((w) => w.retrySchedule).map((w) => w.kind);
|
|
95
|
-
const kinds = only ? deferredKinds.filter((kind) => only.includes(kind)) : deferredKinds;
|
|
96
|
-
if (!kinds.length) return [];
|
|
97
|
-
|
|
98
|
-
const orphaned = await DeferredWork.find({
|
|
99
|
-
kind: { $in: kinds },
|
|
100
|
-
$or: [
|
|
101
|
-
{ status: 'pending' },
|
|
102
|
-
{ status: 'processing', claimedAt: { $lt: new Date(Date.now() - STALE_CLAIM_MS) } }
|
|
103
|
-
]
|
|
104
|
-
});
|
|
105
|
-
|
|
106
|
-
const rearmed = orphaned.map((work) => this._arm(
|
|
107
|
-
this.workflows.get(work.kind),
|
|
108
|
-
work._id,
|
|
109
|
-
this._delayUntilDue(work),
|
|
110
|
-
));
|
|
111
|
-
if (rearmed.length) logger.info('[WorkflowRunner] Swept', { kinds, rearmed: rearmed.length });
|
|
112
|
-
return rearmed;
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
_startSweeping() {
|
|
116
|
-
if (this.stopped || this.sweepTimer || !this.sweepIntervalMs) return;
|
|
117
|
-
this.sweepTimer = setInterval(() => {
|
|
118
|
-
this.sweep().catch((error) => logger.error('[WorkflowRunner] Periodic sweep failed', { error: error.message }));
|
|
119
|
-
}, this.sweepIntervalMs);
|
|
120
|
-
this.sweepTimer.unref();
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
stop() {
|
|
124
|
-
this.stopped = true;
|
|
125
|
-
if (this.sweepTimer) clearInterval(this.sweepTimer);
|
|
126
|
-
this.sweepTimer = null;
|
|
127
|
-
for (const timer of this.timers.values()) clearTimeout(timer);
|
|
128
|
-
this.timers.clear();
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
async _armDeferred(workflow, trigger, dedupeKey) {
|
|
132
|
-
const { kind } = workflow;
|
|
133
|
-
await withIndexTimeout(ensureIndexes());
|
|
134
|
-
const work = await DeferredWork.findOne({ kind, dedupeKey })
|
|
135
|
-
|| await DeferredWork.create({ kind, dedupeKey, trigger }).catch(async (error) => {
|
|
136
|
-
if (error.code !== 11000) throw error;
|
|
137
|
-
return await DeferredWork.findOne({ kind, dedupeKey });
|
|
138
|
-
});
|
|
139
|
-
if (work.status === 'abandoned') {
|
|
140
|
-
const { modifiedCount } = await DeferredWork.updateOne(
|
|
141
|
-
{ _id: work._id, status: 'abandoned' },
|
|
142
|
-
{ $set: { status: 'pending', attempt: 0, nextAttemptAt: new Date(), lastError: null, claimedAt: null } }
|
|
143
|
-
);
|
|
144
|
-
if (!modifiedCount) return null;
|
|
145
|
-
logger.info('[WorkflowRunner] Re-arming abandoned work', { kind, dedupeKey });
|
|
146
|
-
return this._arm(workflow, work._id, 0);
|
|
147
|
-
}
|
|
148
|
-
if (work.status !== 'pending') {
|
|
149
|
-
logger.info('[WorkflowRunner] Skipping, work is in flight or complete', { kind, dedupeKey, status: work.status });
|
|
150
|
-
return null;
|
|
151
|
-
}
|
|
152
|
-
return this._arm(workflow, work._id, this._delayUntilDue(work));
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
_delayUntilDue(work) {
|
|
156
|
-
return Math.max(0, new Date(work.nextAttemptAt).getTime() - Date.now());
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
_arm(workflow, workId, delayMs) {
|
|
160
|
-
const id = String(workId);
|
|
161
|
-
if (this.stopped) return id;
|
|
162
|
-
clearTimeout(this.timers.get(id));
|
|
163
|
-
const timer = setTimeout(() => {
|
|
164
|
-
this.timers.delete(id);
|
|
165
|
-
this._runDeferred(workflow, id).catch((error) => {
|
|
166
|
-
logger.error('[WorkflowRunner] Deferred run failed', { kind: workflow.kind, deferredWorkId: id, error: error.message });
|
|
167
|
-
});
|
|
168
|
-
}, delayMs);
|
|
169
|
-
timer.unref();
|
|
170
|
-
this.timers.set(id, timer);
|
|
171
|
-
return id;
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
async _runDeferred(workflow, deferredWorkId) {
|
|
175
|
-
const claimedAt = new Date();
|
|
176
|
-
const prior = await DeferredWork.findOneAndUpdate(
|
|
177
|
-
{
|
|
178
|
-
_id: deferredWorkId,
|
|
179
|
-
$or: [
|
|
180
|
-
{ status: 'pending', nextAttemptAt: { $lte: claimedAt } },
|
|
181
|
-
{ status: 'processing', claimedAt: { $lt: new Date(claimedAt.getTime() - STALE_CLAIM_MS) } }
|
|
182
|
-
]
|
|
183
|
-
},
|
|
184
|
-
{ $set: { status: 'processing', claimedAt } },
|
|
185
|
-
{ new: false }
|
|
186
|
-
);
|
|
187
|
-
if (!prior) {
|
|
188
|
-
logger.info('[WorkflowRunner] Claimed elsewhere or not due', { kind: workflow.kind, deferredWorkId });
|
|
189
|
-
return { claimed: false };
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
const reclaimed = prior.status === 'processing';
|
|
193
|
-
const work = { ...prior.toObject(), attempt: prior.attempt + (reclaimed ? 1 : 0) };
|
|
194
|
-
if (reclaimed) await this._write(work, claimedAt, { attempt: work.attempt });
|
|
195
|
-
|
|
196
|
-
const checkpoints = { ...(work.checkpoints || {}) };
|
|
197
|
-
const save = async (patch) => {
|
|
198
|
-
Object.assign(checkpoints, patch);
|
|
199
|
-
const $set = {};
|
|
200
|
-
for (const [key, value] of Object.entries(patch || {})) $set[`checkpoints.${key}`] = value;
|
|
201
|
-
if (Object.keys($set).length) await this._write(work, claimedAt, $set);
|
|
202
|
-
};
|
|
203
|
-
|
|
204
|
-
try {
|
|
205
|
-
await workflow.prepare(work.trigger, checkpoints, save);
|
|
206
|
-
} catch (error) {
|
|
207
|
-
return await this._rescheduleOrAbandon(workflow, work, claimedAt, error);
|
|
208
|
-
}
|
|
209
|
-
await this._write(work, claimedAt, { status: 'done', claimedAt: null });
|
|
210
|
-
return { status: 'done' };
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
async _write(work, claimedAt, $set) {
|
|
214
|
-
const { matchedCount } = await DeferredWork.updateOne({ _id: work._id, claimedAt }, { $set });
|
|
215
|
-
if (!matchedCount) logger.warn('[WorkflowRunner] Claim lost, write discarded', { deferredWorkId: String(work._id) });
|
|
216
|
-
return matchedCount > 0;
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
async _rescheduleOrAbandon(workflow, work, claimedAt, error) {
|
|
220
|
-
const lastError = describeThrown(error);
|
|
221
|
-
const delayMinutes = workflow.retrySchedule[work.attempt];
|
|
222
|
-
if (error?.permanent === true || delayMinutes === undefined) {
|
|
223
|
-
await this._write(work, claimedAt, { status: 'abandoned', lastError, claimedAt: null });
|
|
224
|
-
logger.error('[WorkflowRunner] Abandoned', {
|
|
225
|
-
kind: workflow.kind,
|
|
226
|
-
dedupeKey: work.dedupeKey,
|
|
227
|
-
attempt: work.attempt,
|
|
228
|
-
permanent: error?.permanent === true,
|
|
229
|
-
error: lastError
|
|
230
|
-
});
|
|
231
|
-
return { status: 'abandoned' };
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
const attempt = work.attempt + 1;
|
|
235
|
-
const delayMs = delayMinutes * MINUTE_MS;
|
|
236
|
-
const kept = await this._write(work, claimedAt, {
|
|
237
|
-
status: 'pending',
|
|
238
|
-
attempt,
|
|
239
|
-
nextAttemptAt: new Date(Date.now() + delayMs),
|
|
240
|
-
lastError,
|
|
241
|
-
claimedAt: null
|
|
242
|
-
});
|
|
243
|
-
if (!kept) return { claimed: false };
|
|
244
|
-
|
|
245
|
-
logger.warn('[WorkflowRunner] Rescheduled', {
|
|
246
|
-
kind: workflow.kind,
|
|
247
|
-
dedupeKey: work.dedupeKey,
|
|
248
|
-
attempt,
|
|
249
|
-
delayMinutes,
|
|
250
|
-
error: lastError
|
|
251
|
-
});
|
|
252
|
-
this._arm(workflow, work._id, delayMs);
|
|
253
|
-
return { status: 'pending', attempt };
|
|
254
|
-
}
|
|
255
|
-
}
|
|
256
|
-
|
|
257
|
-
module.exports = { WorkflowRunner };
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
const mongoose = require('mongoose');
|
|
2
|
-
|
|
3
|
-
const STATUS_VALUES = ['pending', 'processing', 'done', 'abandoned'];
|
|
4
|
-
const RETENTION_MS = 24 * 60 * 60 * 1000;
|
|
5
|
-
|
|
6
|
-
const deferredWorkSchema = new mongoose.Schema({
|
|
7
|
-
kind: { type: String, required: true },
|
|
8
|
-
dedupeKey: { type: String, required: true },
|
|
9
|
-
trigger: { type: mongoose.Schema.Types.Mixed, default: null },
|
|
10
|
-
checkpoints: { type: mongoose.Schema.Types.Mixed, default: () => ({}) },
|
|
11
|
-
status: { type: String, enum: STATUS_VALUES, default: 'pending' },
|
|
12
|
-
attempt: { type: Number, default: 0 },
|
|
13
|
-
nextAttemptAt: { type: Date, default: Date.now },
|
|
14
|
-
claimedAt: { type: Date, default: null },
|
|
15
|
-
lastError: { type: String, default: null }
|
|
16
|
-
}, { timestamps: true });
|
|
17
|
-
|
|
18
|
-
deferredWorkSchema.index({ kind: 1, dedupeKey: 1 }, { unique: true, name: 'dedupe_idx' });
|
|
19
|
-
deferredWorkSchema.index({ status: 1, nextAttemptAt: 1 }, { name: 'due_idx' });
|
|
20
|
-
deferredWorkSchema.index({ updatedAt: 1 }, { expireAfterSeconds: RETENTION_MS / 1000, name: 'ttl_idx' });
|
|
21
|
-
|
|
22
|
-
const DeferredWork = mongoose.model('DeferredWork', deferredWorkSchema);
|
|
23
|
-
|
|
24
|
-
module.exports = { DeferredWork, RETENTION_MS };
|