@peopl-health/nexus 5.54.0-dev.7997 → 5.54.0-dev.8013
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -32
- package/lib/clinical/config/safetyNetConfig.js +31 -8
- package/lib/clinical/services/assistantService.js +2 -2
- package/lib/clinical/services/pausedSafetyNetService.js +38 -65
- package/lib/clinical/services/safetyNetPledgeService.js +41 -14
- package/lib/core/NexusMessaging.js +2 -8
- package/lib/index.d.ts +4 -6
- package/lib/index.js +1 -0
- package/lib/models/safetyNetPledgeModel.js +0 -1
- package/lib/services/airtableService.js +0 -7
- package/lib/utils/scheduleUtils.js +24 -1
- package/package.json +1 -1
- package/lib/core/DeferredWorkRunner.js +0 -257
- package/lib/models/deferredWorkModel.js +0 -24
package/README.md
CHANGED
|
@@ -116,38 +116,6 @@ const bus = nexus.getMessaging().getEventBus();
|
|
|
116
116
|
bus.on('message:received', (m) => console.log('rx', m.id));
|
|
117
117
|
```
|
|
118
118
|
|
|
119
|
-
## Workflows (Optional)
|
|
120
|
-
|
|
121
|
-
Register a named unit of work, deduplicated by a key you derive from the trigger.
|
|
122
|
-
|
|
123
|
-
```js
|
|
124
|
-
const runner = nexus.getMessaging().getWorkflowRunner();
|
|
125
|
-
|
|
126
|
-
await runner.register({
|
|
127
|
-
kind: 'triage-projection',
|
|
128
|
-
dedupeKey: (trigger) => String(trigger.submissionId),
|
|
129
|
-
prepare: async (trigger, checkpoints, save) => { /* ... */ },
|
|
130
|
-
retrySchedule: [5, 10, 20, 40], // minutes; omit for fire-and-forget
|
|
131
|
-
});
|
|
132
|
-
|
|
133
|
-
await runner.enqueue('triage-projection', { submissionId });
|
|
134
|
-
```
|
|
135
|
-
|
|
136
|
-
With `retrySchedule` the trigger and a checkpoint journal are stored in Mongo and the schedule runs
|
|
137
|
-
on an in-process timer — deferred workflows never touch the queue adapter, so they add no Redis load.
|
|
138
|
-
`prepare` resumes from `checkpoints`; call `save(patch)` before each external call so a retry does
|
|
139
|
-
not repeat completed work. Throw an error carrying `permanent: true` to give up immediately instead
|
|
140
|
-
of walking the schedule; the Airtable helpers tag their own permanent 4xx errors that way, in
|
|
141
|
-
`withAirtableRetry`, which all five of them route through. They rethrow on failure too, so an
|
|
142
|
-
Airtable outage reaches `prepare` as an exception and walks the retry schedule on its own rather
|
|
143
|
-
than reading as success.
|
|
144
|
-
|
|
145
|
-
Recovery is automatic: registering sweeps for work left behind by a previous process, and the runner
|
|
146
|
-
keeps sweeping so work whose worker died mid-`prepare` is reclaimed once its claim goes stale.
|
|
147
|
-
`enqueue` on an abandoned key revives it, keeping its checkpoints. Work is retained for 24 hours
|
|
148
|
-
after its last update and then expires — the dedupe key expires with it, so idempotency has the
|
|
149
|
-
same horizon as retention.
|
|
150
|
-
|
|
151
119
|
## Assistants (Optional)
|
|
152
120
|
|
|
153
121
|
Register assistant classes and (optionally) a custom resolver. OpenAI is supported via `llm: 'openai'`.
|
|
@@ -1,28 +1,46 @@
|
|
|
1
1
|
const runtimeConfig = require('../../config/runtimeConfig');
|
|
2
2
|
|
|
3
|
-
const DEFAULT_OFFER_WINDOW_MS =
|
|
4
|
-
const DEFAULT_HUMAN_QUIET_MS = 60 * 60 * 1000;
|
|
3
|
+
const DEFAULT_OFFER_WINDOW_MS = 24 * 60 * 60 * 1000;
|
|
4
|
+
const DEFAULT_HUMAN_QUIET_MS = 24 * 60 * 60 * 1000;
|
|
5
|
+
const DEFAULT_ACTIVITY_QUIET_MS = 2 * 60 * 60 * 1000;
|
|
5
6
|
const DEFAULT_TAP_WINDOW_MS = 30 * 60 * 1000;
|
|
6
|
-
const DEFAULT_PLEDGE_MS =
|
|
7
|
+
const DEFAULT_PLEDGE_MS = 8 * 60 * 60 * 1000;
|
|
7
8
|
const DEFAULT_SWEEP_INTERVAL_MS = 5 * 60 * 1000;
|
|
8
9
|
|
|
9
10
|
const CONTACT_WINDOW = { timeZone: 'America/Mexico_City', startHour: 8, endHour: 21 };
|
|
10
11
|
|
|
11
12
|
const SAFETY_NET_SOURCE = 'paused_safety_net';
|
|
12
13
|
|
|
13
|
-
const
|
|
14
|
+
const PATIENT_FACING_ORIGINS = ['whatsapp_platform', 'assistant'];
|
|
15
|
+
const HUMAN_ORIGINS = ['whatsapp_platform'];
|
|
16
|
+
const UNDELIVERED_STATUSES = ['failed', 'undelivered'];
|
|
17
|
+
const RECEIVED_STATUSES = ['delivered', 'read'];
|
|
18
|
+
|
|
19
|
+
const anyReplyFilter = (code, since) => ({
|
|
14
20
|
numero: code,
|
|
15
21
|
from_me: true,
|
|
16
22
|
content_sid: null,
|
|
17
|
-
origin: { $
|
|
23
|
+
origin: { $in: PATIENT_FACING_ORIGINS },
|
|
24
|
+
'statusInfo.status': { $nin: UNDELIVERED_STATUSES },
|
|
18
25
|
triggeredBy: { $ne: SAFETY_NET_SOURCE },
|
|
19
26
|
createdAt: { $gt: since }
|
|
20
27
|
});
|
|
21
28
|
|
|
29
|
+
const humanReplyFilter = (code, since) => ({
|
|
30
|
+
...anyReplyFilter(code, since),
|
|
31
|
+
origin: { $in: HUMAN_ORIGINS }
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
const settledHumanReplyFilter = (code, since, unconfirmedBefore) => ({
|
|
35
|
+
...humanReplyFilter(code, since),
|
|
36
|
+
$or: [
|
|
37
|
+
{ 'statusInfo.status': { $in: RECEIVED_STATUSES } },
|
|
38
|
+
{ createdAt: { $lte: unconfirmedBefore } }
|
|
39
|
+
]
|
|
40
|
+
});
|
|
41
|
+
|
|
22
42
|
const CHOICE_LABEL_KEYS = {
|
|
23
|
-
|
|
24
|
-
callback: 'PAUSED_SAFETY_NET_CALLBACK_LABEL',
|
|
25
|
-
later: 'PAUSED_SAFETY_NET_LATER_LABEL'
|
|
43
|
+
callback: 'PAUSED_SAFETY_NET_CALLBACK_LABEL'
|
|
26
44
|
};
|
|
27
45
|
|
|
28
46
|
let injected = {};
|
|
@@ -49,15 +67,20 @@ const duration = (key, envKey, fallback) => {
|
|
|
49
67
|
module.exports = {
|
|
50
68
|
CONTACT_WINDOW,
|
|
51
69
|
SAFETY_NET_SOURCE,
|
|
70
|
+
DEFAULT_PLEDGE_MS,
|
|
71
|
+
anyReplyFilter,
|
|
52
72
|
humanReplyFilter,
|
|
73
|
+
settledHumanReplyFilter,
|
|
53
74
|
CHOICE_LABEL_KEYS,
|
|
54
75
|
configureSafetyNet,
|
|
55
76
|
_resetSafetyNetConfig,
|
|
56
77
|
getSafetyNetContentSid: () => text('contentSid', 'PAUSED_SAFETY_NET_CONTENT_SID'),
|
|
78
|
+
getSafetyNetUnassignedContentSid: () => text('unassignedContentSid', 'PAUSED_SAFETY_NET_UNASSIGNED_CONTENT_SID'),
|
|
57
79
|
getSafetyNetPhone: () => text('phone', 'PAUSED_SAFETY_NET_PHONE'),
|
|
58
80
|
getSafetyNetLabel: (choice) => text(`${choice}Label`, CHOICE_LABEL_KEYS[choice]),
|
|
59
81
|
getOfferWindowMs: () => duration('offerWindowMs', 'PAUSED_SAFETY_NET_WINDOW_MS', DEFAULT_OFFER_WINDOW_MS),
|
|
60
82
|
getHumanQuietMs: () => duration('humanQuietMs', 'SAFETY_NET_HUMAN_QUIET_MS', DEFAULT_HUMAN_QUIET_MS),
|
|
83
|
+
getActivityQuietMs: () => duration('activityQuietMs', 'SAFETY_NET_ACTIVITY_QUIET_MS', DEFAULT_ACTIVITY_QUIET_MS),
|
|
61
84
|
getTapWindowMs: () => duration('tapWindowMs', 'SAFETY_NET_TAP_WINDOW_MS', DEFAULT_TAP_WINDOW_MS),
|
|
62
85
|
getPledgeMs: () => duration('pledgeMs', 'SAFETY_NET_PLEDGE_MS', DEFAULT_PLEDGE_MS),
|
|
63
86
|
getSweepIntervalMs: () => duration('sweepIntervalMs', 'SAFETY_NET_SWEEP_INTERVAL_MS', DEFAULT_SWEEP_INTERVAL_MS)
|
|
@@ -15,7 +15,7 @@ const { cleanupFiles } = require('../../helpers/filesHelper.js');
|
|
|
15
15
|
const { getMessages, countMessages } = require('../../services/messageService');
|
|
16
16
|
|
|
17
17
|
const { getAssistantById } = require('./assistantResolver');
|
|
18
|
-
const { coverUnansweredPatient } = require('./pausedSafetyNetService');
|
|
18
|
+
const { PAUSED, coverUnansweredPatient } = require('./pausedSafetyNetService');
|
|
19
19
|
|
|
20
20
|
const MEDIA_BACKLOG_LIMIT = 10;
|
|
21
21
|
|
|
@@ -112,7 +112,7 @@ const preProcessMessagesCore = async (code, message_ = null, thread) => {
|
|
|
112
112
|
return { shouldProcess: false, messages: null, timings, pendingMediaUpdates: [] };
|
|
113
113
|
}
|
|
114
114
|
|
|
115
|
-
if (thread.stopped) await coverUnansweredPatient({ code });
|
|
115
|
+
if (thread.stopped) await coverUnansweredPatient({ code, reason: PAUSED });
|
|
116
116
|
|
|
117
117
|
const backlog = await fetchMediaBacklog(code, lastMessage[0]);
|
|
118
118
|
const replies = [...backlog, ...lastMessage];
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
const {
|
|
2
|
-
CONTACT_WINDOW,
|
|
3
2
|
SAFETY_NET_SOURCE,
|
|
3
|
+
getActivityQuietMs,
|
|
4
4
|
getHumanQuietMs,
|
|
5
5
|
getOfferWindowMs,
|
|
6
|
-
getPledgeMs,
|
|
7
6
|
getSafetyNetContentSid,
|
|
8
7
|
getSafetyNetLabel,
|
|
9
8
|
getSafetyNetPhone,
|
|
9
|
+
getSafetyNetUnassignedContentSid,
|
|
10
10
|
getTapWindowMs
|
|
11
11
|
} = require('../config/safetyNetConfig');
|
|
12
12
|
const { logger } = require('../../utils/logger');
|
|
@@ -15,28 +15,23 @@ const { acquireSendClaim, completeSendClaim, releaseSendClaim } = require('../..
|
|
|
15
15
|
|
|
16
16
|
const { normalizePayload: normalize, registerInteractiveRoute, unregisterInteractiveRoute } = require('../../core/interactiveRouteService');
|
|
17
17
|
const { getMessages } = require('../../services/messageService');
|
|
18
|
-
const { getPatientTasks, updatePatientTask } = require('../../services/patientTaskService');
|
|
19
18
|
|
|
20
19
|
const { WriteClaimStore } = require('./writeClaimStore');
|
|
21
|
-
const {
|
|
22
|
-
const { humanRepliedSince, openPledge, stopSafetyNetSweep } = require('./safetyNetPledgeService');
|
|
20
|
+
const { recordPausedThreadMessage, recordUnresolvedRequest } = require('./clinicalAirtableService');
|
|
21
|
+
const { anyoneRepliedSince, humanRepliedSince, openPledge, stopSafetyNetSweep } = require('./safetyNetPledgeService');
|
|
23
22
|
|
|
24
23
|
const getMessaging = () => require('../../core/NexusMessaging');
|
|
25
24
|
|
|
26
25
|
const SOURCE = SAFETY_NET_SOURCE;
|
|
27
26
|
const PAYLOAD_PREFIX = 'peopl_safetynet_';
|
|
28
|
-
const CHOICES = ['
|
|
27
|
+
const CHOICES = ['callback'];
|
|
28
|
+
const PAUSED = 'paused';
|
|
29
|
+
const UNASSIGNED = 'unassigned';
|
|
29
30
|
const REQUEST_CATEGORY = 'contact_team_member';
|
|
30
|
-
const CAN_WAIT_NOTE = 'El paciente indicó que su solicitud puede esperar.';
|
|
31
|
-
const ATTENDED_STATUS = 'sin atender';
|
|
32
31
|
|
|
33
32
|
const REQUEST_COPY = {
|
|
34
|
-
urgent: {
|
|
35
|
-
question: 'El paciente indicó que su situación es urgente',
|
|
36
|
-
details: (code) => `El paciente ${code} recibió el aviso de bot en pausa y eligió "urgencia". Se le dio el número para llamar. Revisar la conversación ahora.`
|
|
37
|
-
},
|
|
38
33
|
callback: {
|
|
39
|
-
question: 'El paciente pidió que el equipo lo contacte
|
|
34
|
+
question: 'El paciente pidió que el equipo lo contacte',
|
|
40
35
|
details: (code) => `El paciente ${code} recibió el aviso de bot en pausa y pidió que lo contacten. Se le prometió contacto; revisar la conversación.`
|
|
41
36
|
}
|
|
42
37
|
};
|
|
@@ -81,15 +76,25 @@ function matchSafetyNetChoice(payload, title) {
|
|
|
81
76
|
}) || null;
|
|
82
77
|
}
|
|
83
78
|
|
|
79
|
+
const dialledNumbers = (template) => Object.values(template?.types ?? {})
|
|
80
|
+
.flatMap((type) => type?.actions ?? [])
|
|
81
|
+
.filter((action) => action?.type === 'PHONE_NUMBER')
|
|
82
|
+
.map((action) => action?.phone);
|
|
83
|
+
|
|
84
84
|
async function warnIfPhoneDiffersFromTemplate(contentSid) {
|
|
85
85
|
const phone = getSafetyNetPhone();
|
|
86
86
|
if (!phone || phoneCheckedFor === contentSid) return;
|
|
87
87
|
phoneCheckedFor = contentSid;
|
|
88
88
|
try {
|
|
89
|
-
const
|
|
89
|
+
const provider = getMessaging().requireProvider();
|
|
90
90
|
const digits = (value) => String(value ?? '').replace(/\D/g, '');
|
|
91
|
-
|
|
92
|
-
|
|
91
|
+
const wanted = digits(phone);
|
|
92
|
+
|
|
93
|
+
if (dialledNumbers(await provider.getTemplate(contentSid)).map(digits).includes(wanted)) return;
|
|
94
|
+
|
|
95
|
+
const body = await provider.renderTemplate(contentSid);
|
|
96
|
+
if (body && !digits(body).includes(wanted)) {
|
|
97
|
+
logger.error('[PausedSafetyNet] The configured phone number is absent from the template', { contentSid });
|
|
93
98
|
}
|
|
94
99
|
} catch (error) {
|
|
95
100
|
phoneCheckedFor = null;
|
|
@@ -97,13 +102,19 @@ async function warnIfPhoneDiffersFromTemplate(contentSid) {
|
|
|
97
102
|
}
|
|
98
103
|
}
|
|
99
104
|
|
|
100
|
-
|
|
105
|
+
function templateFor(reason) {
|
|
106
|
+
if (reason !== UNASSIGNED) return getSafetyNetContentSid();
|
|
107
|
+
return getSafetyNetUnassignedContentSid() || getSafetyNetContentSid();
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
async function offerSafetyNet({ code, reason = PAUSED }) {
|
|
101
111
|
try {
|
|
102
|
-
|
|
103
|
-
if (!code || !contentSid || !getSafetyNetPhone()) return { sent: false, reason: 'not_configured' };
|
|
112
|
+
if (!code || !getSafetyNetContentSid() || !getSafetyNetPhone()) return { sent: false, reason: 'not_configured' };
|
|
104
113
|
if (await offerClaims().get(code)) return { sent: false, reason: 'recent_offer' };
|
|
105
114
|
if (await humanRepliedSince(code, new Date(Date.now() - getHumanQuietMs()))) return { sent: false, reason: 'human_active' };
|
|
115
|
+
if (await anyoneRepliedSince(code, new Date(Date.now() - getActivityQuietMs()))) return { sent: false, reason: 'conversation_active' };
|
|
106
116
|
|
|
117
|
+
const contentSid = templateFor(reason);
|
|
107
118
|
await warnIfPhoneDiffersFromTemplate(contentSid);
|
|
108
119
|
|
|
109
120
|
const result = await getMessaging().sendMessage({ code, contentSid, triggeredBy: SOURCE });
|
|
@@ -118,11 +129,13 @@ async function offerSafetyNet({ code }) {
|
|
|
118
129
|
}
|
|
119
130
|
}
|
|
120
131
|
|
|
132
|
+
const offerContentSids = () => [getSafetyNetContentSid(), getSafetyNetUnassignedContentSid()].filter(Boolean);
|
|
133
|
+
|
|
121
134
|
async function offerIsLive(code) {
|
|
122
135
|
try {
|
|
123
136
|
const since = new Date(Date.now() - getOfferWindowMs());
|
|
124
137
|
const [sent] = await getMessages(
|
|
125
|
-
{ numero: code, from_me: true, content_sid:
|
|
138
|
+
{ numero: code, from_me: true, content_sid: { $in: offerContentSids() }, createdAt: { $gt: since } },
|
|
126
139
|
{ sort: { createdAt: -1 }, limit: 1 }
|
|
127
140
|
);
|
|
128
141
|
return Boolean(sent);
|
|
@@ -140,7 +153,7 @@ async function unansweredText(code) {
|
|
|
140
153
|
return written ? written.plainBody || written.body : null;
|
|
141
154
|
}
|
|
142
155
|
|
|
143
|
-
async function coverUnansweredPatient({ code }) {
|
|
156
|
+
async function coverUnansweredPatient({ code, reason = PAUSED }) {
|
|
144
157
|
if (!code) return;
|
|
145
158
|
|
|
146
159
|
const claim = await acquireSendClaim({ idempotencyKey: `safetynet-cover:${code}` });
|
|
@@ -148,7 +161,7 @@ async function coverUnansweredPatient({ code }) {
|
|
|
148
161
|
|
|
149
162
|
const results = await Promise.allSettled([
|
|
150
163
|
unansweredText(code).then((text) => recordPausedThreadMessage({ code, text })),
|
|
151
|
-
offerSafetyNet({ code }),
|
|
164
|
+
offerSafetyNet({ code, reason }),
|
|
152
165
|
]);
|
|
153
166
|
|
|
154
167
|
const covered = results.every((r) => r.status === 'fulfilled');
|
|
@@ -168,32 +181,8 @@ async function fileUrgentRequest(code, choice) {
|
|
|
168
181
|
});
|
|
169
182
|
}
|
|
170
183
|
|
|
171
|
-
async function markCanWait(code) {
|
|
172
|
-
const { requests = [] } = await getPatientTasks(code);
|
|
173
|
-
const target = requests.find((request) => request.status === ATTENDED_STATUS
|
|
174
|
-
&& request.category === REQUEST_CATEGORY
|
|
175
|
-
&& String(request.details || '').startsWith(PAUSED_THREAD_DETAILS_PREFIX));
|
|
176
|
-
if (!target) return false;
|
|
177
|
-
|
|
178
|
-
await updatePatientTask({
|
|
179
|
-
code,
|
|
180
|
-
kind: 'requests',
|
|
181
|
-
recordId: target.recordId,
|
|
182
|
-
changes: {
|
|
183
|
-
urgency: 'routine',
|
|
184
|
-
note: [target.note, CAN_WAIT_NOTE].filter(Boolean).join('\n')
|
|
185
|
-
},
|
|
186
|
-
updatedBy: SOURCE,
|
|
187
|
-
source: SOURCE
|
|
188
|
-
});
|
|
189
|
-
return true;
|
|
190
|
-
}
|
|
191
|
-
|
|
192
184
|
async function applyChoice(code, choice) {
|
|
193
|
-
if (choice === 'later') return { queued: true, downgraded: await markCanWait(code), pledge: null };
|
|
194
|
-
|
|
195
185
|
const queued = await fileUrgentRequest(code, choice);
|
|
196
|
-
if (choice !== 'callback') return { queued, pledge: null };
|
|
197
186
|
if (!queued) return { queued, pledge: null };
|
|
198
187
|
|
|
199
188
|
try {
|
|
@@ -218,30 +207,12 @@ function ackFor(choice, outcome) {
|
|
|
218
207
|
return withPhone('No pudimos registrar tu solicitud.', 'Por favor llámanos al {phone} para que te atendamos.');
|
|
219
208
|
}
|
|
220
209
|
|
|
221
|
-
if (choice === 'urgent') {
|
|
222
|
-
return withPhone('Entendido, ya avisamos al equipo.', 'Si es una urgencia médica, llámanos ahora al {phone}.');
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
if (choice === 'later') {
|
|
226
|
-
return outcome.downgraded
|
|
227
|
-
? withPhone('Gracias, lo anotamos. El equipo revisará tu mensaje.', 'Si necesitas ayuda antes, llámanos al {phone}.')
|
|
228
|
-
: withPhone('Gracias. El equipo revisará tu mensaje.', 'Si necesitas ayuda antes, llámanos al {phone}.');
|
|
229
|
-
}
|
|
230
|
-
|
|
231
210
|
if (!outcome.pledge) {
|
|
232
211
|
return withPhone('Listo, ya avisamos al equipo y te van a contactar.', 'Si te sientes peor antes, llámanos al {phone}.');
|
|
233
212
|
}
|
|
234
213
|
|
|
235
|
-
if (outcome.pledge.deferredToWindow) {
|
|
236
|
-
return withPhone(
|
|
237
|
-
`Listo, ya avisamos al equipo. Te van a contactar en cuanto abran, a partir de las ${CONTACT_WINDOW.startHour}:00.`,
|
|
238
|
-
'Si es una urgencia médica, llámanos ahora al {phone}.'
|
|
239
|
-
);
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
const minutes = Math.round(getPledgeMs() / 60000);
|
|
243
214
|
return withPhone(
|
|
244
|
-
|
|
215
|
+
'Listo. Un miembro del equipo te va a contactar a la brevedad.',
|
|
245
216
|
'Si te sientes peor antes, llámanos al {phone}.'
|
|
246
217
|
);
|
|
247
218
|
}
|
|
@@ -310,6 +281,8 @@ module.exports = {
|
|
|
310
281
|
coverUnansweredPatient,
|
|
311
282
|
disarmSafetyNet,
|
|
312
283
|
offerSafetyNet,
|
|
284
|
+
PAUSED,
|
|
285
|
+
UNASSIGNED,
|
|
313
286
|
handleSafetyNetTap,
|
|
314
287
|
matchSafetyNetChoice,
|
|
315
288
|
registerSafetyNetRoutes,
|
|
@@ -1,7 +1,10 @@
|
|
|
1
|
-
const {
|
|
1
|
+
const {
|
|
2
|
+
CONTACT_WINDOW, DEFAULT_PLEDGE_MS, anyReplyFilter, getPledgeMs, getSweepIntervalMs, humanReplyFilter,
|
|
3
|
+
settledHumanReplyFilter
|
|
4
|
+
} = require('../config/safetyNetConfig');
|
|
2
5
|
const { createIntervalSweeper } = require('../../utils/intervalSweeper');
|
|
3
6
|
const { logger } = require('../../utils/logger');
|
|
4
|
-
const {
|
|
7
|
+
const { addContactHours } = require('../../utils/scheduleUtils');
|
|
5
8
|
|
|
6
9
|
const { SafetyNetPledge } = require('../../models/safetyNetPledgeModel');
|
|
7
10
|
|
|
@@ -13,24 +16,36 @@ const SWEEP_LIMIT = 200;
|
|
|
13
16
|
const BREACH_CATEGORY = 'contact_team_member';
|
|
14
17
|
const BREACH_QUESTION = 'El paciente sigue esperando el contacto que se le prometió';
|
|
15
18
|
|
|
16
|
-
|
|
17
|
-
|
|
19
|
+
const DELIVERY_GRACE_MS = 10 * 60 * 1000;
|
|
20
|
+
const HOUR_MS = 60 * 60 * 1000;
|
|
21
|
+
|
|
22
|
+
async function repliedSince(filter) {
|
|
23
|
+
const [reply] = await getMessages(filter, { sort: { createdAt: -1 }, limit: 1 });
|
|
18
24
|
return Boolean(reply);
|
|
19
25
|
}
|
|
20
26
|
|
|
27
|
+
const humanRepliedSince = (code, since) => repliedSince(humanReplyFilter(code, since));
|
|
28
|
+
const humanReplySettledSince = (code, since) => repliedSince(
|
|
29
|
+
settledHumanReplyFilter(code, since, new Date(Date.now() - DELIVERY_GRACE_MS))
|
|
30
|
+
);
|
|
31
|
+
const anyoneRepliedSince = (code, since) => repliedSince(anyReplyFilter(code, since));
|
|
32
|
+
|
|
21
33
|
function pledgeDueAt(pledgedAt) {
|
|
22
|
-
const
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
34
|
+
const configured = addContactHours(pledgedAt, getPledgeMs() / HOUR_MS, CONTACT_WINDOW);
|
|
35
|
+
if (configured) return { dueAt: configured };
|
|
36
|
+
|
|
37
|
+
logger.error('[SafetyNetPledge] The configured pledge does not fit in working hours; using the default', {
|
|
38
|
+
pledgeMs: getPledgeMs()
|
|
39
|
+
});
|
|
40
|
+
return { dueAt: addContactHours(pledgedAt, DEFAULT_PLEDGE_MS / HOUR_MS, CONTACT_WINDOW) };
|
|
26
41
|
}
|
|
27
42
|
|
|
28
43
|
async function openPledge({ code }) {
|
|
29
44
|
const pledgedAt = new Date();
|
|
30
|
-
const { dueAt
|
|
31
|
-
const pledge = await SafetyNetPledge.create({ code, pledgedAt, dueAt
|
|
32
|
-
logger.info('[SafetyNetPledge] Contact pledged', { code, dueAt
|
|
33
|
-
return { pledgeId: pledge._id, dueAt
|
|
45
|
+
const { dueAt } = pledgeDueAt(pledgedAt);
|
|
46
|
+
const pledge = await SafetyNetPledge.create({ code, pledgedAt, dueAt });
|
|
47
|
+
logger.info('[SafetyNetPledge] Contact pledged', { code, dueAt });
|
|
48
|
+
return { pledgeId: pledge._id, dueAt };
|
|
34
49
|
}
|
|
35
50
|
|
|
36
51
|
async function reportBreach(pledge) {
|
|
@@ -51,6 +66,11 @@ const settle = async (id, from, state) => Boolean(
|
|
|
51
66
|
await SafetyNetPledge.findOneAndUpdate({ _id: id, state: from }, { $set: { state, resolvedAt: new Date() } }).lean()
|
|
52
67
|
);
|
|
53
68
|
|
|
69
|
+
const returnToPending = (id) => SafetyNetPledge.updateOne(
|
|
70
|
+
{ _id: id, state: 'reporting' },
|
|
71
|
+
{ $set: { state: 'pending', reportingSince: null } }
|
|
72
|
+
);
|
|
73
|
+
|
|
54
74
|
const takeReportingLease = async (id) => Boolean(
|
|
55
75
|
await SafetyNetPledge.findOneAndUpdate(
|
|
56
76
|
{ _id: id, $or: [{ state: 'pending' }, { state: 'reporting', reportingSince: { $lt: new Date(Date.now() - REPORT_LEASE_MS) } }] },
|
|
@@ -73,13 +93,18 @@ async function sweepSafetyNetPledges() {
|
|
|
73
93
|
try {
|
|
74
94
|
if (!await takeReportingLease(pledge._id)) continue;
|
|
75
95
|
|
|
76
|
-
if (await
|
|
96
|
+
if (await humanReplySettledSince(pledge.code, pledge.pledgedAt)) {
|
|
77
97
|
await settle(pledge._id, 'reporting', 'met');
|
|
78
98
|
continue;
|
|
79
99
|
}
|
|
80
100
|
|
|
101
|
+
if (await humanRepliedSince(pledge.code, pledge.pledgedAt)) {
|
|
102
|
+
await returnToPending(pledge._id);
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
|
|
81
106
|
if (!await reportBreach(pledge)) {
|
|
82
|
-
await
|
|
107
|
+
await returnToPending(pledge._id);
|
|
83
108
|
logger.error('[SafetyNetPledge] Breach was not recorded, pledge returns to pending for the next sweep', { code: pledge.code });
|
|
84
109
|
continue;
|
|
85
110
|
}
|
|
@@ -104,7 +129,9 @@ const stopSafetyNetSweep = () => sweeper.stop();
|
|
|
104
129
|
module.exports = {
|
|
105
130
|
startSafetyNetSweep,
|
|
106
131
|
stopSafetyNetSweep,
|
|
132
|
+
anyoneRepliedSince,
|
|
107
133
|
humanRepliedSince,
|
|
134
|
+
humanReplySettledSince,
|
|
108
135
|
openPledge,
|
|
109
136
|
pledgeDueAt,
|
|
110
137
|
sweepSafetyNetPledges,
|
|
@@ -31,7 +31,7 @@ const { releaseHeldMessages, sweepHeldMessages } = require('../services/heldMess
|
|
|
31
31
|
const { BatchingManager } = require('../core/BatchingManager');
|
|
32
32
|
const { ProcessingPipeline } = require('../core/ProcessingPipeline');
|
|
33
33
|
const { preProcessMessages, configureOpenAIProvider, AssistantProcessor, TURN_KIND } = require('../clinical');
|
|
34
|
-
const { coverUnansweredPatient } = require('../clinical/services/pausedSafetyNetService');
|
|
34
|
+
const { UNASSIGNED, coverUnansweredPatient } = require('../clinical/services/pausedSafetyNetService');
|
|
35
35
|
const { buildOutreachCarrier } = require('../clinical/services/outreachCarrierService');
|
|
36
36
|
const { routineOutreachCollisionReason } = require('../clinical/services/outreachSweepService');
|
|
37
37
|
const {
|
|
@@ -46,7 +46,6 @@ const { isLlmMonitorEnabled } = require('../clinical/flags/llmMonitorConfig');
|
|
|
46
46
|
const { createQueueAdapter } = require('../queue');
|
|
47
47
|
const { ScheduledMessageJob } = require('../jobs/ScheduledMessageJob');
|
|
48
48
|
const { TemplateApprovalJob } = require('../jobs/TemplateApprovalJob');
|
|
49
|
-
const { WorkflowRunner } = require('./DeferredWorkRunner');
|
|
50
49
|
const { invokeInteractiveRoute } = require('./interactiveRouteService');
|
|
51
50
|
|
|
52
51
|
const { PhiProcessor } = require('./PhiProcessor');
|
|
@@ -136,8 +135,6 @@ class NexusMessaging {
|
|
|
136
135
|
queueAdapter: this.queueAdapter,
|
|
137
136
|
});
|
|
138
137
|
|
|
139
|
-
this.workflowRunner = new WorkflowRunner({ queueAdapter: this.queueAdapter });
|
|
140
|
-
|
|
141
138
|
this.phiProcessor = new PhiProcessor({
|
|
142
139
|
encode: config.phi?.encode || false,
|
|
143
140
|
ner: config.phi?.ner || null,
|
|
@@ -322,7 +319,6 @@ class NexusMessaging {
|
|
|
322
319
|
getAssistantProcessor() { return this.assistantProcessor; }
|
|
323
320
|
getLlmMonitor() { return this.llmMonitor; }
|
|
324
321
|
getQueueAdapter() { return this.queueAdapter; }
|
|
325
|
-
getWorkflowRunner() { return this.workflowRunner; }
|
|
326
322
|
getPhiProcessor() { return this.phiProcessor; }
|
|
327
323
|
isConnected() { return this.provider?.getConnectionStatus() ?? false; }
|
|
328
324
|
isProcessing(chatId) { return this.batchingManager.isProcessing(chatId); }
|
|
@@ -697,7 +693,7 @@ class NexusMessaging {
|
|
|
697
693
|
return null;
|
|
698
694
|
});
|
|
699
695
|
if (!resolved) {
|
|
700
|
-
await coverUnansweredPatient({ code: chatId });
|
|
696
|
+
await coverUnansweredPatient({ code: chatId, reason: UNASSIGNED });
|
|
701
697
|
return null;
|
|
702
698
|
}
|
|
703
699
|
|
|
@@ -1102,8 +1098,6 @@ class NexusMessaging {
|
|
|
1102
1098
|
await withTimeout(this._reconciliationStopped, teardownMs, 'queueReconciliation.stop')
|
|
1103
1099
|
.catch(error => logger.warn('[NexusMessaging] Reconciliation sweep did not settle in time', { error: error.message }));
|
|
1104
1100
|
|
|
1105
|
-
if (this.workflowRunner) this.workflowRunner.stop();
|
|
1106
|
-
|
|
1107
1101
|
if (this.queueAdapter) {
|
|
1108
1102
|
await withTimeout(this.queueAdapter.shutdown(), teardownMs, 'queueAdapter.shutdown')
|
|
1109
1103
|
.catch(error => logger.warn('[NexusMessaging] Queue shutdown did not finish', { error: error.message }));
|
package/lib/index.d.ts
CHANGED
|
@@ -254,7 +254,6 @@ declare module '@peopl-health/nexus' {
|
|
|
254
254
|
getAssistantProcessor(): AssistantProcessor;
|
|
255
255
|
getLlmMonitor(): { start(options?: { scheduleDaily?: boolean }): Promise<{ enabled: boolean }> } | null;
|
|
256
256
|
initializeLlmMonitor(): Promise<{ enabled: boolean; error?: string } | undefined>;
|
|
257
|
-
getWorkflowRunner(): WorkflowRunner;
|
|
258
257
|
processInstruction(code: string, instruction: string, role?: string, options?: { triggeredBy?: string }): Promise<void>;
|
|
259
258
|
processSystemMessage(code: string, messages: string | string[], role?: string, options?: { triggeredBy?: string; reply?: boolean }): Promise<void>;
|
|
260
259
|
processOutreach(code: string, options?: { brief?: OutreachBrief | null; triggeredBy?: string | null; reason?: string | null; firstName?: string | null; dryRun?: boolean }): Promise<OutreachDecision>;
|
|
@@ -548,17 +547,16 @@ declare module '@peopl-health/nexus' {
|
|
|
548
547
|
export interface Workflow {
|
|
549
548
|
kind: string;
|
|
550
549
|
dedupeKey: (trigger: any) => string;
|
|
551
|
-
prepare: (trigger: any
|
|
552
|
-
retrySchedule?: number[];
|
|
550
|
+
prepare: (trigger: any) => Promise<any>;
|
|
553
551
|
}
|
|
554
552
|
|
|
555
553
|
export interface WorkflowRunner {
|
|
556
554
|
register(workflow: Workflow): Promise<Workflow>;
|
|
557
|
-
enqueue(kind: string, trigger: any, options?: QueueJobOptions): Promise<string
|
|
558
|
-
sweep(only?: string[]): Promise<string[]>;
|
|
559
|
-
stop(): void;
|
|
555
|
+
enqueue(kind: string, trigger: any, options?: QueueJobOptions): Promise<string>;
|
|
560
556
|
}
|
|
561
557
|
|
|
558
|
+
export function createWorkflowRunner(options: { queueAdapter: QueueAdapter }): WorkflowRunner;
|
|
559
|
+
|
|
562
560
|
// Memory System
|
|
563
561
|
export interface PatientMemoryDocument {
|
|
564
562
|
_id: any;
|
package/lib/index.js
CHANGED
|
@@ -6,7 +6,6 @@ const safetyNetPledgeSchema = new mongoose.Schema({
|
|
|
6
6
|
code: { type: String, required: true },
|
|
7
7
|
pledgedAt: { type: Date, required: true },
|
|
8
8
|
dueAt: { type: Date, required: true },
|
|
9
|
-
deferredToWindow: { type: Boolean, default: false },
|
|
10
9
|
state: { type: String, enum: PLEDGE_STATES, default: 'pending' },
|
|
11
10
|
reportingSince: { type: Date, default: null },
|
|
12
11
|
resolvedAt: { type: Date, default: null },
|
|
@@ -15,11 +15,6 @@ const AIRTABLE_API_URL = 'https://api.airtable.com/v0';
|
|
|
15
15
|
const AXIOS_TIMEOUT_CODE = 'ECONNABORTED';
|
|
16
16
|
const RETRY_BASE_DELAY_MS = 1000;
|
|
17
17
|
|
|
18
|
-
function isPermanentAirtableError(error) {
|
|
19
|
-
const status = error?.statusCode;
|
|
20
|
-
return typeof status === 'number' && status >= 400 && status < 500 && status !== 429;
|
|
21
|
-
}
|
|
22
|
-
|
|
23
18
|
let isEvalMode = false;
|
|
24
19
|
|
|
25
20
|
function setEvalMode(enabled) {
|
|
@@ -47,7 +42,6 @@ async function withAirtableRetry(operation, label, { retryNetworkErrors = true }
|
|
|
47
42
|
const isTransient = TRANSIENT_STATUS_CODES.includes(error.statusCode)
|
|
48
43
|
|| (retryNetworkErrors && isTransientNetworkError(error));
|
|
49
44
|
if (!isTransient || attempt === MAX_ATTEMPTS) {
|
|
50
|
-
if (isPermanentAirtableError(error)) error.permanent = true;
|
|
51
45
|
throw error;
|
|
52
46
|
}
|
|
53
47
|
const baseDelay = error.statusCode === RATE_LIMITED_STATUS ? RATE_LIMIT_DELAY_MS : RETRY_BASE_DELAY_MS;
|
|
@@ -242,7 +236,6 @@ async function upsertRecord(baseID, tableName, fields, options = {}, context = n
|
|
|
242
236
|
|
|
243
237
|
module.exports = {
|
|
244
238
|
setEvalMode,
|
|
245
|
-
isPermanentAirtableError,
|
|
246
239
|
addRecord,
|
|
247
240
|
getRecords,
|
|
248
241
|
getRecordByFilter,
|
|
@@ -50,6 +50,29 @@ function contactHoursBetween(from, to, { timeZone, startHour, endHour }) {
|
|
|
50
50
|
return cursor.isBefore(end) ? Infinity : total;
|
|
51
51
|
}
|
|
52
52
|
|
|
53
|
+
function addContactHours(from, hours, { timeZone, startHour, endHour }) {
|
|
54
|
+
const start = moment.tz(from, timeZone);
|
|
55
|
+
if (!start.isValid() || !(hours >= 0)) {
|
|
56
|
+
logger.warn('[addContactHours] Invalid input', { from, hours, timeZone });
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
let remaining = hours * 60 * 60 * 1000;
|
|
61
|
+
let cursor = start.clone();
|
|
62
|
+
for (let day = 0; day < MAX_CONTACT_SPAN_DAYS; day += 1) {
|
|
63
|
+
const opens = cursor.clone().startOf('day').hour(startHour);
|
|
64
|
+
const closes = cursor.clone().startOf('day').hour(endHour);
|
|
65
|
+
if (cursor.isBefore(opens)) cursor = opens;
|
|
66
|
+
if (cursor.isBefore(closes)) {
|
|
67
|
+
const available = closes.diff(cursor);
|
|
68
|
+
if (remaining < available) return cursor.clone().add(remaining, 'ms').toDate();
|
|
69
|
+
remaining -= available;
|
|
70
|
+
}
|
|
71
|
+
cursor = opens.clone().add(1, 'day');
|
|
72
|
+
}
|
|
73
|
+
return remaining === 0 ? cursor.toDate() : null;
|
|
74
|
+
}
|
|
75
|
+
|
|
53
76
|
function calculateDelay(sendTime, timeZone) {
|
|
54
77
|
if (!sendTime) return DEFAULT_DELAY;
|
|
55
78
|
|
|
@@ -88,4 +111,4 @@ function calculateDelay(sendTime, timeZone) {
|
|
|
88
111
|
}
|
|
89
112
|
}
|
|
90
113
|
|
|
91
|
-
module.exports = { calculateDelay, clampToHourWindow, isWithinHourWindow, contactHoursBetween, DEFAULT_TIMEZONE };
|
|
114
|
+
module.exports = { addContactHours, calculateDelay, clampToHourWindow, isWithinHourWindow, contactHoursBetween, DEFAULT_TIMEZONE };
|
package/package.json
CHANGED
|
@@ -1,257 +0,0 @@
|
|
|
1
|
-
const { describeThrown } = require('../utils/errorUtils');
|
|
2
|
-
const { logger } = require('../utils/logger');
|
|
3
|
-
|
|
4
|
-
const { DeferredWork, RETENTION_MS } = require('../models/deferredWorkModel');
|
|
5
|
-
|
|
6
|
-
const STALE_CLAIM_MS = 5 * 60 * 1000;
|
|
7
|
-
const MINUTE_MS = 60 * 1000;
|
|
8
|
-
const INDEX_TIMEOUT_MS = 1000;
|
|
9
|
-
const SWEEP_INTERVAL_MS = 60 * 1000;
|
|
10
|
-
|
|
11
|
-
let indexesReady = null;
|
|
12
|
-
|
|
13
|
-
function ensureIndexes() {
|
|
14
|
-
if (!indexesReady) {
|
|
15
|
-
indexesReady = DeferredWork.init().catch((error) => {
|
|
16
|
-
indexesReady = null;
|
|
17
|
-
logger.error('[WorkflowRunner] Index build failed, dedupe and retention are not guaranteed', { error: error.message });
|
|
18
|
-
});
|
|
19
|
-
}
|
|
20
|
-
return indexesReady;
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
function withIndexTimeout(promise) {
|
|
24
|
-
let timer;
|
|
25
|
-
const timeout = new Promise((resolve) => {
|
|
26
|
-
timer = setTimeout(resolve, INDEX_TIMEOUT_MS);
|
|
27
|
-
timer.unref();
|
|
28
|
-
});
|
|
29
|
-
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
class WorkflowRunner {
|
|
33
|
-
constructor({ queueAdapter, sweepIntervalMs = SWEEP_INTERVAL_MS } = {}) {
|
|
34
|
-
if (!queueAdapter) throw new Error('WorkflowRunner requires a queueAdapter');
|
|
35
|
-
this.queueAdapter = queueAdapter;
|
|
36
|
-
this.sweepIntervalMs = sweepIntervalMs;
|
|
37
|
-
this.sweepTimer = null;
|
|
38
|
-
this.timers = new Map();
|
|
39
|
-
this.workflows = new Map();
|
|
40
|
-
this.stopped = false;
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
async register(workflow = {}) {
|
|
44
|
-
if (!workflow.kind || typeof workflow.kind !== 'string') throw new Error('workflow requires a kind');
|
|
45
|
-
if (workflow.kind.includes('__')) throw new Error(`workflow '${workflow.kind}' must not contain '__' (reserved as the jobId delimiter)`);
|
|
46
|
-
if (typeof workflow.dedupeKey !== 'function') throw new Error(`workflow '${workflow.kind}' requires a dedupeKey function`);
|
|
47
|
-
if (typeof workflow.prepare !== 'function') throw new Error(`workflow '${workflow.kind}' requires a prepare function`);
|
|
48
|
-
if (this.workflows.has(workflow.kind)) throw new Error(`workflow '${workflow.kind}' is already registered`);
|
|
49
|
-
if (workflow.retrySchedule !== undefined) {
|
|
50
|
-
if (!Array.isArray(workflow.retrySchedule) || !workflow.retrySchedule.length) {
|
|
51
|
-
throw new Error(`workflow '${workflow.kind}' requires a non-empty retrySchedule array`);
|
|
52
|
-
}
|
|
53
|
-
if (workflow.retrySchedule.some((minutes) => !Number.isFinite(minutes) || minutes <= 0)) {
|
|
54
|
-
throw new Error(`workflow '${workflow.kind}' requires retrySchedule entries to be positive minutes`);
|
|
55
|
-
}
|
|
56
|
-
if (workflow.retrySchedule.some((minutes) => minutes * MINUTE_MS >= RETENTION_MS)) {
|
|
57
|
-
throw new Error(`workflow '${workflow.kind}' requires retrySchedule entries under the ${RETENTION_MS / MINUTE_MS}m retention, or the record expires before the attempt fires`);
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
this.workflows.set(workflow.kind, workflow);
|
|
62
|
-
if (workflow.retrySchedule) {
|
|
63
|
-
await this.sweep([workflow.kind]).catch((error) => {
|
|
64
|
-
logger.error('[WorkflowRunner] Sweep on register failed', { kind: workflow.kind, error: error.message });
|
|
65
|
-
});
|
|
66
|
-
this._startSweeping();
|
|
67
|
-
return workflow;
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
try {
|
|
71
|
-
await this.queueAdapter.process(workflow.kind, (trigger) => workflow.prepare(trigger));
|
|
72
|
-
} catch (error) {
|
|
73
|
-
this.workflows.delete(workflow.kind);
|
|
74
|
-
throw error;
|
|
75
|
-
}
|
|
76
|
-
return workflow;
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
enqueue(kind, trigger, options = {}) {
|
|
80
|
-
const workflow = this.workflows.get(kind);
|
|
81
|
-
if (!workflow) throw new Error(`no workflow registered for '${kind}'`);
|
|
82
|
-
const dedupeKey = workflow.dedupeKey(trigger);
|
|
83
|
-
if (dedupeKey == null || (typeof dedupeKey === 'string' && !dedupeKey.trim())) {
|
|
84
|
-
throw new Error(`workflow '${kind}' produced an empty dedupe key`);
|
|
85
|
-
}
|
|
86
|
-
if (typeof dedupeKey !== 'string') throw new Error(`workflow '${kind}' produced a non-string dedupe key`);
|
|
87
|
-
if (!workflow.retrySchedule) {
|
|
88
|
-
return this.queueAdapter.enqueue(kind, trigger, { ...options, jobId: `${kind}__${dedupeKey}` });
|
|
89
|
-
}
|
|
90
|
-
return this._armDeferred(workflow, trigger, dedupeKey);
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
async sweep(only = null) {
|
|
94
|
-
const deferredKinds = [...this.workflows.values()].filter((w) => w.retrySchedule).map((w) => w.kind);
|
|
95
|
-
const kinds = only ? deferredKinds.filter((kind) => only.includes(kind)) : deferredKinds;
|
|
96
|
-
if (!kinds.length) return [];
|
|
97
|
-
|
|
98
|
-
const orphaned = await DeferredWork.find({
|
|
99
|
-
kind: { $in: kinds },
|
|
100
|
-
$or: [
|
|
101
|
-
{ status: 'pending' },
|
|
102
|
-
{ status: 'processing', claimedAt: { $lt: new Date(Date.now() - STALE_CLAIM_MS) } }
|
|
103
|
-
]
|
|
104
|
-
});
|
|
105
|
-
|
|
106
|
-
const rearmed = orphaned.map((work) => this._arm(
|
|
107
|
-
this.workflows.get(work.kind),
|
|
108
|
-
work._id,
|
|
109
|
-
this._delayUntilDue(work),
|
|
110
|
-
));
|
|
111
|
-
if (rearmed.length) logger.info('[WorkflowRunner] Swept', { kinds, rearmed: rearmed.length });
|
|
112
|
-
return rearmed;
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
_startSweeping() {
|
|
116
|
-
if (this.stopped || this.sweepTimer || !this.sweepIntervalMs) return;
|
|
117
|
-
this.sweepTimer = setInterval(() => {
|
|
118
|
-
this.sweep().catch((error) => logger.error('[WorkflowRunner] Periodic sweep failed', { error: error.message }));
|
|
119
|
-
}, this.sweepIntervalMs);
|
|
120
|
-
this.sweepTimer.unref();
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
stop() {
|
|
124
|
-
this.stopped = true;
|
|
125
|
-
if (this.sweepTimer) clearInterval(this.sweepTimer);
|
|
126
|
-
this.sweepTimer = null;
|
|
127
|
-
for (const timer of this.timers.values()) clearTimeout(timer);
|
|
128
|
-
this.timers.clear();
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
async _armDeferred(workflow, trigger, dedupeKey) {
|
|
132
|
-
const { kind } = workflow;
|
|
133
|
-
await withIndexTimeout(ensureIndexes());
|
|
134
|
-
const work = await DeferredWork.findOne({ kind, dedupeKey })
|
|
135
|
-
|| await DeferredWork.create({ kind, dedupeKey, trigger }).catch(async (error) => {
|
|
136
|
-
if (error.code !== 11000) throw error;
|
|
137
|
-
return await DeferredWork.findOne({ kind, dedupeKey });
|
|
138
|
-
});
|
|
139
|
-
if (work.status === 'abandoned') {
|
|
140
|
-
const { modifiedCount } = await DeferredWork.updateOne(
|
|
141
|
-
{ _id: work._id, status: 'abandoned' },
|
|
142
|
-
{ $set: { status: 'pending', attempt: 0, nextAttemptAt: new Date(), lastError: null, claimedAt: null } }
|
|
143
|
-
);
|
|
144
|
-
if (!modifiedCount) return null;
|
|
145
|
-
logger.info('[WorkflowRunner] Re-arming abandoned work', { kind, dedupeKey });
|
|
146
|
-
return this._arm(workflow, work._id, 0);
|
|
147
|
-
}
|
|
148
|
-
if (work.status !== 'pending') {
|
|
149
|
-
logger.info('[WorkflowRunner] Skipping, work is in flight or complete', { kind, dedupeKey, status: work.status });
|
|
150
|
-
return null;
|
|
151
|
-
}
|
|
152
|
-
return this._arm(workflow, work._id, this._delayUntilDue(work));
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
_delayUntilDue(work) {
|
|
156
|
-
return Math.max(0, new Date(work.nextAttemptAt).getTime() - Date.now());
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
_arm(workflow, workId, delayMs) {
|
|
160
|
-
const id = String(workId);
|
|
161
|
-
if (this.stopped) return id;
|
|
162
|
-
clearTimeout(this.timers.get(id));
|
|
163
|
-
const timer = setTimeout(() => {
|
|
164
|
-
this.timers.delete(id);
|
|
165
|
-
this._runDeferred(workflow, id).catch((error) => {
|
|
166
|
-
logger.error('[WorkflowRunner] Deferred run failed', { kind: workflow.kind, deferredWorkId: id, error: error.message });
|
|
167
|
-
});
|
|
168
|
-
}, delayMs);
|
|
169
|
-
timer.unref();
|
|
170
|
-
this.timers.set(id, timer);
|
|
171
|
-
return id;
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
async _runDeferred(workflow, deferredWorkId) {
|
|
175
|
-
const claimedAt = new Date();
|
|
176
|
-
const prior = await DeferredWork.findOneAndUpdate(
|
|
177
|
-
{
|
|
178
|
-
_id: deferredWorkId,
|
|
179
|
-
$or: [
|
|
180
|
-
{ status: 'pending', nextAttemptAt: { $lte: claimedAt } },
|
|
181
|
-
{ status: 'processing', claimedAt: { $lt: new Date(claimedAt.getTime() - STALE_CLAIM_MS) } }
|
|
182
|
-
]
|
|
183
|
-
},
|
|
184
|
-
{ $set: { status: 'processing', claimedAt } },
|
|
185
|
-
{ new: false }
|
|
186
|
-
);
|
|
187
|
-
if (!prior) {
|
|
188
|
-
logger.info('[WorkflowRunner] Claimed elsewhere or not due', { kind: workflow.kind, deferredWorkId });
|
|
189
|
-
return { claimed: false };
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
const reclaimed = prior.status === 'processing';
|
|
193
|
-
const work = { ...prior.toObject(), attempt: prior.attempt + (reclaimed ? 1 : 0) };
|
|
194
|
-
if (reclaimed) await this._write(work, claimedAt, { attempt: work.attempt });
|
|
195
|
-
|
|
196
|
-
const checkpoints = { ...(work.checkpoints || {}) };
|
|
197
|
-
const save = async (patch) => {
|
|
198
|
-
Object.assign(checkpoints, patch);
|
|
199
|
-
const $set = {};
|
|
200
|
-
for (const [key, value] of Object.entries(patch || {})) $set[`checkpoints.${key}`] = value;
|
|
201
|
-
if (Object.keys($set).length) await this._write(work, claimedAt, $set);
|
|
202
|
-
};
|
|
203
|
-
|
|
204
|
-
try {
|
|
205
|
-
await workflow.prepare(work.trigger, checkpoints, save);
|
|
206
|
-
} catch (error) {
|
|
207
|
-
return await this._rescheduleOrAbandon(workflow, work, claimedAt, error);
|
|
208
|
-
}
|
|
209
|
-
await this._write(work, claimedAt, { status: 'done', claimedAt: null });
|
|
210
|
-
return { status: 'done' };
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
async _write(work, claimedAt, $set) {
|
|
214
|
-
const { matchedCount } = await DeferredWork.updateOne({ _id: work._id, claimedAt }, { $set });
|
|
215
|
-
if (!matchedCount) logger.warn('[WorkflowRunner] Claim lost, write discarded', { deferredWorkId: String(work._id) });
|
|
216
|
-
return matchedCount > 0;
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
async _rescheduleOrAbandon(workflow, work, claimedAt, error) {
|
|
220
|
-
const lastError = describeThrown(error);
|
|
221
|
-
const delayMinutes = workflow.retrySchedule[work.attempt];
|
|
222
|
-
if (error?.permanent === true || delayMinutes === undefined) {
|
|
223
|
-
await this._write(work, claimedAt, { status: 'abandoned', lastError, claimedAt: null });
|
|
224
|
-
logger.error('[WorkflowRunner] Abandoned', {
|
|
225
|
-
kind: workflow.kind,
|
|
226
|
-
dedupeKey: work.dedupeKey,
|
|
227
|
-
attempt: work.attempt,
|
|
228
|
-
permanent: error?.permanent === true,
|
|
229
|
-
error: lastError
|
|
230
|
-
});
|
|
231
|
-
return { status: 'abandoned' };
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
const attempt = work.attempt + 1;
|
|
235
|
-
const delayMs = delayMinutes * MINUTE_MS;
|
|
236
|
-
const kept = await this._write(work, claimedAt, {
|
|
237
|
-
status: 'pending',
|
|
238
|
-
attempt,
|
|
239
|
-
nextAttemptAt: new Date(Date.now() + delayMs),
|
|
240
|
-
lastError,
|
|
241
|
-
claimedAt: null
|
|
242
|
-
});
|
|
243
|
-
if (!kept) return { claimed: false };
|
|
244
|
-
|
|
245
|
-
logger.warn('[WorkflowRunner] Rescheduled', {
|
|
246
|
-
kind: workflow.kind,
|
|
247
|
-
dedupeKey: work.dedupeKey,
|
|
248
|
-
attempt,
|
|
249
|
-
delayMinutes,
|
|
250
|
-
error: lastError
|
|
251
|
-
});
|
|
252
|
-
this._arm(workflow, work._id, delayMs);
|
|
253
|
-
return { status: 'pending', attempt };
|
|
254
|
-
}
|
|
255
|
-
}
|
|
256
|
-
|
|
257
|
-
module.exports = { WorkflowRunner };
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
const mongoose = require('mongoose');
|
|
2
|
-
|
|
3
|
-
const STATUS_VALUES = ['pending', 'processing', 'done', 'abandoned'];
|
|
4
|
-
const RETENTION_MS = 24 * 60 * 60 * 1000;
|
|
5
|
-
|
|
6
|
-
const deferredWorkSchema = new mongoose.Schema({
|
|
7
|
-
kind: { type: String, required: true },
|
|
8
|
-
dedupeKey: { type: String, required: true },
|
|
9
|
-
trigger: { type: mongoose.Schema.Types.Mixed, default: null },
|
|
10
|
-
checkpoints: { type: mongoose.Schema.Types.Mixed, default: () => ({}) },
|
|
11
|
-
status: { type: String, enum: STATUS_VALUES, default: 'pending' },
|
|
12
|
-
attempt: { type: Number, default: 0 },
|
|
13
|
-
nextAttemptAt: { type: Date, default: Date.now },
|
|
14
|
-
claimedAt: { type: Date, default: null },
|
|
15
|
-
lastError: { type: String, default: null }
|
|
16
|
-
}, { timestamps: true });
|
|
17
|
-
|
|
18
|
-
deferredWorkSchema.index({ kind: 1, dedupeKey: 1 }, { unique: true, name: 'dedupe_idx' });
|
|
19
|
-
deferredWorkSchema.index({ status: 1, nextAttemptAt: 1 }, { name: 'due_idx' });
|
|
20
|
-
deferredWorkSchema.index({ updatedAt: 1 }, { expireAfterSeconds: RETENTION_MS / 1000, name: 'ttl_idx' });
|
|
21
|
-
|
|
22
|
-
const DeferredWork = mongoose.model('DeferredWork', deferredWorkSchema);
|
|
23
|
-
|
|
24
|
-
module.exports = { DeferredWork, RETENTION_MS };
|