@peopl-health/nexus 5.44.0-dev.6247 → 5.44.0-dev.6258
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +0 -32
- package/lib/clinical/helpers/llmsHelper.js +54 -6
- package/lib/core/NexusMessaging.js +0 -5
- package/lib/core/workflowRunner.js +35 -0
- package/lib/helpers/processHelper.js +15 -2
- package/lib/index.d.ts +4 -6
- package/lib/index.js +2 -0
- package/lib/services/airtableService.js +0 -7
- package/package.json +1 -1
- package/lib/core/WorkflowRunner.js +0 -265
- package/lib/models/deferredWorkModel.js +0 -24
package/README.md
CHANGED
|
@@ -116,38 +116,6 @@ const bus = nexus.getMessaging().getEventBus();
|
|
|
116
116
|
bus.on('message:received', (m) => console.log('rx', m.id));
|
|
117
117
|
```
|
|
118
118
|
|
|
119
|
-
## Workflows (Optional)
|
|
120
|
-
|
|
121
|
-
Register a named unit of work, deduplicated by a key you derive from the trigger.
|
|
122
|
-
|
|
123
|
-
```js
|
|
124
|
-
const runner = nexus.getMessaging().getWorkflowRunner();
|
|
125
|
-
|
|
126
|
-
await runner.register({
|
|
127
|
-
kind: 'triage-projection',
|
|
128
|
-
dedupeKey: (trigger) => String(trigger.submissionId),
|
|
129
|
-
prepare: async (trigger, checkpoints, save) => { /* ... */ },
|
|
130
|
-
retrySchedule: [5, 10, 20, 40], // minutes; omit for fire-and-forget
|
|
131
|
-
});
|
|
132
|
-
|
|
133
|
-
await runner.enqueue('triage-projection', { submissionId });
|
|
134
|
-
```
|
|
135
|
-
|
|
136
|
-
With `retrySchedule` the trigger and a checkpoint journal are stored in Mongo and the schedule runs
|
|
137
|
-
on an in-process timer — deferred workflows never touch the queue adapter, so they add no Redis load.
|
|
138
|
-
`prepare` resumes from `checkpoints`; call `save(patch)` before each external call so a retry does
|
|
139
|
-
not repeat completed work. Throw an error carrying `permanent: true` to give up immediately instead
|
|
140
|
-
of walking the schedule; the Airtable helpers tag their own permanent 4xx errors that way, in
|
|
141
|
-
`withAirtableRetry`, which all five of them route through. They rethrow on failure too, so an
|
|
142
|
-
Airtable outage reaches `prepare` as an exception and walks the retry schedule on its own rather
|
|
143
|
-
than reading as success.
|
|
144
|
-
|
|
145
|
-
Recovery is automatic: registering sweeps for work left behind by a previous process, and the runner
|
|
146
|
-
keeps sweeping so work whose worker died mid-`prepare` is reclaimed once its claim goes stale.
|
|
147
|
-
`enqueue` on an abandoned key revives it, keeping its checkpoints. Work is retained for 24 hours
|
|
148
|
-
after its last update and then expires — the dedupe key expires with it, so idempotency has the
|
|
149
|
-
same horizon as retention.
|
|
150
|
-
|
|
151
119
|
## Assistants (Optional)
|
|
152
120
|
|
|
153
121
|
Register assistant classes and (optionally) a custom resolver. OpenAI is supported via `llm: 'openai'`.
|
|
@@ -13,7 +13,8 @@ const UNSUPPORTED_RESULT = {
|
|
|
13
13
|
medical_analysis: 'NOT_MEDICAL',
|
|
14
14
|
medical_relevance: false,
|
|
15
15
|
has_table: false,
|
|
16
|
-
table_data: null
|
|
16
|
+
table_data: null,
|
|
17
|
+
measurements: null
|
|
17
18
|
};
|
|
18
19
|
|
|
19
20
|
const MIME_MAP = {
|
|
@@ -23,22 +24,35 @@ const MIME_MAP = {
|
|
|
23
24
|
|
|
24
25
|
const SUPPORTED_MIMES = ['image/jpeg', 'image/png', 'image/gif', 'image/webp'];
|
|
25
26
|
|
|
26
|
-
const
|
|
27
|
+
const CONFIDENCE_LEVELS = ['high', 'medium', 'low'];
|
|
28
|
+
|
|
29
|
+
const DISPLAYED_NUMBER = /^-?\d+(?:[.,]\d+)?$/;
|
|
30
|
+
|
|
31
|
+
const ANALYSIS_MODEL = 'claude-sonnet-5';
|
|
27
32
|
const MAX_ANALYSIS_TOKENS = 4096;
|
|
28
33
|
|
|
29
|
-
const ANALYSIS_PROMPT = `Eres un oncólogo clínico con experiencia. Recibirás una imagen que un paciente envió por WhatsApp. Puede ser una receta, un resultado de laboratorio, un estudio de imagen, una nota clínica, la caja de un medicamento, o algo sin relación médica.
|
|
34
|
+
const ANALYSIS_PROMPT = `Eres un oncólogo clínico con experiencia. Recibirás una imagen que un paciente envió por WhatsApp. Puede ser una receta, un resultado de laboratorio, un estudio de imagen, una nota clínica, la pantalla de un aparato de medición casero, la caja de un medicamento, o algo sin relación médica.
|
|
30
35
|
|
|
31
36
|
Responde ÚNICAMENTE con un objeto JSON, sin texto antes ni después y sin bloque de código, con esta forma exacta:
|
|
32
37
|
|
|
33
38
|
{
|
|
34
|
-
"doc_type": "prescription" | "lab_result" | "imaging" | "clinical_note" | "other" | "not_medical",
|
|
39
|
+
"doc_type": "prescription" | "lab_result" | "imaging" | "clinical_note" | "device_reading" | "other" | "not_medical",
|
|
35
40
|
"quality": "clear" | "unclear" | "incomplete",
|
|
36
41
|
"medical_relevance": true | false,
|
|
37
42
|
"transcription": "<todo el texto legible de la imagen, transcrito tal cual, incluida la letra manuscrita; si la imagen no tiene texto legible, describe en su lugar lo que se ve>",
|
|
43
|
+
"measurements": [{ "label": "<la etiqueta impresa junto al número: SYS, DIA, PUL, mg/dL, SpO2, °C...>", "value": "<el número tal cual>", "unit": "<la unidad impresa, o null>", "confidence": "high" | "medium" | "low" }] | null,
|
|
38
44
|
"table_md": "<la tabla en markdown>" | null,
|
|
39
45
|
"reading": "<tu lectura clínica, en el formato que corresponde al doc_type>"
|
|
40
46
|
}
|
|
41
47
|
|
|
48
|
+
Usa "doc_type": "device_reading" cuando la imagen sea la pantalla de un aparato de medición casero (tensiómetro, glucómetro, oxímetro, termómetro, báscula).
|
|
49
|
+
|
|
50
|
+
Reglas para "measurements": inclúyelo solo cuando doc_type sea "device_reading"; en cualquier otro caso usa null.
|
|
51
|
+
- Es una pantalla LCD de segmentos: los segmentos apagados se ven como sombras grises tenues. Cuenta únicamente los segmentos oscuros y bien contrastados.
|
|
52
|
+
- Incluye una entrada por cada número que tenga una etiqueta impresa al lado. Copia la etiqueta tal como aparece en el aparato.
|
|
53
|
+
- NUNCA conviertas en medición la hora, la fecha, el número de memoria ni el índice de usuario. Esos datos van en "transcription", no aquí.
|
|
54
|
+
- Si no estás seguro de algún dígito, marca esa entrada con "confidence": "low". No adivines.
|
|
55
|
+
|
|
42
56
|
Reglas para "table_md": inclúyela solo si la imagen contiene una verdadera tabla de filas y columnas. Transcríbela completa, conservando el orden de las columnas, todas las unidades y todos los rangos de referencia tal como aparecen. Verifica la alineación vertical de cada valor con el nombre de su parámetro antes de escribirlo. Si no hay tabla, usa null; nunca expliques por qué no la hay.
|
|
43
57
|
|
|
44
58
|
Reglas para "reading" según "doc_type":
|
|
@@ -52,6 +66,7 @@ Reglas para "reading" según "doc_type":
|
|
|
52
66
|
- hallazgos clave y anomalías; estructuras normales relevantes; medidas críticas si aplican
|
|
53
67
|
3. Action Items:
|
|
54
68
|
- estudios o proyecciones adicionales recomendadas; limitaciones de la evaluación
|
|
69
|
+
- device_reading: escribe UNA sola línea que nombre el aparato y, si son legibles, la fecha y la hora que muestra la pantalla. NO repitas los valores medidos: esos ya van en "measurements". No los interpretes ni des recomendaciones; la lectura clínica la hace el agente. Ejemplo: "Tensiómetro digital de muñeca Neutek; la pantalla marca el 24/08 a las 5:54 PM."
|
|
55
70
|
- clinical_note / other: resume la información clínica que contiene el documento.
|
|
56
71
|
- not_medical: escribe exactamente NOT_MEDICAL.
|
|
57
72
|
|
|
@@ -109,6 +124,31 @@ const parseAnalysis = (text) => {
|
|
|
109
124
|
}
|
|
110
125
|
};
|
|
111
126
|
|
|
127
|
+
const isText = (value) => typeof value === 'string' && value.trim() !== '';
|
|
128
|
+
|
|
129
|
+
const toMeasurement = (entry) => {
|
|
130
|
+
if (!entry || typeof entry !== 'object') return null;
|
|
131
|
+
const { label, value, unit, confidence } = entry;
|
|
132
|
+
if (!isText(label)) return null;
|
|
133
|
+
if (unit !== null && unit !== undefined && typeof unit !== 'string') return null;
|
|
134
|
+
|
|
135
|
+
const displayed = String(value ?? '').trim();
|
|
136
|
+
if (!DISPLAYED_NUMBER.test(displayed)) return null;
|
|
137
|
+
|
|
138
|
+
return {
|
|
139
|
+
label: label.trim(),
|
|
140
|
+
value: displayed,
|
|
141
|
+
unit: isText(unit) ? unit.trim() : null,
|
|
142
|
+
confidence: CONFIDENCE_LEVELS.includes(confidence) ? confidence : 'low'
|
|
143
|
+
};
|
|
144
|
+
};
|
|
145
|
+
|
|
146
|
+
const toMeasurements = (raw) => {
|
|
147
|
+
if (!Array.isArray(raw) || !raw.length) return null;
|
|
148
|
+
const measurements = raw.map(toMeasurement);
|
|
149
|
+
return measurements.every(Boolean) ? measurements : null;
|
|
150
|
+
};
|
|
151
|
+
|
|
112
152
|
const toResult = (analysis, rawText) => {
|
|
113
153
|
if (!analysis) return { ...UNSUPPORTED_RESULT, description: rawText, medical_analysis: '' };
|
|
114
154
|
|
|
@@ -116,12 +156,20 @@ const toResult = (analysis, rawText) => {
|
|
|
116
156
|
const reading = typeof analysis.reading === 'string' ? analysis.reading.trim() : '';
|
|
117
157
|
const transcription = typeof analysis.transcription === 'string' ? analysis.transcription.trim() : '';
|
|
118
158
|
|
|
159
|
+
const isDevice = analysis.doc_type === 'device_reading';
|
|
160
|
+
const measurements = isDevice ? toMeasurements(analysis.measurements) : null;
|
|
161
|
+
|
|
162
|
+
let medicalAnalysis = reading;
|
|
163
|
+
if (analysis.doc_type === 'not_medical') medicalAnalysis = 'NOT_MEDICAL';
|
|
164
|
+
else if (isDevice && !measurements) medicalAnalysis = 'QUALITY_INSUFFICIENT';
|
|
165
|
+
|
|
119
166
|
return {
|
|
120
167
|
description: transcription,
|
|
121
|
-
medical_analysis:
|
|
168
|
+
medical_analysis: medicalAnalysis,
|
|
122
169
|
medical_relevance: analysis.medical_relevance === true,
|
|
123
170
|
has_table: table !== null,
|
|
124
|
-
table_data: table
|
|
171
|
+
table_data: table,
|
|
172
|
+
measurements
|
|
125
173
|
};
|
|
126
174
|
};
|
|
127
175
|
|
|
@@ -32,7 +32,6 @@ const { isLlmMonitorEnabled } = require('../clinical/flags/llmMonitorConfig');
|
|
|
32
32
|
const { createQueueAdapter } = require('../queue');
|
|
33
33
|
const { ScheduledMessageJob } = require('../jobs/ScheduledMessageJob');
|
|
34
34
|
const { TemplateApprovalJob } = require('../jobs/TemplateApprovalJob');
|
|
35
|
-
const { WorkflowRunner } = require('./WorkflowRunner');
|
|
36
35
|
|
|
37
36
|
const { PhiProcessor } = require('./PhiProcessor');
|
|
38
37
|
|
|
@@ -117,8 +116,6 @@ class NexusMessaging {
|
|
|
117
116
|
queueAdapter: this.queueAdapter,
|
|
118
117
|
});
|
|
119
118
|
|
|
120
|
-
this.workflowRunner = new WorkflowRunner({ queueAdapter: this.queueAdapter });
|
|
121
|
-
|
|
122
119
|
this.phiProcessor = new PhiProcessor({
|
|
123
120
|
encode: config.phi?.encode || false,
|
|
124
121
|
ner: config.phi?.ner || null,
|
|
@@ -301,7 +298,6 @@ class NexusMessaging {
|
|
|
301
298
|
getAssistantProcessor() { return this.assistantProcessor; }
|
|
302
299
|
getLlmMonitor() { return this.llmMonitor; }
|
|
303
300
|
getQueueAdapter() { return this.queueAdapter; }
|
|
304
|
-
getWorkflowRunner() { return this.workflowRunner; }
|
|
305
301
|
getPhiProcessor() { return this.phiProcessor; }
|
|
306
302
|
isConnected() { return this.provider?.getConnectionStatus() ?? false; }
|
|
307
303
|
isProcessing(chatId) { return this.batchingManager.isProcessing(chatId); }
|
|
@@ -737,7 +733,6 @@ class NexusMessaging {
|
|
|
737
733
|
async disconnect() {
|
|
738
734
|
this.queueReconciliation?.stop();
|
|
739
735
|
if (this.provider) await this.provider.disconnect();
|
|
740
|
-
if (this.workflowRunner) this.workflowRunner.stop();
|
|
741
736
|
if (this.queueAdapter) await this.queueAdapter.shutdown();
|
|
742
737
|
this.events.removeAllListeners();
|
|
743
738
|
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
function createWorkflowRunner({ queueAdapter } = {}) {
|
|
2
|
+
if (!queueAdapter) throw new Error('createWorkflowRunner requires a queueAdapter');
|
|
3
|
+
const workflows = new Map();
|
|
4
|
+
|
|
5
|
+
async function register(workflow = {}) {
|
|
6
|
+
if (!workflow.kind || typeof workflow.kind !== 'string') throw new Error('workflow requires a kind');
|
|
7
|
+
if (workflow.kind.includes('__')) throw new Error(`workflow '${workflow.kind}' must not contain '__' (reserved as the jobId delimiter)`);
|
|
8
|
+
if (typeof workflow.dedupeKey !== 'function') throw new Error(`workflow '${workflow.kind}' requires a dedupeKey function`);
|
|
9
|
+
if (typeof workflow.prepare !== 'function') throw new Error(`workflow '${workflow.kind}' requires a prepare function`);
|
|
10
|
+
if (workflows.has(workflow.kind)) throw new Error(`workflow '${workflow.kind}' is already registered`);
|
|
11
|
+
workflows.set(workflow.kind, workflow);
|
|
12
|
+
try {
|
|
13
|
+
await queueAdapter.process(workflow.kind, (trigger) => workflow.prepare(trigger));
|
|
14
|
+
} catch (error) {
|
|
15
|
+
workflows.delete(workflow.kind);
|
|
16
|
+
throw error;
|
|
17
|
+
}
|
|
18
|
+
return workflow;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function enqueue(kind, trigger, options = {}) {
|
|
22
|
+
const workflow = workflows.get(kind);
|
|
23
|
+
if (!workflow) throw new Error(`no workflow registered for '${kind}'`);
|
|
24
|
+
const dedupeKey = workflow.dedupeKey(trigger);
|
|
25
|
+
if (dedupeKey == null || (typeof dedupeKey === 'string' && !dedupeKey.trim())) {
|
|
26
|
+
throw new Error(`workflow '${kind}' produced an empty dedupe key`);
|
|
27
|
+
}
|
|
28
|
+
if (typeof dedupeKey !== 'string') throw new Error(`workflow '${kind}' produced a non-string dedupe key`);
|
|
29
|
+
return queueAdapter.enqueue(kind, trigger, { ...options, jobId: `${kind}__${dedupeKey}` });
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
return { register, enqueue };
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
module.exports = { createWorkflowRunner };
|
|
@@ -40,6 +40,16 @@ const processTextMessage = (reply) => {
|
|
|
40
40
|
return [{ type: 'text', text: reply.body }];
|
|
41
41
|
};
|
|
42
42
|
|
|
43
|
+
const renderMeasurements = (measurements) => {
|
|
44
|
+
const values = measurements
|
|
45
|
+
.map(({ label, value, unit }) => [label, value, unit].filter(Boolean).join(' '))
|
|
46
|
+
.join(' - ');
|
|
47
|
+
const unsure = measurements.filter(({ confidence }) => confidence !== 'high').map(({ label }) => label);
|
|
48
|
+
const line = `Valores leídos en pantalla: ${values}.`;
|
|
49
|
+
if (!unsure.length) return line;
|
|
50
|
+
return `${line}\nLectura dudosa en ${unsure.join(', ')}: pídele al paciente que confirme ese número antes de darlo por bueno.`;
|
|
51
|
+
};
|
|
52
|
+
|
|
43
53
|
const processImageFileCore = async (fileName, reply) => {
|
|
44
54
|
const messagesChat = [];
|
|
45
55
|
const timings = { analysis_ms: 0, url_generation_ms: 0 };
|
|
@@ -62,6 +72,7 @@ const processImageFileCore = async (fileName, reply) => {
|
|
|
62
72
|
timings.analysis_ms = analysisDuration;
|
|
63
73
|
|
|
64
74
|
const invalidAnalysis = ['NOT_MEDICAL', 'QUALITY_INSUFFICIENT'];
|
|
75
|
+
const unusableAnalysis = invalidAnalysis.some(tag => imageAnalysis?.medical_analysis?.includes(tag));
|
|
65
76
|
|
|
66
77
|
if (imageAnalysis?.medical_relevance && !isSticker) {
|
|
67
78
|
const { result: presignedUrl, duration: urlDuration } = await withTracing(
|
|
@@ -74,11 +85,13 @@ const processImageFileCore = async (fileName, reply) => {
|
|
|
74
85
|
}
|
|
75
86
|
|
|
76
87
|
const parts = [];
|
|
88
|
+
if (!isSticker && !unusableAnalysis && imageAnalysis?.measurements?.length) {
|
|
89
|
+
parts.push(renderMeasurements(imageAnalysis.measurements));
|
|
90
|
+
}
|
|
77
91
|
if (!isSticker && imageAnalysis?.has_table && imageAnalysis.table_data) {
|
|
78
92
|
parts.push(imageAnalysis.table_data);
|
|
79
93
|
}
|
|
80
|
-
if (!isSticker && imageAnalysis?.medical_analysis &&
|
|
81
|
-
!invalidAnalysis.some(tag => imageAnalysis.medical_analysis.includes(tag))) {
|
|
94
|
+
if (!isSticker && imageAnalysis?.medical_analysis && !unusableAnalysis) {
|
|
82
95
|
parts.push(imageAnalysis.medical_analysis);
|
|
83
96
|
}
|
|
84
97
|
|
package/lib/index.d.ts
CHANGED
|
@@ -242,7 +242,6 @@ declare module '@peopl-health/nexus' {
|
|
|
242
242
|
getAssistantProcessor(): AssistantProcessor;
|
|
243
243
|
getLlmMonitor(): { start(options?: { scheduleDaily?: boolean }): Promise<{ enabled: boolean }> } | null;
|
|
244
244
|
initializeLlmMonitor(): Promise<{ enabled: boolean; error?: string } | undefined>;
|
|
245
|
-
getWorkflowRunner(): WorkflowRunner;
|
|
246
245
|
processInstruction(code: string, instruction: string, role?: string, options?: { triggeredBy?: string }): Promise<void>;
|
|
247
246
|
processSystemMessage(code: string, messages: string | string[], role?: string, options?: { triggeredBy?: string; reply?: boolean }): Promise<void>;
|
|
248
247
|
isConnected(): boolean;
|
|
@@ -455,17 +454,16 @@ declare module '@peopl-health/nexus' {
|
|
|
455
454
|
export interface Workflow {
|
|
456
455
|
kind: string;
|
|
457
456
|
dedupeKey: (trigger: any) => string;
|
|
458
|
-
prepare: (trigger: any
|
|
459
|
-
retrySchedule?: number[];
|
|
457
|
+
prepare: (trigger: any) => Promise<any>;
|
|
460
458
|
}
|
|
461
459
|
|
|
462
460
|
export interface WorkflowRunner {
|
|
463
461
|
register(workflow: Workflow): Promise<Workflow>;
|
|
464
|
-
enqueue(kind: string, trigger: any, options?: QueueJobOptions): Promise<string
|
|
465
|
-
sweep(only?: string[]): Promise<string[]>;
|
|
466
|
-
stop(): void;
|
|
462
|
+
enqueue(kind: string, trigger: any, options?: QueueJobOptions): Promise<string>;
|
|
467
463
|
}
|
|
468
464
|
|
|
465
|
+
export function createWorkflowRunner(options: { queueAdapter: QueueAdapter }): WorkflowRunner;
|
|
466
|
+
|
|
469
467
|
// Memory System
|
|
470
468
|
export interface PatientMemoryDocument {
|
|
471
469
|
_id: any;
|
package/lib/index.js
CHANGED
|
@@ -27,6 +27,7 @@ const { BaileysProvider } = require('./adapters/BaileysProvider');
|
|
|
27
27
|
const { setPreprocessingHandler, hasPreprocessingHandler, invokePreprocessingHandler } = require('./services/preprocessingService');
|
|
28
28
|
const { requestIdMiddleware, getRequestId } = require('./middleware/requestId');
|
|
29
29
|
const { QueueAdapter, LocalQueueAdapter, RedisQueueAdapter, createQueueAdapter, registerQueueAdapter } = require('./queue');
|
|
30
|
+
const { createWorkflowRunner } = require('./core/workflowRunner');
|
|
30
31
|
const routes = require('./routes');
|
|
31
32
|
const { resetAll } = require('./config/lifecycle');
|
|
32
33
|
const { EvalProvider } = require('./eval/EvalProvider');
|
|
@@ -220,6 +221,7 @@ class Nexus {
|
|
|
220
221
|
}
|
|
221
222
|
|
|
222
223
|
module.exports = {
|
|
224
|
+
createWorkflowRunner,
|
|
223
225
|
Nexus,
|
|
224
226
|
TwilioProvider,
|
|
225
227
|
BaileysProvider,
|
|
@@ -11,11 +11,6 @@ const TRANSIENT_ERROR_CODES = ['ETIMEDOUT', 'ECONNRESET', 'ECONNREFUSED', 'ENOTF
|
|
|
11
11
|
const MAX_ATTEMPTS = 3;
|
|
12
12
|
const RETRY_BASE_DELAY_MS = 1000;
|
|
13
13
|
|
|
14
|
-
function isPermanentAirtableError(error) {
|
|
15
|
-
const status = error?.statusCode;
|
|
16
|
-
return typeof status === 'number' && status >= 400 && status < 500 && status !== 429;
|
|
17
|
-
}
|
|
18
|
-
|
|
19
14
|
let isEvalMode = false;
|
|
20
15
|
|
|
21
16
|
function setEvalMode(enabled) {
|
|
@@ -43,7 +38,6 @@ async function withAirtableRetry(operation, label, { retryNetworkErrors = true }
|
|
|
43
38
|
const isTransient = TRANSIENT_STATUS_CODES.includes(error.statusCode)
|
|
44
39
|
|| (retryNetworkErrors && isTransientNetworkError(error));
|
|
45
40
|
if (!isTransient || attempt === MAX_ATTEMPTS) {
|
|
46
|
-
if (isPermanentAirtableError(error)) error.permanent = true;
|
|
47
41
|
throw error;
|
|
48
42
|
}
|
|
49
43
|
const baseDelay = error.statusCode === RATE_LIMITED_STATUS ? RATE_LIMIT_DELAY_MS : RETRY_BASE_DELAY_MS;
|
|
@@ -175,7 +169,6 @@ async function addLinkedRecord(baseID, targetTable, fields, linkConfig, context
|
|
|
175
169
|
|
|
176
170
|
module.exports = {
|
|
177
171
|
setEvalMode,
|
|
178
|
-
isPermanentAirtableError,
|
|
179
172
|
addRecord,
|
|
180
173
|
getRecords,
|
|
181
174
|
getRecordByFilter,
|
package/package.json
CHANGED
|
@@ -1,265 +0,0 @@
|
|
|
1
|
-
const { describeThrown } = require('../utils/errorUtils');
|
|
2
|
-
const { logger } = require('../utils/logger');
|
|
3
|
-
|
|
4
|
-
const { DeferredWork, RETENTION_MS } = require('../models/deferredWorkModel');
|
|
5
|
-
|
|
6
|
-
const STALE_CLAIM_MS = 5 * 60 * 1000;
|
|
7
|
-
const MINUTE_MS = 60 * 1000;
|
|
8
|
-
const INDEX_TIMEOUT_MS = 1000;
|
|
9
|
-
const SWEEP_INTERVAL_MS = 60 * 1000;
|
|
10
|
-
|
|
11
|
-
let indexesReady = null;
|
|
12
|
-
|
|
13
|
-
function ensureIndexes() {
|
|
14
|
-
if (!indexesReady) {
|
|
15
|
-
indexesReady = DeferredWork.init().catch((error) => {
|
|
16
|
-
indexesReady = null;
|
|
17
|
-
logger.error('[WorkflowRunner] Index build failed, dedupe and retention are not guaranteed', { error: error.message });
|
|
18
|
-
});
|
|
19
|
-
}
|
|
20
|
-
return indexesReady;
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
function withIndexTimeout(promise) {
|
|
24
|
-
let timer;
|
|
25
|
-
const timeout = new Promise((resolve) => {
|
|
26
|
-
timer = setTimeout(resolve, INDEX_TIMEOUT_MS);
|
|
27
|
-
timer.unref();
|
|
28
|
-
});
|
|
29
|
-
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer));
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
class WorkflowRunner {
|
|
33
|
-
constructor({ queueAdapter, sweepIntervalMs = SWEEP_INTERVAL_MS } = {}) {
|
|
34
|
-
if (!queueAdapter) throw new Error('WorkflowRunner requires a queueAdapter');
|
|
35
|
-
this.queueAdapter = queueAdapter;
|
|
36
|
-
this.sweepIntervalMs = sweepIntervalMs;
|
|
37
|
-
this.sweepTimer = null;
|
|
38
|
-
this.timers = new Map();
|
|
39
|
-
this.workflows = new Map();
|
|
40
|
-
this.stopped = false;
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
async register(workflow = {}) {
|
|
44
|
-
if (!workflow.kind || typeof workflow.kind !== 'string') throw new Error('workflow requires a kind');
|
|
45
|
-
if (workflow.kind.includes('__')) throw new Error(`workflow '${workflow.kind}' must not contain '__' (reserved as the jobId delimiter)`);
|
|
46
|
-
if (typeof workflow.dedupeKey !== 'function') throw new Error(`workflow '${workflow.kind}' requires a dedupeKey function`);
|
|
47
|
-
if (typeof workflow.prepare !== 'function') throw new Error(`workflow '${workflow.kind}' requires a prepare function`);
|
|
48
|
-
if (this.workflows.has(workflow.kind)) throw new Error(`workflow '${workflow.kind}' is already registered`);
|
|
49
|
-
if (workflow.retrySchedule !== undefined) {
|
|
50
|
-
if (!Array.isArray(workflow.retrySchedule) || !workflow.retrySchedule.length) {
|
|
51
|
-
throw new Error(`workflow '${workflow.kind}' requires a non-empty retrySchedule array`);
|
|
52
|
-
}
|
|
53
|
-
if (workflow.retrySchedule.some((minutes) => !Number.isFinite(minutes) || minutes <= 0)) {
|
|
54
|
-
throw new Error(`workflow '${workflow.kind}' requires retrySchedule entries to be positive minutes`);
|
|
55
|
-
}
|
|
56
|
-
if (workflow.retrySchedule.some((minutes) => minutes * MINUTE_MS >= RETENTION_MS)) {
|
|
57
|
-
throw new Error(`workflow '${workflow.kind}' requires retrySchedule entries under the ${RETENTION_MS / MINUTE_MS}m retention, or the record expires before the attempt fires`);
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
this.workflows.set(workflow.kind, workflow);
|
|
62
|
-
if (workflow.retrySchedule) {
|
|
63
|
-
await this.sweep([workflow.kind]).catch((error) => {
|
|
64
|
-
logger.error('[WorkflowRunner] Sweep on register failed', { kind: workflow.kind, error: error.message });
|
|
65
|
-
});
|
|
66
|
-
this._startSweeping();
|
|
67
|
-
return workflow;
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
try {
|
|
71
|
-
await this.queueAdapter.process(workflow.kind, (trigger) => workflow.prepare(trigger));
|
|
72
|
-
} catch (error) {
|
|
73
|
-
this.workflows.delete(workflow.kind);
|
|
74
|
-
throw error;
|
|
75
|
-
}
|
|
76
|
-
return workflow;
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
enqueue(kind, trigger, options = {}) {
|
|
80
|
-
const workflow = this.workflows.get(kind);
|
|
81
|
-
if (!workflow) throw new Error(`no workflow registered for '${kind}'`);
|
|
82
|
-
const dedupeKey = workflow.dedupeKey(trigger);
|
|
83
|
-
if (dedupeKey == null || (typeof dedupeKey === 'string' && !dedupeKey.trim())) {
|
|
84
|
-
throw new Error(`workflow '${kind}' produced an empty dedupe key`);
|
|
85
|
-
}
|
|
86
|
-
if (typeof dedupeKey !== 'string') throw new Error(`workflow '${kind}' produced a non-string dedupe key`);
|
|
87
|
-
if (!workflow.retrySchedule) {
|
|
88
|
-
return this.queueAdapter.enqueue(kind, trigger, { ...options, jobId: `${kind}__${dedupeKey}` });
|
|
89
|
-
}
|
|
90
|
-
return this._armDeferred(workflow, trigger, dedupeKey);
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
async sweep(only = null) {
|
|
94
|
-
const deferredKinds = [...this.workflows.values()].filter((w) => w.retrySchedule).map((w) => w.kind);
|
|
95
|
-
const kinds = only ? deferredKinds.filter((kind) => only.includes(kind)) : deferredKinds;
|
|
96
|
-
if (!kinds.length) return [];
|
|
97
|
-
|
|
98
|
-
const orphaned = await DeferredWork.find({
|
|
99
|
-
kind: { $in: kinds },
|
|
100
|
-
$or: [
|
|
101
|
-
{ status: 'pending' },
|
|
102
|
-
{ status: 'processing', claimedAt: { $lt: new Date(Date.now() - STALE_CLAIM_MS) } }
|
|
103
|
-
]
|
|
104
|
-
});
|
|
105
|
-
|
|
106
|
-
const rearmed = orphaned.map((work) => this._arm(
|
|
107
|
-
this.workflows.get(work.kind),
|
|
108
|
-
work._id,
|
|
109
|
-
this._delayUntilDue(work),
|
|
110
|
-
));
|
|
111
|
-
if (rearmed.length) logger.info('[WorkflowRunner] Swept', { kinds, rearmed: rearmed.length });
|
|
112
|
-
return rearmed;
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
_startSweeping() {
|
|
116
|
-
if (this.stopped || this.sweepTimer || !this.sweepIntervalMs) return;
|
|
117
|
-
this.sweepTimer = setInterval(() => {
|
|
118
|
-
this.sweep().catch((error) => logger.error('[WorkflowRunner] Periodic sweep failed', { error: error.message }));
|
|
119
|
-
}, this.sweepIntervalMs);
|
|
120
|
-
this.sweepTimer.unref();
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
stop() {
|
|
124
|
-
this.stopped = true;
|
|
125
|
-
if (this.sweepTimer) clearInterval(this.sweepTimer);
|
|
126
|
-
this.sweepTimer = null;
|
|
127
|
-
for (const timer of this.timers.values()) clearTimeout(timer);
|
|
128
|
-
this.timers.clear();
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
async _armDeferred(workflow, trigger, dedupeKey) {
|
|
132
|
-
const { kind } = workflow;
|
|
133
|
-
await withIndexTimeout(ensureIndexes());
|
|
134
|
-
const work = await DeferredWork.findOne({ kind, dedupeKey })
|
|
135
|
-
|| await DeferredWork.create({ kind, dedupeKey, trigger }).catch(async (error) => {
|
|
136
|
-
if (error.code !== 11000) throw error;
|
|
137
|
-
return await DeferredWork.findOne({ kind, dedupeKey });
|
|
138
|
-
});
|
|
139
|
-
if (work.status === 'abandoned') {
|
|
140
|
-
const { modifiedCount } = await DeferredWork.updateOne(
|
|
141
|
-
{ _id: work._id, status: 'abandoned' },
|
|
142
|
-
{ $set: { status: 'pending', attempt: 0, nextAttemptAt: new Date(), lastError: null, claimedAt: null } }
|
|
143
|
-
);
|
|
144
|
-
if (!modifiedCount) return null;
|
|
145
|
-
logger.info('[WorkflowRunner] Re-arming abandoned work', { kind, dedupeKey });
|
|
146
|
-
return this._arm(workflow, work._id, 0);
|
|
147
|
-
}
|
|
148
|
-
if (work.status !== 'pending') {
|
|
149
|
-
logger.info('[WorkflowRunner] Skipping, work is in flight or complete', { kind, dedupeKey, status: work.status });
|
|
150
|
-
return null;
|
|
151
|
-
}
|
|
152
|
-
return this._arm(workflow, work._id, this._delayUntilDue(work));
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
_delayUntilDue(work) {
|
|
156
|
-
return Math.max(0, new Date(work.nextAttemptAt).getTime() - Date.now());
|
|
157
|
-
}
|
|
158
|
-
|
|
159
|
-
_arm(workflow, workId, delayMs) {
|
|
160
|
-
const id = String(workId);
|
|
161
|
-
// An attempt still in flight when stop() lands must not re-arm after shutdown. The
|
|
162
|
-
// record stays pending in Mongo, so the next process recovers it on its sweep.
|
|
163
|
-
if (this.stopped) return id;
|
|
164
|
-
clearTimeout(this.timers.get(id));
|
|
165
|
-
const timer = setTimeout(() => {
|
|
166
|
-
this.timers.delete(id);
|
|
167
|
-
this._runDeferred(workflow, id).catch((error) => {
|
|
168
|
-
logger.error('[WorkflowRunner] Deferred run failed', { kind: workflow.kind, deferredWorkId: id, error: error.message });
|
|
169
|
-
});
|
|
170
|
-
}, delayMs);
|
|
171
|
-
timer.unref();
|
|
172
|
-
this.timers.set(id, timer);
|
|
173
|
-
return id;
|
|
174
|
-
}
|
|
175
|
-
|
|
176
|
-
async _runDeferred(workflow, deferredWorkId) {
|
|
177
|
-
const claimedAt = new Date();
|
|
178
|
-
const prior = await DeferredWork.findOneAndUpdate(
|
|
179
|
-
{
|
|
180
|
-
_id: deferredWorkId,
|
|
181
|
-
$or: [
|
|
182
|
-
{ status: 'pending', nextAttemptAt: { $lte: claimedAt } },
|
|
183
|
-
{ status: 'processing', claimedAt: { $lt: new Date(claimedAt.getTime() - STALE_CLAIM_MS) } }
|
|
184
|
-
]
|
|
185
|
-
},
|
|
186
|
-
{ $set: { status: 'processing', claimedAt } },
|
|
187
|
-
{ new: false }
|
|
188
|
-
);
|
|
189
|
-
if (!prior) {
|
|
190
|
-
logger.info('[WorkflowRunner] Claimed elsewhere or not due', { kind: workflow.kind, deferredWorkId });
|
|
191
|
-
return { claimed: false };
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
const reclaimed = prior.status === 'processing';
|
|
195
|
-
const work = { ...prior.toObject(), attempt: prior.attempt + (reclaimed ? 1 : 0) };
|
|
196
|
-
if (reclaimed) await this._write(work, claimedAt, { attempt: work.attempt });
|
|
197
|
-
|
|
198
|
-
const checkpoints = { ...(work.checkpoints || {}) };
|
|
199
|
-
const save = async (patch) => {
|
|
200
|
-
Object.assign(checkpoints, patch);
|
|
201
|
-
const $set = {};
|
|
202
|
-
for (const [key, value] of Object.entries(patch || {})) $set[`checkpoints.${key}`] = value;
|
|
203
|
-
if (Object.keys($set).length) await this._write(work, claimedAt, $set);
|
|
204
|
-
};
|
|
205
|
-
|
|
206
|
-
try {
|
|
207
|
-
await workflow.prepare(work.trigger, checkpoints, save);
|
|
208
|
-
} catch (error) {
|
|
209
|
-
return await this._rescheduleOrAbandon(workflow, work, claimedAt, error);
|
|
210
|
-
}
|
|
211
|
-
await this._write(work, claimedAt, { status: 'done', claimedAt: null });
|
|
212
|
-
return { status: 'done' };
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
// matchedCount, not modifiedCount: the fence is "do we still hold the claim", and
|
|
216
|
-
// modifiedCount is also 0 for a write that matched but set an identical value - a
|
|
217
|
-
// save() rewriting an unchanged checkpoint is not a lost claim.
|
|
218
|
-
async _write(work, claimedAt, $set) {
|
|
219
|
-
const { matchedCount } = await DeferredWork.updateOne({ _id: work._id, claimedAt }, { $set });
|
|
220
|
-
if (!matchedCount) logger.warn('[WorkflowRunner] Claim lost, write discarded', { deferredWorkId: String(work._id) });
|
|
221
|
-
return matchedCount > 0;
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
async _rescheduleOrAbandon(workflow, work, claimedAt, error) {
|
|
225
|
-
// prepare() is consumer code and can throw anything, including null. Dereferencing
|
|
226
|
-
// it here used to raise a TypeError out of this handler, so neither branch ran and
|
|
227
|
-
// the record stayed `processing` until stale recovery hit the same failure again.
|
|
228
|
-
const lastError = describeThrown(error);
|
|
229
|
-
const delayMinutes = workflow.retrySchedule[work.attempt];
|
|
230
|
-
if (error?.permanent === true || delayMinutes === undefined) {
|
|
231
|
-
await this._write(work, claimedAt, { status: 'abandoned', lastError, claimedAt: null });
|
|
232
|
-
logger.error('[WorkflowRunner] Abandoned', {
|
|
233
|
-
kind: workflow.kind,
|
|
234
|
-
dedupeKey: work.dedupeKey,
|
|
235
|
-
attempt: work.attempt,
|
|
236
|
-
permanent: error?.permanent === true,
|
|
237
|
-
error: lastError
|
|
238
|
-
});
|
|
239
|
-
return { status: 'abandoned' };
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
const attempt = work.attempt + 1;
|
|
243
|
-
const delayMs = delayMinutes * MINUTE_MS;
|
|
244
|
-
const kept = await this._write(work, claimedAt, {
|
|
245
|
-
status: 'pending',
|
|
246
|
-
attempt,
|
|
247
|
-
nextAttemptAt: new Date(Date.now() + delayMs),
|
|
248
|
-
lastError,
|
|
249
|
-
claimedAt: null
|
|
250
|
-
});
|
|
251
|
-
if (!kept) return { claimed: false };
|
|
252
|
-
|
|
253
|
-
logger.warn('[WorkflowRunner] Rescheduled', {
|
|
254
|
-
kind: workflow.kind,
|
|
255
|
-
dedupeKey: work.dedupeKey,
|
|
256
|
-
attempt,
|
|
257
|
-
delayMinutes,
|
|
258
|
-
error: lastError
|
|
259
|
-
});
|
|
260
|
-
this._arm(workflow, work._id, delayMs);
|
|
261
|
-
return { status: 'pending', attempt };
|
|
262
|
-
}
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
module.exports = { WorkflowRunner };
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
const mongoose = require('mongoose');
|
|
2
|
-
|
|
3
|
-
const STATUS_VALUES = ['pending', 'processing', 'done', 'abandoned'];
|
|
4
|
-
const RETENTION_MS = 24 * 60 * 60 * 1000;
|
|
5
|
-
|
|
6
|
-
const deferredWorkSchema = new mongoose.Schema({
|
|
7
|
-
kind: { type: String, required: true },
|
|
8
|
-
dedupeKey: { type: String, required: true },
|
|
9
|
-
trigger: { type: mongoose.Schema.Types.Mixed, default: null },
|
|
10
|
-
checkpoints: { type: mongoose.Schema.Types.Mixed, default: () => ({}) },
|
|
11
|
-
status: { type: String, enum: STATUS_VALUES, default: 'pending' },
|
|
12
|
-
attempt: { type: Number, default: 0 },
|
|
13
|
-
nextAttemptAt: { type: Date, default: Date.now },
|
|
14
|
-
claimedAt: { type: Date, default: null },
|
|
15
|
-
lastError: { type: String, default: null }
|
|
16
|
-
}, { timestamps: true });
|
|
17
|
-
|
|
18
|
-
deferredWorkSchema.index({ kind: 1, dedupeKey: 1 }, { unique: true, name: 'dedupe_idx' });
|
|
19
|
-
deferredWorkSchema.index({ status: 1, nextAttemptAt: 1 }, { name: 'due_idx' });
|
|
20
|
-
deferredWorkSchema.index({ updatedAt: 1 }, { expireAfterSeconds: RETENTION_MS / 1000, name: 'ttl_idx' });
|
|
21
|
-
|
|
22
|
-
const DeferredWork = mongoose.model('DeferredWork', deferredWorkSchema);
|
|
23
|
-
|
|
24
|
-
module.exports = { DeferredWork, RETENTION_MS };
|