@wildo-ai/saas-minions-lib 1.1.7 → 1.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esm/base-classes/services/services/minion-scoped-services-registry-handler.backend.service.d.ts.map +1 -1
- package/dist/esm/base-classes/services/services/minion-scoped-services-registry-handler.backend.service.js.map +1 -1
- package/dist/esm/index.d.ts.map +1 -1
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/inject/models/container.d.ts.map +1 -1
- package/dist/esm/inject/models/container.js.map +1 -1
- package/dist/esm/inject/models/service-types.d.ts.map +1 -1
- package/dist/esm/inject/models/service-types.js.map +1 -1
- package/dist/esm/runtime/queue/minion-tick-consumer.backend.service.d.ts +5 -1
- package/dist/esm/runtime/queue/minion-tick-consumer.backend.service.d.ts.map +1 -1
- package/dist/esm/runtime/queue/minion-tick-consumer.backend.service.js +24 -9
- package/dist/esm/runtime/queue/minion-tick-consumer.backend.service.js.map +1 -1
- package/dist/esm/runtime/services/define-minion-init.d.ts.map +1 -1
- package/dist/esm/runtime/services/define-minion-init.js.map +1 -1
- package/dist/esm/runtime/services/own-schedule-reader.backend.service.d.ts.map +1 -1
- package/dist/esm/runtime/services/own-schedule-reader.backend.service.js.map +1 -1
- package/dist/esm/runtime/services/start-minion-application.backend.service.d.ts.map +1 -1
- package/dist/esm/runtime/services/start-minion-application.backend.service.js.map +1 -1
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/package.json +4 -4
- package/src/runtime/queue/__tests__/minion-tick-consumer.backend.service.test.ts +63 -14
- package/src/runtime/queue/minion-tick-consumer.backend.service.ts +23 -8
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@wildo-ai/saas-minions-lib",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.9",
|
|
4
4
|
"description": "Node minion runtime host layer for Wildo SaaS applications",
|
|
5
5
|
"license": "SEE LICENSE IN LICENSE",
|
|
6
6
|
"author": "Wildo AI",
|
|
@@ -30,9 +30,9 @@
|
|
|
30
30
|
"test:watch": "vitest"
|
|
31
31
|
},
|
|
32
32
|
"dependencies": {
|
|
33
|
-
"@wildo-ai/platform-config-lib": "1.1.
|
|
34
|
-
"@wildo-ai/saas-backend-lib": "1.1.
|
|
35
|
-
"@wildo-ai/saas-models": "1.1.
|
|
33
|
+
"@wildo-ai/platform-config-lib": "1.1.9",
|
|
34
|
+
"@wildo-ai/saas-backend-lib": "1.1.9",
|
|
35
|
+
"@wildo-ai/saas-models": "1.1.9",
|
|
36
36
|
"reflect-metadata": "0.2.2"
|
|
37
37
|
},
|
|
38
38
|
"devDependencies": {
|
|
@@ -5,23 +5,31 @@ import type {
|
|
|
5
5
|
JobPayloadStorageBackendService, JobStatusTrackingBackendService, LoggerService, WorkerShutdownManagerBackendService,
|
|
6
6
|
RabbitMQAdapterBackendService, ScheduledExecutionResolver, ScheduledMessageVerifierBackendService,
|
|
7
7
|
} from '@wildo-ai/saas-backend-lib';
|
|
8
|
-
import { JobPayloadSchema, JobType, ScheduledReplaySafety, ScheduledReviewOutcome, type JobPayload } from '@wildo-ai/saas-models';
|
|
8
|
+
import { JobPayloadSchema, JobType, ScheduledDeliveryLateness, ScheduledReplaySafety, ScheduledReviewOutcome, type JobPayload } from '@wildo-ai/saas-models';
|
|
9
|
+
import { ScheduledDeliveryVerdict } from '@wildo-ai/saas-backend-lib';
|
|
9
10
|
import { MinionTickConsumerBackendService } from '../minion-tick-consumer.backend.service';
|
|
10
11
|
|
|
11
12
|
// This suite pins the minion authentication/lifecycle adapter. Durable concurrency
|
|
12
13
|
// and recovery policy are exercised by the coordinator's real-store suites.
|
|
13
|
-
|
|
14
|
+
// The consumer ADMITS (durable receipt), acknowledges, then LAUNCHES; a late delivery is RECORDED instead.
|
|
15
|
+
const coordinator = vi.hoisted(() => ({ admit: vi.fn(), launch: vi.fn(), recordLateDelivery: vi.fn(), start: vi.fn(), stop: vi.fn(),
|
|
16
|
+
resolveReview: vi.fn(), construct: vi.fn() }));
|
|
14
17
|
vi.mock('@wildo-ai/saas-backend-lib', async (importOriginal) => ({
|
|
15
18
|
...await importOriginal<Record<string, unknown>>(),
|
|
16
19
|
ScheduledExecutionRunner: class {
|
|
17
20
|
constructor(...args: unknown[]) { coordinator.construct(...args); }
|
|
18
|
-
|
|
21
|
+
admit = coordinator.admit;
|
|
22
|
+
launch = coordinator.launch;
|
|
23
|
+
recordLateDelivery = coordinator.recordLateDelivery;
|
|
19
24
|
start = coordinator.start;
|
|
20
25
|
stop = coordinator.stop;
|
|
21
26
|
resolveReview = coordinator.resolveReview;
|
|
22
27
|
},
|
|
23
28
|
}));
|
|
24
29
|
|
|
30
|
+
/** The verifier's answer for an authentic, on-time tick (`ScheduledDeliveryVerdict.ADMISSIBLE`). */
|
|
31
|
+
const onTime = (value: unknown) => ({ verdict: ScheduledDeliveryVerdict.ADMISSIBLE, value });
|
|
32
|
+
|
|
25
33
|
function fixture() {
|
|
26
34
|
let deliver: ((message: ConsumeMessage, channel: Channel) => Promise<void>) | undefined;
|
|
27
35
|
const adapter = {
|
|
@@ -35,7 +43,7 @@ function fixture() {
|
|
|
35
43
|
jobType: JobType.MINION_TICK, iat: 1750000000, exp: 1750000300,
|
|
36
44
|
iss: 'PLATFORM_CRONTABS_BATCHES_MANAGER', aud: 'SCHEDULED_JOB', inputDataJson: '{"target":"one"}',
|
|
37
45
|
};
|
|
38
|
-
const verifier = { verifyAndExtractMinionTick: vi.fn().mockResolvedValue(tick) };
|
|
46
|
+
const verifier = { verifyAndExtractMinionTick: vi.fn().mockResolvedValue(onTime(tick)) };
|
|
39
47
|
const logger = { debugByType: () => vi.fn(), setDebugTypeLogging: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() };
|
|
40
48
|
const shutdown = { jobStarted: vi.fn(), jobCompleted: vi.fn(), registerStopHandler: vi.fn() };
|
|
41
49
|
const service = new MinionTickConsumerBackendService(
|
|
@@ -57,7 +65,10 @@ function fixture() {
|
|
|
57
65
|
};
|
|
58
66
|
}
|
|
59
67
|
|
|
60
|
-
beforeEach(() => {
|
|
68
|
+
beforeEach(() => {
|
|
69
|
+
vi.clearAllMocks();
|
|
70
|
+
coordinator.admit.mockImplementation(async (job: JobPayload) => ({ jobId: job.jobId }));
|
|
71
|
+
});
|
|
61
72
|
|
|
62
73
|
describe('minion scheduled coordination boundary', () => {
|
|
63
74
|
it('joins the shared shutdown drain and stops admissions before draining', async () => {
|
|
@@ -79,9 +90,9 @@ describe('minion scheduled coordination boundary', () => {
|
|
|
79
90
|
evidencePolicy: { policyRef: 'tick-v1', retainForMs: 86400000, resultFields: [] } });
|
|
80
91
|
// The runner is started on a QUEUE only; the platform id shapes the queue name and is never handed to the scanner.
|
|
81
92
|
expect(coordinator.start).toHaveBeenCalledWith(expect.stringContaining('worker'), expect.any(Function));
|
|
82
|
-
expect(coordinator.
|
|
93
|
+
expect(coordinator.admit).not.toHaveBeenCalled();
|
|
83
94
|
await f.deliver();
|
|
84
|
-
const [job, queue, resolver] = coordinator.
|
|
95
|
+
const [job, queue, resolver] = coordinator.admit.mock.calls[0] as [JobPayload, string, ScheduledExecutionResolver];
|
|
85
96
|
expect(job).toMatchObject({ jobId: f.tick.jobId, applicationId: 'app-a', minionName: 'worker', inputDataJson: f.tick.inputDataJson });
|
|
86
97
|
expect(job.createdAt).toEqual(new Date(f.tick.iat * 1000));
|
|
87
98
|
expect(job.scheduledAt).toBeUndefined();
|
|
@@ -105,12 +116,12 @@ describe('minion scheduled coordination boundary', () => {
|
|
|
105
116
|
it('preserves the signed occurrence time through durable serialization into execute and reconcile', async () => {
|
|
106
117
|
const f = fixture();
|
|
107
118
|
const scheduledAt = new Date(f.tick.iat * 1000 - 60000);
|
|
108
|
-
f.verifier.verifyAndExtractMinionTick.mockResolvedValue({ ...f.tick, scheduledAt: scheduledAt.toISOString() });
|
|
119
|
+
f.verifier.verifyAndExtractMinionTick.mockResolvedValue(onTime({ ...f.tick, scheduledAt: scheduledAt.toISOString() }));
|
|
109
120
|
const handler = vi.fn();
|
|
110
121
|
const reconcile = vi.fn();
|
|
111
122
|
await f.service.start({ applicationId: 'app-a', minionName: 'worker', handler, reconcile });
|
|
112
123
|
await f.deliver();
|
|
113
|
-
const [job, , resolver] = coordinator.
|
|
124
|
+
const [job, , resolver] = coordinator.admit.mock.calls[0] as [JobPayload, string, ScheduledExecutionResolver];
|
|
114
125
|
expect(job.scheduledAt).toEqual(scheduledAt);
|
|
115
126
|
// Recovery restores JSON through the same schema as ScheduledExecutionRunner.
|
|
116
127
|
const recovered = JobPayloadSchema.parse({ ...JSON.parse(JSON.stringify(job)),
|
|
@@ -144,10 +155,10 @@ describe('minion scheduled coordination boundary', () => {
|
|
|
144
155
|
|
|
145
156
|
it.each(['applicationId', 'minionName'])('rejects a signed wrong %s before durable receipt', async (field) => {
|
|
146
157
|
const f = fixture();
|
|
147
|
-
f.verifier.verifyAndExtractMinionTick.mockResolvedValue({ ...f.tick, [field]: 'other' });
|
|
158
|
+
f.verifier.verifyAndExtractMinionTick.mockResolvedValue(onTime({ ...f.tick, [field]: 'other' }));
|
|
148
159
|
await f.service.start({ applicationId: 'app-a', minionName: 'worker', handler: vi.fn() });
|
|
149
160
|
await f.deliver();
|
|
150
|
-
expect(coordinator.
|
|
161
|
+
expect(coordinator.admit).not.toHaveBeenCalled();
|
|
151
162
|
expect(f.channel.nack).toHaveBeenCalledWith(expect.anything(), false, false);
|
|
152
163
|
});
|
|
153
164
|
|
|
@@ -157,8 +168,8 @@ describe('minion scheduled coordination boundary', () => {
|
|
|
157
168
|
await f.service.start({ applicationId: 'app-a', minionName: 'worker', handler });
|
|
158
169
|
f.verifier.verifyAndExtractMinionTick.mockRejectedValueOnce(new Error('invalid signature'));
|
|
159
170
|
await f.deliver();
|
|
160
|
-
expect(coordinator.
|
|
161
|
-
coordinator.
|
|
171
|
+
expect(coordinator.admit).not.toHaveBeenCalled();
|
|
172
|
+
coordinator.admit.mockRejectedValueOnce(new Error('database unavailable'));
|
|
162
173
|
await f.deliver();
|
|
163
174
|
expect(handler).not.toHaveBeenCalled();
|
|
164
175
|
expect(f.channel.ack).not.toHaveBeenCalled();
|
|
@@ -170,8 +181,46 @@ describe('minion scheduled coordination boundary', () => {
|
|
|
170
181
|
f.channel.ack.mockImplementation(() => { throw new Error('channel closed'); });
|
|
171
182
|
await f.service.start({ applicationId: 'app-a', minionName: 'worker', handler: vi.fn() });
|
|
172
183
|
await f.deliver();
|
|
173
|
-
expect(coordinator.
|
|
184
|
+
expect(coordinator.admit).toHaveBeenCalledTimes(1);
|
|
174
185
|
expect(f.channel.nack).not.toHaveBeenCalled();
|
|
175
186
|
expect(f.logger.error).toHaveBeenCalledWith('Minion tick settlement failed', expect.anything());
|
|
176
187
|
});
|
|
188
|
+
|
|
189
|
+
it('records a LATE tick as a missed occurrence and acknowledges it, never admitting or running it (#2212)', async () => {
|
|
190
|
+
const f = fixture();
|
|
191
|
+
coordinator.recordLateDelivery.mockResolvedValue(true);
|
|
192
|
+
f.verifier.verifyAndExtractMinionTick.mockResolvedValue({ verdict: ScheduledDeliveryVerdict.LATE, value: f.tick,
|
|
193
|
+
lateness: ScheduledDeliveryLateness.TOKEN_EXPIRED });
|
|
194
|
+
const handler = vi.fn();
|
|
195
|
+
await f.service.start({ applicationId: 'app-a', minionName: 'worker', handler });
|
|
196
|
+
await f.deliver();
|
|
197
|
+
expect(coordinator.recordLateDelivery).toHaveBeenCalledWith(expect.objectContaining({ jobId: f.tick.jobId, minionName: 'worker' }),
|
|
198
|
+
expect.stringContaining('worker'), expect.any(Function), ScheduledDeliveryLateness.TOKEN_EXPIRED);
|
|
199
|
+
expect(coordinator.admit).not.toHaveBeenCalled();
|
|
200
|
+
expect(coordinator.launch).not.toHaveBeenCalled();
|
|
201
|
+
expect(handler).not.toHaveBeenCalled();
|
|
202
|
+
expect(f.channel.ack).toHaveBeenCalledOnce();
|
|
203
|
+
expect(f.logger.error).not.toHaveBeenCalled();
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
it('dead-letters a LATE tick the runner could not record, so its copy survives', async () => {
|
|
207
|
+
const f = fixture();
|
|
208
|
+
coordinator.recordLateDelivery.mockResolvedValue(false);
|
|
209
|
+
f.verifier.verifyAndExtractMinionTick.mockResolvedValue({ verdict: ScheduledDeliveryVerdict.LATE, value: f.tick,
|
|
210
|
+
lateness: ScheduledDeliveryLateness.OUTSIDE_ADMISSION_WINDOW });
|
|
211
|
+
await f.service.start({ applicationId: 'app-a', minionName: 'worker', handler: vi.fn() });
|
|
212
|
+
await f.deliver();
|
|
213
|
+
expect(f.channel.ack).not.toHaveBeenCalled();
|
|
214
|
+
expect(f.channel.nack).toHaveBeenCalledWith(expect.anything(), false, false);
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
it('refuses a LATE tick addressed to another application before recording it', async () => {
|
|
218
|
+
const f = fixture();
|
|
219
|
+
f.verifier.verifyAndExtractMinionTick.mockResolvedValue({ verdict: ScheduledDeliveryVerdict.LATE,
|
|
220
|
+
value: { ...f.tick, applicationId: 'other' }, lateness: ScheduledDeliveryLateness.TOKEN_EXPIRED });
|
|
221
|
+
await f.service.start({ applicationId: 'app-a', minionName: 'worker', handler: vi.fn() });
|
|
222
|
+
await f.deliver();
|
|
223
|
+
expect(coordinator.recordLateDelivery).not.toHaveBeenCalled();
|
|
224
|
+
expect(f.channel.nack).toHaveBeenCalledWith(expect.anything(), false, false);
|
|
225
|
+
});
|
|
177
226
|
});
|
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
RabbitMQAdapterBackendService,
|
|
7
7
|
SAAS_SERVICE_TYPES,
|
|
8
8
|
ScheduledMessageVerifierBackendService,
|
|
9
|
+
ScheduledDeliveryVerdict,
|
|
9
10
|
ScheduledExecutionRunner,
|
|
10
11
|
JobStatusTrackingBackendService,
|
|
11
12
|
JobPayloadStorageBackendService,
|
|
@@ -41,7 +42,11 @@ export type MinionTickReconciler = (tick: MinionTickInvocation, attempt: Schedul
|
|
|
41
42
|
* - Verifies every inbound JWT through
|
|
42
43
|
* `ScheduledMessageVerifierBackendService.verifyAndExtractMinionTick(token, expectedMinionName)`
|
|
43
44
|
* — invalid signature, malformed payload, wrong issuer/audience, or
|
|
44
|
-
* wrong target minion all NACK without requeue.
|
|
45
|
+
* wrong target minion all NACK without requeue. An AUTHENTIC tick that
|
|
46
|
+
* arrived after its admission window (usually because the minion was down;
|
|
47
|
+
* see `ScheduledDeliveryLateness`) is recorded as
|
|
48
|
+
* a missed occurrence and acknowledged, never run (#2212); only a late
|
|
49
|
+
* tick that could not be recorded is NACKed.
|
|
45
50
|
* - Dispatches `MINION_TICK` tokens that match this minion's identity to
|
|
46
51
|
* the registered handler. The handler typically routes into the
|
|
47
52
|
* locally-hosted `flowsActorsRegistry` per Q9 LOCKED.
|
|
@@ -212,10 +217,12 @@ export class MinionTickConsumerBackendService {
|
|
|
212
217
|
try {
|
|
213
218
|
let accepted = false;
|
|
214
219
|
try {
|
|
215
|
-
const
|
|
220
|
+
const delivery = await this.scheduledMessageVerifier.verifyAndExtractMinionTick(msg.content.toString(), expectedMinionName);
|
|
221
|
+
const tick = delivery?.value;
|
|
216
222
|
// Signature/target checks do not establish receiving-application ownership.
|
|
217
|
-
// This comparison must precede every durable insert and handler resolution
|
|
218
|
-
|
|
223
|
+
// This comparison must precede every durable insert and handler resolution — a LATE tick's
|
|
224
|
+
// included: a genuine token addressed elsewhere is foreign whether or not it is on time.
|
|
225
|
+
if (!delivery || !tick || tick.applicationId !== applicationId || tick.minionName !== expectedMinionName) {
|
|
219
226
|
throw new Error('Scheduled minion token does not match the receiving application and minion');
|
|
220
227
|
}
|
|
221
228
|
const job = JobPayloadSchema.parse({
|
|
@@ -224,10 +231,18 @@ export class MinionTickConsumerBackendService {
|
|
|
224
231
|
initiatedBy: JobInitiator.CRONTAB, createdAt: new Date(tick.iat * 1000), maxRetries: 0,
|
|
225
232
|
scheduledAt: tick.scheduledAt === undefined ? undefined : new Date(tick.scheduledAt),
|
|
226
233
|
});
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
234
|
+
if (delivery.verdict === ScheduledDeliveryVerdict.LATE) {
|
|
235
|
+
// Genuine, past its admission deadline: the tick was missed — usually because this minion was
|
|
236
|
+
// not consuming, though the scheduler can share the delay (see `ScheduledDeliveryLateness`).
|
|
237
|
+
// Recorded and acknowledged, reported once per drain by the runner (#2212);
|
|
238
|
+
// dead-lettered only if it could not be recorded.
|
|
239
|
+
accepted = await this.runner.recordLateDelivery(job, queueName, resolver, delivery.lateness);
|
|
240
|
+
} else {
|
|
241
|
+
// Durable receipt, then acknowledge, THEN run — a long tick must not hold later deliveries
|
|
242
|
+
// unreceived past their five-minute token (`ScheduledExecutionRunner.admit`).
|
|
243
|
+
admitted = await this.runner.admit(job, queueName, resolver);
|
|
244
|
+
accepted = true;
|
|
245
|
+
}
|
|
231
246
|
} catch (error) {
|
|
232
247
|
this.logger.error('Minion tick admission failed', { expectedMinionName, error: describeCaughtError(error) });
|
|
233
248
|
}
|