@wildo-ai/saas-minions-lib 1.1.7 → 1.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (22) hide show
  1. package/dist/esm/base-classes/services/services/minion-scoped-services-registry-handler.backend.service.d.ts.map +1 -1
  2. package/dist/esm/base-classes/services/services/minion-scoped-services-registry-handler.backend.service.js.map +1 -1
  3. package/dist/esm/index.d.ts.map +1 -1
  4. package/dist/esm/index.js.map +1 -1
  5. package/dist/esm/inject/models/container.d.ts.map +1 -1
  6. package/dist/esm/inject/models/container.js.map +1 -1
  7. package/dist/esm/inject/models/service-types.d.ts.map +1 -1
  8. package/dist/esm/inject/models/service-types.js.map +1 -1
  9. package/dist/esm/runtime/queue/minion-tick-consumer.backend.service.d.ts +5 -1
  10. package/dist/esm/runtime/queue/minion-tick-consumer.backend.service.d.ts.map +1 -1
  11. package/dist/esm/runtime/queue/minion-tick-consumer.backend.service.js +24 -9
  12. package/dist/esm/runtime/queue/minion-tick-consumer.backend.service.js.map +1 -1
  13. package/dist/esm/runtime/services/define-minion-init.d.ts.map +1 -1
  14. package/dist/esm/runtime/services/define-minion-init.js.map +1 -1
  15. package/dist/esm/runtime/services/own-schedule-reader.backend.service.d.ts.map +1 -1
  16. package/dist/esm/runtime/services/own-schedule-reader.backend.service.js.map +1 -1
  17. package/dist/esm/runtime/services/start-minion-application.backend.service.d.ts.map +1 -1
  18. package/dist/esm/runtime/services/start-minion-application.backend.service.js.map +1 -1
  19. package/dist/tsconfig.build.tsbuildinfo +1 -1
  20. package/package.json +4 -4
  21. package/src/runtime/queue/__tests__/minion-tick-consumer.backend.service.test.ts +63 -14
  22. package/src/runtime/queue/minion-tick-consumer.backend.service.ts +23 -8
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@wildo-ai/saas-minions-lib",
3
- "version": "1.1.7",
3
+ "version": "1.1.9",
4
4
  "description": "Node minion runtime host layer for Wildo SaaS applications",
5
5
  "license": "SEE LICENSE IN LICENSE",
6
6
  "author": "Wildo AI",
@@ -30,9 +30,9 @@
30
30
  "test:watch": "vitest"
31
31
  },
32
32
  "dependencies": {
33
- "@wildo-ai/platform-config-lib": "1.1.7",
34
- "@wildo-ai/saas-backend-lib": "1.1.7",
35
- "@wildo-ai/saas-models": "1.1.7",
33
+ "@wildo-ai/platform-config-lib": "1.1.9",
34
+ "@wildo-ai/saas-backend-lib": "1.1.9",
35
+ "@wildo-ai/saas-models": "1.1.9",
36
36
  "reflect-metadata": "0.2.2"
37
37
  },
38
38
  "devDependencies": {
@@ -5,23 +5,31 @@ import type {
5
5
  JobPayloadStorageBackendService, JobStatusTrackingBackendService, LoggerService, WorkerShutdownManagerBackendService,
6
6
  RabbitMQAdapterBackendService, ScheduledExecutionResolver, ScheduledMessageVerifierBackendService,
7
7
  } from '@wildo-ai/saas-backend-lib';
8
- import { JobPayloadSchema, JobType, ScheduledReplaySafety, ScheduledReviewOutcome, type JobPayload } from '@wildo-ai/saas-models';
8
+ import { JobPayloadSchema, JobType, ScheduledDeliveryLateness, ScheduledReplaySafety, ScheduledReviewOutcome, type JobPayload } from '@wildo-ai/saas-models';
9
+ import { ScheduledDeliveryVerdict } from '@wildo-ai/saas-backend-lib';
9
10
  import { MinionTickConsumerBackendService } from '../minion-tick-consumer.backend.service';
10
11
 
11
12
  // This suite pins the minion authentication/lifecycle adapter. Durable concurrency
12
13
  // and recovery policy are exercised by the coordinator's real-store suites.
13
- const coordinator = vi.hoisted(() => ({ receive: vi.fn(), start: vi.fn(), stop: vi.fn(), resolveReview: vi.fn(), construct: vi.fn() }));
14
+ // The consumer ADMITS (durable receipt), acknowledges, then LAUNCHES; a late delivery is RECORDED instead.
15
+ const coordinator = vi.hoisted(() => ({ admit: vi.fn(), launch: vi.fn(), recordLateDelivery: vi.fn(), start: vi.fn(), stop: vi.fn(),
16
+ resolveReview: vi.fn(), construct: vi.fn() }));
14
17
  vi.mock('@wildo-ai/saas-backend-lib', async (importOriginal) => ({
15
18
  ...await importOriginal<Record<string, unknown>>(),
16
19
  ScheduledExecutionRunner: class {
17
20
  constructor(...args: unknown[]) { coordinator.construct(...args); }
18
- receive = coordinator.receive;
21
+ admit = coordinator.admit;
22
+ launch = coordinator.launch;
23
+ recordLateDelivery = coordinator.recordLateDelivery;
19
24
  start = coordinator.start;
20
25
  stop = coordinator.stop;
21
26
  resolveReview = coordinator.resolveReview;
22
27
  },
23
28
  }));
24
29
 
30
+ /** The verifier's answer for an authentic, on-time tick (`ScheduledDeliveryVerdict.ADMISSIBLE`). */
31
+ const onTime = (value: unknown) => ({ verdict: ScheduledDeliveryVerdict.ADMISSIBLE, value });
32
+
25
33
  function fixture() {
26
34
  let deliver: ((message: ConsumeMessage, channel: Channel) => Promise<void>) | undefined;
27
35
  const adapter = {
@@ -35,7 +43,7 @@ function fixture() {
35
43
  jobType: JobType.MINION_TICK, iat: 1750000000, exp: 1750000300,
36
44
  iss: 'PLATFORM_CRONTABS_BATCHES_MANAGER', aud: 'SCHEDULED_JOB', inputDataJson: '{"target":"one"}',
37
45
  };
38
- const verifier = { verifyAndExtractMinionTick: vi.fn().mockResolvedValue(tick) };
46
+ const verifier = { verifyAndExtractMinionTick: vi.fn().mockResolvedValue(onTime(tick)) };
39
47
  const logger = { debugByType: () => vi.fn(), setDebugTypeLogging: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() };
40
48
  const shutdown = { jobStarted: vi.fn(), jobCompleted: vi.fn(), registerStopHandler: vi.fn() };
41
49
  const service = new MinionTickConsumerBackendService(
@@ -57,7 +65,10 @@ function fixture() {
57
65
  };
58
66
  }
59
67
 
60
- beforeEach(() => { vi.clearAllMocks(); });
68
+ beforeEach(() => {
69
+ vi.clearAllMocks();
70
+ coordinator.admit.mockImplementation(async (job: JobPayload) => ({ jobId: job.jobId }));
71
+ });
61
72
 
62
73
  describe('minion scheduled coordination boundary', () => {
63
74
  it('joins the shared shutdown drain and stops admissions before draining', async () => {
@@ -79,9 +90,9 @@ describe('minion scheduled coordination boundary', () => {
79
90
  evidencePolicy: { policyRef: 'tick-v1', retainForMs: 86400000, resultFields: [] } });
80
91
  // The runner is started on a QUEUE only; the platform id shapes the queue name and is never handed to the scanner.
81
92
  expect(coordinator.start).toHaveBeenCalledWith(expect.stringContaining('worker'), expect.any(Function));
82
- expect(coordinator.receive).not.toHaveBeenCalled();
93
+ expect(coordinator.admit).not.toHaveBeenCalled();
83
94
  await f.deliver();
84
- const [job, queue, resolver] = coordinator.receive.mock.calls[0] as [JobPayload, string, ScheduledExecutionResolver];
95
+ const [job, queue, resolver] = coordinator.admit.mock.calls[0] as [JobPayload, string, ScheduledExecutionResolver];
85
96
  expect(job).toMatchObject({ jobId: f.tick.jobId, applicationId: 'app-a', minionName: 'worker', inputDataJson: f.tick.inputDataJson });
86
97
  expect(job.createdAt).toEqual(new Date(f.tick.iat * 1000));
87
98
  expect(job.scheduledAt).toBeUndefined();
@@ -105,12 +116,12 @@ describe('minion scheduled coordination boundary', () => {
105
116
  it('preserves the signed occurrence time through durable serialization into execute and reconcile', async () => {
106
117
  const f = fixture();
107
118
  const scheduledAt = new Date(f.tick.iat * 1000 - 60000);
108
- f.verifier.verifyAndExtractMinionTick.mockResolvedValue({ ...f.tick, scheduledAt: scheduledAt.toISOString() });
119
+ f.verifier.verifyAndExtractMinionTick.mockResolvedValue(onTime({ ...f.tick, scheduledAt: scheduledAt.toISOString() }));
109
120
  const handler = vi.fn();
110
121
  const reconcile = vi.fn();
111
122
  await f.service.start({ applicationId: 'app-a', minionName: 'worker', handler, reconcile });
112
123
  await f.deliver();
113
- const [job, , resolver] = coordinator.receive.mock.calls[0] as [JobPayload, string, ScheduledExecutionResolver];
124
+ const [job, , resolver] = coordinator.admit.mock.calls[0] as [JobPayload, string, ScheduledExecutionResolver];
114
125
  expect(job.scheduledAt).toEqual(scheduledAt);
115
126
  // Recovery restores JSON through the same schema as ScheduledExecutionRunner.
116
127
  const recovered = JobPayloadSchema.parse({ ...JSON.parse(JSON.stringify(job)),
@@ -144,10 +155,10 @@ describe('minion scheduled coordination boundary', () => {
144
155
 
145
156
  it.each(['applicationId', 'minionName'])('rejects a signed wrong %s before durable receipt', async (field) => {
146
157
  const f = fixture();
147
- f.verifier.verifyAndExtractMinionTick.mockResolvedValue({ ...f.tick, [field]: 'other' });
158
+ f.verifier.verifyAndExtractMinionTick.mockResolvedValue(onTime({ ...f.tick, [field]: 'other' }));
148
159
  await f.service.start({ applicationId: 'app-a', minionName: 'worker', handler: vi.fn() });
149
160
  await f.deliver();
150
- expect(coordinator.receive).not.toHaveBeenCalled();
161
+ expect(coordinator.admit).not.toHaveBeenCalled();
151
162
  expect(f.channel.nack).toHaveBeenCalledWith(expect.anything(), false, false);
152
163
  });
153
164
 
@@ -157,8 +168,8 @@ describe('minion scheduled coordination boundary', () => {
157
168
  await f.service.start({ applicationId: 'app-a', minionName: 'worker', handler });
158
169
  f.verifier.verifyAndExtractMinionTick.mockRejectedValueOnce(new Error('invalid signature'));
159
170
  await f.deliver();
160
- expect(coordinator.receive).not.toHaveBeenCalled();
161
- coordinator.receive.mockRejectedValueOnce(new Error('database unavailable'));
171
+ expect(coordinator.admit).not.toHaveBeenCalled();
172
+ coordinator.admit.mockRejectedValueOnce(new Error('database unavailable'));
162
173
  await f.deliver();
163
174
  expect(handler).not.toHaveBeenCalled();
164
175
  expect(f.channel.ack).not.toHaveBeenCalled();
@@ -170,8 +181,46 @@ describe('minion scheduled coordination boundary', () => {
170
181
  f.channel.ack.mockImplementation(() => { throw new Error('channel closed'); });
171
182
  await f.service.start({ applicationId: 'app-a', minionName: 'worker', handler: vi.fn() });
172
183
  await f.deliver();
173
- expect(coordinator.receive).toHaveBeenCalledTimes(1);
184
+ expect(coordinator.admit).toHaveBeenCalledTimes(1);
174
185
  expect(f.channel.nack).not.toHaveBeenCalled();
175
186
  expect(f.logger.error).toHaveBeenCalledWith('Minion tick settlement failed', expect.anything());
176
187
  });
188
+
189
+ it('records a LATE tick as a missed occurrence and acknowledges it, never admitting or running it (#2212)', async () => {
190
+ const f = fixture();
191
+ coordinator.recordLateDelivery.mockResolvedValue(true);
192
+ f.verifier.verifyAndExtractMinionTick.mockResolvedValue({ verdict: ScheduledDeliveryVerdict.LATE, value: f.tick,
193
+ lateness: ScheduledDeliveryLateness.TOKEN_EXPIRED });
194
+ const handler = vi.fn();
195
+ await f.service.start({ applicationId: 'app-a', minionName: 'worker', handler });
196
+ await f.deliver();
197
+ expect(coordinator.recordLateDelivery).toHaveBeenCalledWith(expect.objectContaining({ jobId: f.tick.jobId, minionName: 'worker' }),
198
+ expect.stringContaining('worker'), expect.any(Function), ScheduledDeliveryLateness.TOKEN_EXPIRED);
199
+ expect(coordinator.admit).not.toHaveBeenCalled();
200
+ expect(coordinator.launch).not.toHaveBeenCalled();
201
+ expect(handler).not.toHaveBeenCalled();
202
+ expect(f.channel.ack).toHaveBeenCalledOnce();
203
+ expect(f.logger.error).not.toHaveBeenCalled();
204
+ });
205
+
206
+ it('dead-letters a LATE tick the runner could not record, so its copy survives', async () => {
207
+ const f = fixture();
208
+ coordinator.recordLateDelivery.mockResolvedValue(false);
209
+ f.verifier.verifyAndExtractMinionTick.mockResolvedValue({ verdict: ScheduledDeliveryVerdict.LATE, value: f.tick,
210
+ lateness: ScheduledDeliveryLateness.OUTSIDE_ADMISSION_WINDOW });
211
+ await f.service.start({ applicationId: 'app-a', minionName: 'worker', handler: vi.fn() });
212
+ await f.deliver();
213
+ expect(f.channel.ack).not.toHaveBeenCalled();
214
+ expect(f.channel.nack).toHaveBeenCalledWith(expect.anything(), false, false);
215
+ });
216
+
217
+ it('refuses a LATE tick addressed to another application before recording it', async () => {
218
+ const f = fixture();
219
+ f.verifier.verifyAndExtractMinionTick.mockResolvedValue({ verdict: ScheduledDeliveryVerdict.LATE,
220
+ value: { ...f.tick, applicationId: 'other' }, lateness: ScheduledDeliveryLateness.TOKEN_EXPIRED });
221
+ await f.service.start({ applicationId: 'app-a', minionName: 'worker', handler: vi.fn() });
222
+ await f.deliver();
223
+ expect(coordinator.recordLateDelivery).not.toHaveBeenCalled();
224
+ expect(f.channel.nack).toHaveBeenCalledWith(expect.anything(), false, false);
225
+ });
177
226
  });
@@ -6,6 +6,7 @@ import {
6
6
  RabbitMQAdapterBackendService,
7
7
  SAAS_SERVICE_TYPES,
8
8
  ScheduledMessageVerifierBackendService,
9
+ ScheduledDeliveryVerdict,
9
10
  ScheduledExecutionRunner,
10
11
  JobStatusTrackingBackendService,
11
12
  JobPayloadStorageBackendService,
@@ -41,7 +42,11 @@ export type MinionTickReconciler = (tick: MinionTickInvocation, attempt: Schedul
41
42
  * - Verifies every inbound JWT through
42
43
  * `ScheduledMessageVerifierBackendService.verifyAndExtractMinionTick(token, expectedMinionName)`
43
44
  * — invalid signature, malformed payload, wrong issuer/audience, or
44
- * wrong target minion all NACK without requeue.
45
+ * wrong target minion all NACK without requeue. An AUTHENTIC tick that
46
+ * arrived after its admission window (usually because the minion was down;
47
+ * see `ScheduledDeliveryLateness`) is recorded as
48
+ * a missed occurrence and acknowledged, never run (#2212); only a late
49
+ * tick that could not be recorded is NACKed.
45
50
  * - Dispatches `MINION_TICK` tokens that match this minion's identity to
46
51
  * the registered handler. The handler typically routes into the
47
52
  * locally-hosted `flowsActorsRegistry` per Q9 LOCKED.
@@ -212,10 +217,12 @@ export class MinionTickConsumerBackendService {
212
217
  try {
213
218
  let accepted = false;
214
219
  try {
215
- const tick = await this.scheduledMessageVerifier.verifyAndExtractMinionTick(msg.content.toString(), expectedMinionName);
220
+ const delivery = await this.scheduledMessageVerifier.verifyAndExtractMinionTick(msg.content.toString(), expectedMinionName);
221
+ const tick = delivery?.value;
216
222
  // Signature/target checks do not establish receiving-application ownership.
217
- // This comparison must precede every durable insert and handler resolution.
218
- if (!tick || tick.applicationId !== applicationId || tick.minionName !== expectedMinionName) {
223
+ // This comparison must precede every durable insert and handler resolution — a LATE tick's
224
+ // included: a genuine token addressed elsewhere is foreign whether or not it is on time.
225
+ if (!delivery || !tick || tick.applicationId !== applicationId || tick.minionName !== expectedMinionName) {
219
226
  throw new Error('Scheduled minion token does not match the receiving application and minion');
220
227
  }
221
228
  const job = JobPayloadSchema.parse({
@@ -224,10 +231,18 @@ export class MinionTickConsumerBackendService {
224
231
  initiatedBy: JobInitiator.CRONTAB, createdAt: new Date(tick.iat * 1000), maxRetries: 0,
225
232
  scheduledAt: tick.scheduledAt === undefined ? undefined : new Date(tick.scheduledAt),
226
233
  });
227
- // Durable receipt, then acknowledge, THEN run — a long tick must not hold later deliveries
228
- // unreceived past their five-minute token (`ScheduledExecutionRunner.admit`).
229
- admitted = await this.runner.admit(job, queueName, resolver);
230
- accepted = true;
234
+ if (delivery.verdict === ScheduledDeliveryVerdict.LATE) {
235
+ // Genuine, past its admission deadline: the tick was missed — usually because this minion was
236
+ // not consuming, though the scheduler can share the delay (see `ScheduledDeliveryLateness`).
237
+ // Recorded and acknowledged, reported once per drain by the runner (#2212);
238
+ // dead-lettered only if it could not be recorded.
239
+ accepted = await this.runner.recordLateDelivery(job, queueName, resolver, delivery.lateness);
240
+ } else {
241
+ // Durable receipt, then acknowledge, THEN run — a long tick must not hold later deliveries
242
+ // unreceived past their five-minute token (`ScheduledExecutionRunner.admit`).
243
+ admitted = await this.runner.admit(job, queueName, resolver);
244
+ accepted = true;
245
+ }
231
246
  } catch (error) {
232
247
  this.logger.error('Minion tick admission failed', { expectedMinionName, error: describeCaughtError(error) });
233
248
  }