@wildo-ai/saas-minions-lib 1.1.4 → 1.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esm/base-classes/services/services/minion-scoped-services-registry-handler.backend.service.d.ts.map +1 -1
- package/dist/esm/index.d.ts.map +1 -1
- package/dist/esm/inject/models/container.d.ts.map +1 -1
- package/dist/esm/inject/models/service-types.d.ts.map +1 -1
- package/dist/esm/runtime/queue/minion-tick-consumer.backend.service.d.ts +35 -14
- package/dist/esm/runtime/queue/minion-tick-consumer.backend.service.d.ts.map +1 -1
- package/dist/esm/runtime/queue/minion-tick-consumer.backend.service.js +99 -42
- package/dist/esm/runtime/queue/minion-tick-consumer.backend.service.js.map +1 -1
- package/dist/esm/runtime/services/define-minion-init.d.ts +20 -7
- package/dist/esm/runtime/services/define-minion-init.d.ts.map +1 -1
- package/dist/esm/runtime/services/define-minion-init.js +1 -0
- package/dist/esm/runtime/services/define-minion-init.js.map +1 -1
- package/dist/esm/runtime/services/own-schedule-reader.backend.service.d.ts.map +1 -1
- package/dist/esm/runtime/services/own-schedule-reader.backend.service.js.map +1 -1
- package/dist/esm/runtime/services/start-minion-application.backend.service.d.ts.map +1 -1
- package/dist/esm/runtime/services/start-minion-application.backend.service.js +25 -15
- package/dist/esm/runtime/services/start-minion-application.backend.service.js.map +1 -1
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/package.json +6 -7
- package/src/runtime/queue/__tests__/minion-tick-consumer.backend.service.test.ts +177 -0
- package/src/runtime/queue/minion-tick-consumer.backend.service.ts +121 -55
- package/src/runtime/services/__tests__/start-minion-application.backend.service.test.ts +19 -3
- package/src/runtime/services/define-minion-init.ts +24 -7
- package/src/runtime/services/start-minion-application.backend.service.ts +31 -20
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@wildo-ai/saas-minions-lib",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.6",
|
|
4
4
|
"description": "Node minion runtime host layer for Wildo SaaS applications",
|
|
5
5
|
"license": "SEE LICENSE IN LICENSE",
|
|
6
6
|
"author": "Wildo AI",
|
|
@@ -30,10 +30,10 @@
|
|
|
30
30
|
"test:watch": "vitest"
|
|
31
31
|
},
|
|
32
32
|
"dependencies": {
|
|
33
|
-
"
|
|
34
|
-
"@wildo-ai/
|
|
35
|
-
"@wildo-ai/saas-
|
|
36
|
-
"
|
|
33
|
+
"@wildo-ai/platform-config-lib": "1.1.6",
|
|
34
|
+
"@wildo-ai/saas-backend-lib": "1.1.6",
|
|
35
|
+
"@wildo-ai/saas-models": "1.1.6",
|
|
36
|
+
"reflect-metadata": "0.2.2"
|
|
37
37
|
},
|
|
38
38
|
"devDependencies": {
|
|
39
39
|
"@types/node": "^24.0.15",
|
|
@@ -45,8 +45,7 @@
|
|
|
45
45
|
"vitest": "^1.6.1"
|
|
46
46
|
},
|
|
47
47
|
"peerDependencies": {
|
|
48
|
-
"inversify": "^8.1.0"
|
|
49
|
-
"reflect-metadata": "^0.2.2"
|
|
48
|
+
"inversify": "^8.1.0"
|
|
50
49
|
},
|
|
51
50
|
"repository": {
|
|
52
51
|
"type": "git",
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
import 'reflect-metadata';
|
|
2
|
+
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
|
3
|
+
import type { Channel, ConsumeMessage } from 'amqplib';
|
|
4
|
+
import type {
|
|
5
|
+
JobPayloadStorageBackendService, JobStatusTrackingBackendService, LoggerService, WorkerShutdownManagerBackendService,
|
|
6
|
+
RabbitMQAdapterBackendService, ScheduledExecutionResolver, ScheduledMessageVerifierBackendService,
|
|
7
|
+
} from '@wildo-ai/saas-backend-lib';
|
|
8
|
+
import { JobPayloadSchema, JobType, ScheduledReplaySafety, ScheduledReviewOutcome, type JobPayload } from '@wildo-ai/saas-models';
|
|
9
|
+
import { MinionTickConsumerBackendService } from '../minion-tick-consumer.backend.service';
|
|
10
|
+
|
|
11
|
+
// This suite pins the minion authentication/lifecycle adapter. Durable concurrency
|
|
12
|
+
// and recovery policy are exercised by the coordinator's real-store suites.
|
|
13
|
+
const coordinator = vi.hoisted(() => ({ receive: vi.fn(), start: vi.fn(), stop: vi.fn(), resolveReview: vi.fn(), construct: vi.fn() }));
|
|
14
|
+
vi.mock('@wildo-ai/saas-backend-lib', async (importOriginal) => ({
|
|
15
|
+
...await importOriginal<Record<string, unknown>>(),
|
|
16
|
+
ScheduledExecutionRunner: class {
|
|
17
|
+
constructor(...args: unknown[]) { coordinator.construct(...args); }
|
|
18
|
+
receive = coordinator.receive;
|
|
19
|
+
start = coordinator.start;
|
|
20
|
+
stop = coordinator.stop;
|
|
21
|
+
resolveReview = coordinator.resolveReview;
|
|
22
|
+
},
|
|
23
|
+
}));
|
|
24
|
+
|
|
25
|
+
function fixture() {
|
|
26
|
+
let deliver: ((message: ConsumeMessage, channel: Channel) => Promise<void>) | undefined;
|
|
27
|
+
const adapter = {
|
|
28
|
+
isConnected: () => true,
|
|
29
|
+
getTopologyChannel: vi.fn().mockResolvedValue({ assertQueue: vi.fn() }),
|
|
30
|
+
consume: vi.fn(async (_queue: string, callback: typeof deliver) => { deliver = callback; }),
|
|
31
|
+
cancelConsumer: vi.fn(),
|
|
32
|
+
};
|
|
33
|
+
const tick = {
|
|
34
|
+
jobId: '11111111-1111-4111-8111-111111111111', applicationId: 'app-a', minionName: 'worker',
|
|
35
|
+
jobType: JobType.MINION_TICK, iat: 1750000000, exp: 1750000300,
|
|
36
|
+
iss: 'PLATFORM_CRONTABS_BATCHES_MANAGER', aud: 'SCHEDULED_JOB', inputDataJson: '{"target":"one"}',
|
|
37
|
+
};
|
|
38
|
+
const verifier = { verifyAndExtractMinionTick: vi.fn().mockResolvedValue(tick) };
|
|
39
|
+
const logger = { debugByType: () => vi.fn(), setDebugTypeLogging: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() };
|
|
40
|
+
const shutdown = { jobStarted: vi.fn(), jobCompleted: vi.fn(), registerStopHandler: vi.fn() };
|
|
41
|
+
const service = new MinionTickConsumerBackendService(
|
|
42
|
+
adapter as unknown as RabbitMQAdapterBackendService,
|
|
43
|
+
verifier as unknown as ScheduledMessageVerifierBackendService,
|
|
44
|
+
logger as unknown as LoggerService,
|
|
45
|
+
{} as JobStatusTrackingBackendService,
|
|
46
|
+
{} as JobPayloadStorageBackendService,
|
|
47
|
+
shutdown as unknown as WorkerShutdownManagerBackendService,
|
|
48
|
+
);
|
|
49
|
+
const channel = { ack: vi.fn(), nack: vi.fn() };
|
|
50
|
+
return {
|
|
51
|
+
service, adapter, verifier, tick, logger, channel, shutdown,
|
|
52
|
+
deliver: async () => {
|
|
53
|
+
if (!deliver) throw new Error('consumer was not registered');
|
|
54
|
+
await deliver({ content: Buffer.from('signed-jwt') } as ConsumeMessage, channel as unknown as Channel);
|
|
55
|
+
expect(shutdown.jobStarted.mock.calls.length).toBe(shutdown.jobCompleted.mock.calls.length);
|
|
56
|
+
},
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
beforeEach(() => { vi.clearAllMocks(); });
|
|
61
|
+
|
|
62
|
+
describe('minion scheduled coordination boundary', () => {
|
|
63
|
+
it('joins the shared shutdown drain and stops admissions before draining', async () => {
|
|
64
|
+
const f = fixture();
|
|
65
|
+
const activity = coordinator.construct.mock.calls[0][3] as { started(): void; completed(): void };
|
|
66
|
+
activity.started();
|
|
67
|
+
activity.completed();
|
|
68
|
+
expect(f.shutdown.jobStarted).toHaveBeenCalledOnce();
|
|
69
|
+
expect(f.shutdown.jobCompleted).toHaveBeenCalledOnce();
|
|
70
|
+
await f.shutdown.registerStopHandler.mock.calls[0][0]();
|
|
71
|
+
expect(coordinator.stop).toHaveBeenCalledOnce();
|
|
72
|
+
});
|
|
73
|
+
it('starts recovery before any delivery, routes local policy and preserves payload/attempt identity', async () => {
|
|
74
|
+
const f = fixture();
|
|
75
|
+
const handler = vi.fn().mockResolvedValue(undefined);
|
|
76
|
+
const reconcile = vi.fn();
|
|
77
|
+
await f.service.start({ applicationId: 'app-a', minionName: 'worker', handler, reconcile,
|
|
78
|
+
recoveryPolicy: { replaySafety: ScheduledReplaySafety.IDEMPOTENT, maxAttempts: 3 },
|
|
79
|
+
evidencePolicy: { policyRef: 'tick-v1', retainForMs: 86400000, resultFields: [] } });
|
|
80
|
+
// The runner is started on a QUEUE only; the platform id shapes the queue name and is never handed to the scanner.
|
|
81
|
+
expect(coordinator.start).toHaveBeenCalledWith(expect.stringContaining('worker'), expect.any(Function));
|
|
82
|
+
expect(coordinator.receive).not.toHaveBeenCalled();
|
|
83
|
+
await f.deliver();
|
|
84
|
+
const [job, queue, resolver] = coordinator.receive.mock.calls[0] as [JobPayload, string, ScheduledExecutionResolver];
|
|
85
|
+
expect(job).toMatchObject({ jobId: f.tick.jobId, applicationId: 'app-a', minionName: 'worker', inputDataJson: f.tick.inputDataJson });
|
|
86
|
+
expect(job.createdAt).toEqual(new Date(f.tick.iat * 1000));
|
|
87
|
+
expect(job.scheduledAt).toBeUndefined();
|
|
88
|
+
expect(queue).toContain('worker');
|
|
89
|
+
const contract = await resolver(job);
|
|
90
|
+
expect(contract?.policy?.maxAttempts).toBe(3);
|
|
91
|
+
expect(contract?.evidencePolicy).toEqual({ policyRef: 'tick-v1', retainForMs: 86400000, resultFields: [] });
|
|
92
|
+
const attempt = { occurrenceId: job.jobId, attemptId: 'new-attempt', attemptNumber: 2 };
|
|
93
|
+
await contract?.execute(attempt);
|
|
94
|
+
expect(handler).toHaveBeenCalledWith(expect.objectContaining({ jobId: job.jobId, inputDataJson: f.tick.inputDataJson }), attempt);
|
|
95
|
+
expect(handler.mock.calls[0][0]).not.toHaveProperty('exp');
|
|
96
|
+
await contract?.reconcile?.(attempt);
|
|
97
|
+
expect(reconcile).toHaveBeenCalledWith(expect.objectContaining({ jobId: job.jobId }), attempt);
|
|
98
|
+
expect(await resolver({ ...job, minionName: 'other' })).toBeUndefined();
|
|
99
|
+
expect(f.channel.ack).toHaveBeenCalledTimes(1);
|
|
100
|
+
await f.service.stop();
|
|
101
|
+
expect(coordinator.stop).toHaveBeenCalledTimes(1);
|
|
102
|
+
expect(f.adapter.cancelConsumer).toHaveBeenCalledWith(queue);
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
it('preserves the signed occurrence time through durable serialization into execute and reconcile', async () => {
|
|
106
|
+
const f = fixture();
|
|
107
|
+
const scheduledAt = new Date(f.tick.iat * 1000 - 60000);
|
|
108
|
+
f.verifier.verifyAndExtractMinionTick.mockResolvedValue({ ...f.tick, scheduledAt: scheduledAt.toISOString() });
|
|
109
|
+
const handler = vi.fn();
|
|
110
|
+
const reconcile = vi.fn();
|
|
111
|
+
await f.service.start({ applicationId: 'app-a', minionName: 'worker', handler, reconcile });
|
|
112
|
+
await f.deliver();
|
|
113
|
+
const [job, , resolver] = coordinator.receive.mock.calls[0] as [JobPayload, string, ScheduledExecutionResolver];
|
|
114
|
+
expect(job.scheduledAt).toEqual(scheduledAt);
|
|
115
|
+
// Recovery restores JSON through the same schema as ScheduledExecutionRunner.
|
|
116
|
+
const recovered = JobPayloadSchema.parse({ ...JSON.parse(JSON.stringify(job)),
|
|
117
|
+
createdAt: new Date(job.createdAt), scheduledAt: new Date(job.scheduledAt!) });
|
|
118
|
+
const contract = await resolver(recovered);
|
|
119
|
+
expect(contract?.evidencePolicy).toBeUndefined();
|
|
120
|
+
const attempt = { occurrenceId: job.jobId, attemptId: 'recovery-attempt', attemptNumber: 2 };
|
|
121
|
+
await contract?.execute(attempt);
|
|
122
|
+
await contract?.reconcile?.(attempt);
|
|
123
|
+
const expected = expect.objectContaining({ createdAt: new Date(f.tick.iat * 1000), scheduledAt });
|
|
124
|
+
expect(handler).toHaveBeenCalledWith(expected, attempt);
|
|
125
|
+
expect(reconcile).toHaveBeenCalledWith(expected, attempt);
|
|
126
|
+
await f.service.stop();
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
it('derives review ownership from initialized runtime and refuses use after shutdown', async () => {
|
|
130
|
+
const f = fixture();
|
|
131
|
+
const decision = { jobId: f.tick.jobId, expectedRevision: 3, actor: 'operator-1', reason: 'Provider confirms completion',
|
|
132
|
+
outcome: ScheduledReviewOutcome.CONFIRM_COMPLETED };
|
|
133
|
+
await expect(f.service.resolveTickReview(decision)).rejects.toThrow('initialized consumer');
|
|
134
|
+
await f.service.start({ applicationId: 'app-a', minionName: 'worker', handler: vi.fn() });
|
|
135
|
+
await f.service.resolveTickReview(decision);
|
|
136
|
+
// The review carries the QUEUE the consumer started on and nothing else: the row's application
|
|
137
|
+
// identity is the tracker's to resolve, so no application id travels with the decision.
|
|
138
|
+
expect(coordinator.resolveReview).toHaveBeenCalledWith({ ...decision,
|
|
139
|
+
queueName: coordinator.start.mock.calls[0][0] }, expect.any(Function));
|
|
140
|
+
await f.service.stop();
|
|
141
|
+
await expect(f.service.resolveTickReview(decision)).rejects.toThrow('initialized consumer');
|
|
142
|
+
expect(coordinator.resolveReview).toHaveBeenCalledTimes(1);
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
it.each(['applicationId', 'minionName'])('rejects a signed wrong %s before durable receipt', async (field) => {
|
|
146
|
+
const f = fixture();
|
|
147
|
+
f.verifier.verifyAndExtractMinionTick.mockResolvedValue({ ...f.tick, [field]: 'other' });
|
|
148
|
+
await f.service.start({ applicationId: 'app-a', minionName: 'worker', handler: vi.fn() });
|
|
149
|
+
await f.deliver();
|
|
150
|
+
expect(coordinator.receive).not.toHaveBeenCalled();
|
|
151
|
+
expect(f.channel.nack).toHaveBeenCalledWith(expect.anything(), false, false);
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
it('refuses forged credentials and durable-store failure without executing directly', async () => {
|
|
155
|
+
const f = fixture();
|
|
156
|
+
const handler = vi.fn();
|
|
157
|
+
await f.service.start({ applicationId: 'app-a', minionName: 'worker', handler });
|
|
158
|
+
f.verifier.verifyAndExtractMinionTick.mockRejectedValueOnce(new Error('invalid signature'));
|
|
159
|
+
await f.deliver();
|
|
160
|
+
expect(coordinator.receive).not.toHaveBeenCalled();
|
|
161
|
+
coordinator.receive.mockRejectedValueOnce(new Error('database unavailable'));
|
|
162
|
+
await f.deliver();
|
|
163
|
+
expect(handler).not.toHaveBeenCalled();
|
|
164
|
+
expect(f.channel.ack).not.toHaveBeenCalled();
|
|
165
|
+
expect(f.channel.nack).toHaveBeenCalledTimes(2);
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
it('does not turn an ACK failure into a second settlement or business failure', async () => {
|
|
169
|
+
const f = fixture();
|
|
170
|
+
f.channel.ack.mockImplementation(() => { throw new Error('channel closed'); });
|
|
171
|
+
await f.service.start({ applicationId: 'app-a', minionName: 'worker', handler: vi.fn() });
|
|
172
|
+
await f.deliver();
|
|
173
|
+
expect(coordinator.receive).toHaveBeenCalledTimes(1);
|
|
174
|
+
expect(f.channel.nack).not.toHaveBeenCalled();
|
|
175
|
+
expect(f.logger.error).toHaveBeenCalledWith('Minion tick settlement failed', expect.anything());
|
|
176
|
+
});
|
|
177
|
+
});
|
|
@@ -6,18 +6,29 @@ import {
|
|
|
6
6
|
RabbitMQAdapterBackendService,
|
|
7
7
|
SAAS_SERVICE_TYPES,
|
|
8
8
|
ScheduledMessageVerifierBackendService,
|
|
9
|
+
ScheduledExecutionRunner,
|
|
10
|
+
JobStatusTrackingBackendService,
|
|
11
|
+
JobPayloadStorageBackendService,
|
|
12
|
+
WorkerShutdownManagerBackendService,
|
|
13
|
+
type ScheduledExecutionResolver,
|
|
9
14
|
type MinionTickJobToken,
|
|
10
15
|
} from '@wildo-ai/saas-backend-lib';
|
|
11
|
-
import {
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
16
|
+
import {
|
|
17
|
+
QueueNamingUtils, JobPayloadSchema, JobType, JobInitiator, JobStatus,
|
|
18
|
+
type JobExecution, type ScheduledAttemptContext, type ScheduledEvidencePolicy, type ScheduledRecoveryPolicy, type ScheduledReconciliationResult,
|
|
19
|
+
} from '@wildo-ai/saas-models';
|
|
20
|
+
|
|
21
|
+
/** Application payload after authentication; transport credentials are never replayed to recovery handlers. */
|
|
22
|
+
export type MinionTickInvocation = Pick<MinionTickJobToken, 'jobId' | 'applicationId' | 'jobType' | 'minionName' | 'inputDataJson'> & {
|
|
23
|
+
/** Original authenticated publication time, preserved across recovery attempts. */
|
|
24
|
+
createdAt: Date;
|
|
25
|
+
/** Authenticated cron occurrence time; absent for independent manual invocations. */
|
|
26
|
+
scheduledAt?: Date;
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
/** Stable occurrence plus a fresh attempt identity; retries are owned by the durable coordinator. */
|
|
30
|
+
export type MinionTickHandler = (tick: MinionTickInvocation, attempt: ScheduledAttemptContext) => Promise<void>;
|
|
31
|
+
export type MinionTickReconciler = (tick: MinionTickInvocation, attempt: ScheduledAttemptContext) => Promise<ScheduledReconciliationResult>;
|
|
21
32
|
|
|
22
33
|
/**
|
|
23
34
|
* Minion-side consumer for the per-minion tick queue
|
|
@@ -35,14 +46,16 @@ export type MinionTickHandler = (token: MinionTickJobToken) => Promise<void>;
|
|
|
35
46
|
* the registered handler. The handler typically routes into the
|
|
36
47
|
* locally-hosted `flowsActorsRegistry` per Q9 LOCKED.
|
|
37
48
|
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
49
|
+
* Duplicate deliveries and recovery of one occurrence belong to the durable
|
|
50
|
+
* coordinator. Overlap between DISTINCT occurrences remains the handler's
|
|
51
|
+
* responsibility; these are separate guarantees.
|
|
41
52
|
*/
|
|
42
53
|
@injectable()
|
|
43
54
|
export class MinionTickConsumerBackendService {
|
|
44
55
|
private isConsuming = false;
|
|
45
56
|
private consumingQueueName?: string;
|
|
57
|
+
private readonly runner: ScheduledExecutionRunner;
|
|
58
|
+
private reviewScope?: { queueName: string; resolver: ScheduledExecutionResolver };
|
|
46
59
|
|
|
47
60
|
protected readonly logDebug: (message: string, context?: Record<string, unknown>) => void;
|
|
48
61
|
|
|
@@ -53,21 +66,61 @@ export class MinionTickConsumerBackendService {
|
|
|
53
66
|
private readonly scheduledMessageVerifier: ScheduledMessageVerifierBackendService,
|
|
54
67
|
@inject(SAAS_SERVICE_TYPES.Logger)
|
|
55
68
|
private readonly logger: LoggerService,
|
|
69
|
+
@inject(SAAS_SERVICE_TYPES.JobStatusTrackingService)
|
|
70
|
+
tracker: JobStatusTrackingBackendService,
|
|
71
|
+
@inject(SAAS_SERVICE_TYPES.JobPayloadStorageService)
|
|
72
|
+
payloadStorage: JobPayloadStorageBackendService,
|
|
73
|
+
@inject(SAAS_SERVICE_TYPES.WorkerShutdownManagerService)
|
|
74
|
+
private readonly shutdownManager: WorkerShutdownManagerBackendService,
|
|
56
75
|
) {
|
|
76
|
+
this.runner = new ScheduledExecutionRunner(tracker, logger, payloadStorage, {
|
|
77
|
+
started: () => shutdownManager.jobStarted(),
|
|
78
|
+
completed: () => shutdownManager.jobCompleted(),
|
|
79
|
+
});
|
|
80
|
+
// Stop broker receipts and recovery admissions before the shared signal coordinator drains work.
|
|
81
|
+
shutdownManager.registerStopHandler(() => this.stop());
|
|
57
82
|
this.logDebug = this.logger.debugByType('MinionTickConsumerBackendService');
|
|
58
83
|
this.logger.setDebugTypeLogging('MinionTickConsumerBackendService', false);
|
|
59
84
|
}
|
|
60
85
|
|
|
61
86
|
public async start(input: {
|
|
87
|
+
/**
|
|
88
|
+
* The PLATFORM's id for the application this minion serves — the id its tick queue is named
|
|
89
|
+
* from and the id the crontabs-manager signs into every tick. It is addressing, never a row
|
|
90
|
+
* identity: the tick rows and stored payloads are anchored by the tracker and the payload
|
|
91
|
+
* storage to the application RECORD id, which they resolve themselves.
|
|
92
|
+
*/
|
|
62
93
|
applicationId: string;
|
|
63
94
|
minionName: string;
|
|
64
|
-
handler
|
|
95
|
+
handler?: MinionTickHandler;
|
|
96
|
+
recoveryPolicy?: Partial<ScheduledRecoveryPolicy>;
|
|
97
|
+
evidencePolicy?: ScheduledEvidencePolicy;
|
|
98
|
+
reconcile?: MinionTickReconciler;
|
|
65
99
|
}): Promise<void> {
|
|
66
100
|
if (this.isConsuming) {
|
|
67
101
|
throw new Error('MinionTickConsumerBackendService.start called while already consuming.');
|
|
68
102
|
}
|
|
69
103
|
|
|
70
|
-
const { applicationId, minionName, handler } = input;
|
|
104
|
+
const { applicationId, minionName, handler, recoveryPolicy, evidencePolicy, reconcile } = input;
|
|
105
|
+
// The scanner resolves only this runtime's locally authored handler. Durable input
|
|
106
|
+
// is trusted after first receipt; it is never turned back into a signed JWT.
|
|
107
|
+
const resolver: ScheduledExecutionResolver = async (job) => {
|
|
108
|
+
if (job.applicationId !== applicationId || job.jobType !== JobType.MINION_TICK || job.minionName !== minionName || !handler) return undefined;
|
|
109
|
+
const tick: MinionTickInvocation = {
|
|
110
|
+
jobId: job.jobId, applicationId, jobType: JobType.MINION_TICK, minionName,
|
|
111
|
+
inputDataJson: job.inputDataJson, createdAt: job.createdAt, scheduledAt: job.scheduledAt,
|
|
112
|
+
};
|
|
113
|
+
return {
|
|
114
|
+
policy: recoveryPolicy,
|
|
115
|
+
evidencePolicy,
|
|
116
|
+
execute: async (attempt) => {
|
|
117
|
+
const startedAt = Date.now();
|
|
118
|
+
await handler(tick, attempt);
|
|
119
|
+
return { jobId: job.jobId, status: JobStatus.COMPLETED, startedAt: new Date(startedAt), durationMs: Date.now() - startedAt, completedAt: new Date() };
|
|
120
|
+
},
|
|
121
|
+
reconcile: reconcile ? (attempt) => reconcile(tick, attempt) : undefined,
|
|
122
|
+
};
|
|
123
|
+
};
|
|
71
124
|
const queueName = QueueNamingUtils.buildMinionTickQueueName(applicationId, minionName);
|
|
72
125
|
|
|
73
126
|
// M5 — adapter readiness guard. The minion entrypoint calls this
|
|
@@ -104,11 +157,13 @@ export class MinionTickConsumerBackendService {
|
|
|
104
157
|
});
|
|
105
158
|
|
|
106
159
|
await this.adapter.consume(queueName, async (msg: ConsumeMessage, channel: Channel) => {
|
|
107
|
-
await this.handleMessage(msg, channel, minionName,
|
|
160
|
+
await this.handleMessage(msg, channel, applicationId, minionName, queueName, resolver);
|
|
108
161
|
});
|
|
109
162
|
|
|
110
163
|
this.consumingQueueName = queueName;
|
|
111
164
|
this.isConsuming = true;
|
|
165
|
+
this.runner.start(queueName, resolver);
|
|
166
|
+
this.reviewScope = { queueName, resolver };
|
|
112
167
|
this.logger.info('Minion tick consumer started', {
|
|
113
168
|
queueName,
|
|
114
169
|
applicationId,
|
|
@@ -117,6 +172,8 @@ export class MinionTickConsumerBackendService {
|
|
|
117
172
|
}
|
|
118
173
|
|
|
119
174
|
public async stop(): Promise<void> {
|
|
175
|
+
this.runner.stop();
|
|
176
|
+
this.reviewScope = undefined;
|
|
120
177
|
if (!this.isConsuming || !this.consumingQueueName) {
|
|
121
178
|
return;
|
|
122
179
|
}
|
|
@@ -125,6 +182,19 @@ export class MinionTickConsumerBackendService {
|
|
|
125
182
|
this.consumingQueueName = undefined;
|
|
126
183
|
}
|
|
127
184
|
|
|
185
|
+
/**
|
|
186
|
+
* Invoke from an application-owned operator operation AFTER authorization.
|
|
187
|
+
* Runtime app/queue ownership is derived here, never supplied by the caller;
|
|
188
|
+
* actor, reason and expected revision are persisted atomically with the decision.
|
|
189
|
+
*/
|
|
190
|
+
public async resolveTickReview(
|
|
191
|
+
input: Omit<Parameters<ScheduledExecutionRunner['resolveReview']>[0], 'queueName'>,
|
|
192
|
+
): Promise<void> {
|
|
193
|
+
const scope = this.reviewScope;
|
|
194
|
+
if (!scope) throw new Error('Minion tick review requires an initialized consumer');
|
|
195
|
+
await this.runner.resolveReview({ ...input, queueName: scope.queueName }, scope.resolver);
|
|
196
|
+
}
|
|
197
|
+
|
|
128
198
|
public isRunning(): boolean {
|
|
129
199
|
return this.isConsuming;
|
|
130
200
|
}
|
|
@@ -132,50 +202,46 @@ export class MinionTickConsumerBackendService {
|
|
|
132
202
|
private async handleMessage(
|
|
133
203
|
msg: ConsumeMessage,
|
|
134
204
|
channel: Channel,
|
|
205
|
+
applicationId: string,
|
|
135
206
|
expectedMinionName: string,
|
|
136
|
-
|
|
207
|
+
queueName: string,
|
|
208
|
+
resolver: ScheduledExecutionResolver,
|
|
137
209
|
): Promise<void> {
|
|
138
|
-
|
|
210
|
+
this.shutdownManager.jobStarted();
|
|
211
|
+
let admitted: JobExecution | undefined;
|
|
139
212
|
try {
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
expectedMinionName
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
213
|
+
let accepted = false;
|
|
214
|
+
try {
|
|
215
|
+
const tick = await this.scheduledMessageVerifier.verifyAndExtractMinionTick(msg.content.toString(), expectedMinionName);
|
|
216
|
+
// Signature/target checks do not establish receiving-application ownership.
|
|
217
|
+
// This comparison must precede every durable insert and handler resolution.
|
|
218
|
+
if (!tick || tick.applicationId !== applicationId || tick.minionName !== expectedMinionName) {
|
|
219
|
+
throw new Error('Scheduled minion token does not match the receiving application and minion');
|
|
220
|
+
}
|
|
221
|
+
const job = JobPayloadSchema.parse({
|
|
222
|
+
jobId: tick.jobId, jobType: JobType.MINION_TICK, applicationId: tick.applicationId,
|
|
223
|
+
minionName: tick.minionName, inputDataJson: tick.inputDataJson,
|
|
224
|
+
initiatedBy: JobInitiator.CRONTAB, createdAt: new Date(tick.iat * 1000), maxRetries: 0,
|
|
225
|
+
scheduledAt: tick.scheduledAt === undefined ? undefined : new Date(tick.scheduledAt),
|
|
153
226
|
});
|
|
154
|
-
|
|
155
|
-
|
|
227
|
+
// Durable receipt, then acknowledge, THEN run — a long tick must not hold later deliveries
|
|
228
|
+
// unreceived past their five-minute token (`ScheduledExecutionRunner.admit`).
|
|
229
|
+
admitted = await this.runner.admit(job, queueName, resolver);
|
|
230
|
+
accepted = true;
|
|
231
|
+
} catch (error) {
|
|
232
|
+
this.logger.error('Minion tick admission failed', { expectedMinionName, error: describeCaughtError(error) });
|
|
156
233
|
}
|
|
157
234
|
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
}
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
// verification rejection that is by far the most common cause here. Two masking
|
|
170
|
-
// layers stacked (this one and the verifier's) is why a tick that simply outlived
|
|
171
|
-
// its 5-minute token TTL looked identical to a forged one.
|
|
172
|
-
this.logger.error('Minion tick processing failed', {
|
|
173
|
-
expectedMinionName,
|
|
174
|
-
error: describeCaughtError(error),
|
|
175
|
-
});
|
|
176
|
-
// Per KD-6: scheduled jobs don't retry via queue — the next cron
|
|
177
|
-
// cycle re-publishes a fresh tick. NACK without requeue.
|
|
178
|
-
channel.nack(msg, false, false);
|
|
179
|
-
}
|
|
235
|
+
// Settlement failure is a broker problem, not a new business failure. Never
|
|
236
|
+
// NACK after an ACK throws or run the handler again from this catch boundary.
|
|
237
|
+
try {
|
|
238
|
+
if (accepted) channel.ack(msg);
|
|
239
|
+
else channel.nack(msg, false, false);
|
|
240
|
+
} catch (error) {
|
|
241
|
+
this.logger.error('Minion tick settlement failed', { expectedMinionName, error: describeCaughtError(error) });
|
|
242
|
+
}
|
|
243
|
+
} finally { this.shutdownManager.jobCompleted(); }
|
|
244
|
+
// Started only after the delivery is settled; the runner tracks the run's own activity.
|
|
245
|
+
if (admitted) this.runner.launch(admitted, resolver);
|
|
180
246
|
}
|
|
181
247
|
}
|
|
@@ -4,6 +4,7 @@ import type {
|
|
|
4
4
|
LoggerService,
|
|
5
5
|
StartApplicationBackendService,
|
|
6
6
|
} from '@wildo-ai/saas-backend-lib';
|
|
7
|
+
import { ScheduledReconciliationOutcome, ScheduledReplaySafety } from '@wildo-ai/saas-models';
|
|
7
8
|
import { PROVIDER_RUNTIME_ENV_KEYS } from '@wildo-ai/platform-config-lib';
|
|
8
9
|
import {
|
|
9
10
|
defineMinionInit,
|
|
@@ -39,6 +40,7 @@ function createTickConsumerStub() {
|
|
|
39
40
|
function createAppConfigServiceStub(applicationId = 'wonder-todos') {
|
|
40
41
|
return {
|
|
41
42
|
applicationId,
|
|
43
|
+
config: { scheduledEvidence: { minions: { marketingScrapper: { policyRef: 'worker-v1', retainForMs: 86400000, resultFields: [] } } } },
|
|
42
44
|
} as unknown as AppConfigurationService;
|
|
43
45
|
}
|
|
44
46
|
|
|
@@ -312,6 +314,8 @@ describe('StartMinionApplicationBackendService', () => {
|
|
|
312
314
|
} as unknown as StartApplicationBackendService;
|
|
313
315
|
const { service, tickConsumer } = instantiate(delegate);
|
|
314
316
|
const onTick = vi.fn().mockResolvedValue(undefined);
|
|
317
|
+
const reconcileTick = vi.fn().mockResolvedValue({ outcome: ScheduledReconciliationOutcome.INCONCLUSIVE });
|
|
318
|
+
const tickRecovery = { replaySafety: ScheduledReplaySafety.RECONCILE, maxAttempts: 3 };
|
|
315
319
|
|
|
316
320
|
await service.initialize(defineMinionInit({
|
|
317
321
|
minionName: 'marketingScrapper',
|
|
@@ -320,6 +324,8 @@ describe('StartMinionApplicationBackendService', () => {
|
|
|
320
324
|
sharedResourceRelationships: [],
|
|
321
325
|
customServiceImplementations: [],
|
|
322
326
|
onTick,
|
|
327
|
+
tickRecovery,
|
|
328
|
+
reconcileTick,
|
|
323
329
|
}));
|
|
324
330
|
|
|
325
331
|
expect(tickConsumer.start).toHaveBeenCalledTimes(1);
|
|
@@ -336,19 +342,29 @@ describe('StartMinionApplicationBackendService', () => {
|
|
|
336
342
|
// contained system-access door and the minion's own name.
|
|
337
343
|
const registered = (tickConsumer.start as any).mock.calls[0][0].handler;
|
|
338
344
|
const token = { jobId: 'job-1', minionName: 'marketingScrapper', applicationId: 'wonder-todos' };
|
|
339
|
-
|
|
345
|
+
const attempt = { occurrenceId: token.jobId, attemptId: 'attempt-1', attemptNumber: 1 };
|
|
346
|
+
await registered(token, attempt);
|
|
340
347
|
|
|
341
348
|
expect(onTick).toHaveBeenCalledTimes(1);
|
|
342
349
|
expect(onTick).toHaveBeenCalledWith(token, expect.objectContaining({
|
|
343
350
|
minionName: 'marketingScrapper',
|
|
344
351
|
systemAccess: expect.anything(),
|
|
352
|
+
scheduledAttempt: attempt,
|
|
345
353
|
}));
|
|
346
354
|
|
|
355
|
+
const startInput = (tickConsumer.start as any).mock.calls[0][0];
|
|
356
|
+
expect(startInput.recoveryPolicy).toEqual(tickRecovery);
|
|
357
|
+
expect(startInput.evidencePolicy).toEqual({ policyRef: 'worker-v1', retainForMs: 86400000, resultFields: [] });
|
|
358
|
+
await startInput.reconcile(token, attempt);
|
|
359
|
+
expect(reconcileTick).toHaveBeenCalledWith(token, expect.objectContaining({ scheduledAttempt: attempt, systemAccess: expect.anything() }));
|
|
360
|
+
expect(initialize.mock.calls[0][0]).not.toHaveProperty('tickRecovery');
|
|
361
|
+
expect(initialize.mock.calls[0][0]).not.toHaveProperty('reconcileTick');
|
|
362
|
+
|
|
347
363
|
await service.shutdown();
|
|
348
364
|
expect(tickConsumer.stop).toHaveBeenCalledTimes(1);
|
|
349
365
|
});
|
|
350
366
|
|
|
351
|
-
it('
|
|
367
|
+
it('subscribes without inventing a successful handler when onTick is absent', async () => {
|
|
352
368
|
const initialize = vi.fn().mockResolvedValue(undefined);
|
|
353
369
|
const delegate = {
|
|
354
370
|
initialize,
|
|
@@ -367,7 +383,7 @@ describe('StartMinionApplicationBackendService', () => {
|
|
|
367
383
|
|
|
368
384
|
expect(tickConsumer.start).toHaveBeenCalledTimes(1);
|
|
369
385
|
const firstCall = (tickConsumer.start as unknown as { mock: { calls: Array<[{ handler: unknown }]> } }).mock.calls[0];
|
|
370
|
-
expect(firstCall?.[0].handler).
|
|
386
|
+
expect(firstCall?.[0].handler).toBeUndefined();
|
|
371
387
|
});
|
|
372
388
|
});
|
|
373
389
|
|
|
@@ -3,6 +3,7 @@ import type {
|
|
|
3
3
|
ProviderRuntimeArtifact,
|
|
4
4
|
SystemAccessBackendService,
|
|
5
5
|
} from '@wildo-ai/saas-backend-lib';
|
|
6
|
+
import type { ScheduledAttemptContext, ScheduledRecoveryPolicy, ScheduledReconciliationResult } from '@wildo-ai/saas-models';
|
|
6
7
|
import type { MinionTickHandler } from '../queue/minion-tick-consumer.backend.service';
|
|
7
8
|
import type { OwnScheduleReaderBackendService } from './own-schedule-reader.backend.service';
|
|
8
9
|
|
|
@@ -23,6 +24,8 @@ export interface MinionProviderRuntimeArtifact {
|
|
|
23
24
|
* which would assert, across a package boundary, the very thing nobody checked.
|
|
24
25
|
*/
|
|
25
26
|
providerRuntimeImports?: ProviderRuntimeArtifact['providerRuntimeImports'];
|
|
27
|
+
/** Mirrored for the same reason: the projection narrows and forwards it (#1672). */
|
|
28
|
+
providerRuntimeAugmentations?: ProviderRuntimeArtifact['providerRuntimeAugmentations'];
|
|
26
29
|
providerGraphDigest?: ProviderRuntimeArtifact['providerGraphDigest'];
|
|
27
30
|
lockfileSectionDigest?: ProviderRuntimeArtifact['lockfileSectionDigest'];
|
|
28
31
|
artifactKind: 'minions';
|
|
@@ -52,7 +55,10 @@ export interface MinionProviderRuntimeArtifact {
|
|
|
52
55
|
* system-access door would have allowed it, and a read the ACL permits is still
|
|
53
56
|
* refused if it cannot be recorded.
|
|
54
57
|
*/
|
|
58
|
+
/** @wildo_source:part:start minions.tick-contract facet:layer:backend facet:family:minion facet:audience:app-developer */
|
|
55
59
|
export interface MinionTickContext {
|
|
60
|
+
/** Stable occurrence and fresh attempt identities for effect idempotency and reconciliation. */
|
|
61
|
+
scheduledAttempt: ScheduledAttemptContext;
|
|
56
62
|
/**
|
|
57
63
|
* The sanctioned privileged-access door, already contained by this minion's
|
|
58
64
|
* `resourceAccess` whitelist.
|
|
@@ -82,28 +88,38 @@ export interface MinionTickContext {
|
|
|
82
88
|
}
|
|
83
89
|
|
|
84
90
|
/**
|
|
85
|
-
* Tick handler invoked
|
|
86
|
-
*
|
|
91
|
+
* Tick handler invoked for each admitted execution attempt of an authenticated
|
|
92
|
+
* occurrence addressed to this minion. Duplicate deliveries do not create new attempts.
|
|
87
93
|
*/
|
|
88
94
|
export type MinionTickAuthorHandler = (
|
|
89
95
|
token: Parameters<MinionTickHandler>[0],
|
|
90
96
|
context: MinionTickContext,
|
|
91
97
|
) => Promise<void>;
|
|
92
98
|
|
|
99
|
+
/** Reconcile durable/provider evidence of an earlier attempt; uncertainty must remain inconclusive. */
|
|
100
|
+
export type MinionTickReconciliationHandler = (
|
|
101
|
+
token: Parameters<MinionTickHandler>[0],
|
|
102
|
+
context: MinionTickContext,
|
|
103
|
+
) => Promise<ScheduledReconciliationResult>;
|
|
104
|
+
|
|
93
105
|
export type MinionInitializationConfig = Omit<ProjectedMinionInitializationConfig, 'providerRuntime'> & {
|
|
94
106
|
providerRuntime?: ProviderRuntimeArtifact | MinionProviderRuntimeArtifact;
|
|
95
107
|
/**
|
|
96
|
-
* Tick handler
|
|
97
|
-
*
|
|
108
|
+
* Tick handler for admitted execution attempts addressed to this minion,
|
|
109
|
+
* receiving the framework-supplied {@link MinionTickContext}. Recovery follows
|
|
110
|
+
* the locally declared tickRecovery contract, never broker redelivery alone.
|
|
98
111
|
*
|
|
99
112
|
* Minion authors typically route the tick into the locally-hosted
|
|
100
113
|
* `flowsActorsRegistry` (per Q9 LOCKED — per-minion-local actor
|
|
101
114
|
* catalog). If omitted, the consumer still subscribes to the
|
|
102
|
-
* per-minion queue but
|
|
103
|
-
*
|
|
104
|
-
* registered any tick-handling logic yet.
|
|
115
|
+
* per-minion queue but refuses execution. A missing handler must never
|
|
116
|
+
* be recorded as successful work.
|
|
105
117
|
*/
|
|
106
118
|
onTick?: MinionTickAuthorHandler;
|
|
119
|
+
/** Locally authored effect-safety contract. Undeclared handlers default to manual review after uncertainty. */
|
|
120
|
+
tickRecovery?: Partial<ScheduledRecoveryPolicy>;
|
|
121
|
+
/** Required for reconciliation-based recovery; runs with the same contained access surface as onTick. */
|
|
122
|
+
reconcileTick?: MinionTickReconciliationHandler;
|
|
107
123
|
};
|
|
108
124
|
|
|
109
125
|
export function defineMinionInit(
|
|
@@ -111,3 +127,4 @@ export function defineMinionInit(
|
|
|
111
127
|
): MinionInitializationConfig {
|
|
112
128
|
return config;
|
|
113
129
|
}
|
|
130
|
+
/** @wildo_source:part:end minions.tick-contract */
|