@wildo-ai/saas-minions-lib 1.1.11 → 1.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esm/base-classes/services/services/minion-scoped-services-registry-handler.backend.service.d.ts.map +1 -1
- package/dist/esm/base-classes/services/services/minion-scoped-services-registry-handler.backend.service.js.map +1 -1
- package/dist/esm/index.d.ts.map +1 -1
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/inject/models/container.d.ts.map +1 -1
- package/dist/esm/inject/models/container.js.map +1 -1
- package/dist/esm/inject/models/service-types.d.ts.map +1 -1
- package/dist/esm/inject/models/service-types.js.map +1 -1
- package/dist/esm/runtime/queue/minion-tick-consumer.backend.service.d.ts +10 -1
- package/dist/esm/runtime/queue/minion-tick-consumer.backend.service.d.ts.map +1 -1
- package/dist/esm/runtime/queue/minion-tick-consumer.backend.service.js +12 -1
- package/dist/esm/runtime/queue/minion-tick-consumer.backend.service.js.map +1 -1
- package/dist/esm/runtime/services/define-minion-init.d.ts +6 -3
- package/dist/esm/runtime/services/define-minion-init.d.ts.map +1 -1
- package/dist/esm/runtime/services/define-minion-init.js.map +1 -1
- package/dist/esm/runtime/services/minion-liveness-heartbeat.d.ts +57 -0
- package/dist/esm/runtime/services/minion-liveness-heartbeat.d.ts.map +1 -0
- package/dist/esm/runtime/services/minion-liveness-heartbeat.js +125 -0
- package/dist/esm/runtime/services/minion-liveness-heartbeat.js.map +1 -0
- package/dist/esm/runtime/services/own-schedule-reader.backend.service.d.ts.map +1 -1
- package/dist/esm/runtime/services/own-schedule-reader.backend.service.js.map +1 -1
- package/dist/esm/runtime/services/start-minion-application.backend.service.d.ts +14 -0
- package/dist/esm/runtime/services/start-minion-application.backend.service.d.ts.map +1 -1
- package/dist/esm/runtime/services/start-minion-application.backend.service.js +58 -8
- package/dist/esm/runtime/services/start-minion-application.backend.service.js.map +1 -1
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/package.json +6 -6
- package/src/runtime/queue/__tests__/minion-tick-consumer.backend.service.test.ts +16 -1
- package/src/runtime/queue/minion-tick-consumer.backend.service.ts +13 -1
- package/src/runtime/services/__tests__/minion-liveness-heartbeat.test.ts +116 -0
- package/src/runtime/services/__tests__/start-minion-application.backend.service.test.ts +149 -2
- package/src/runtime/services/define-minion-init.ts +6 -3
- package/src/runtime/services/minion-liveness-heartbeat.ts +126 -0
- package/src/runtime/services/start-minion-application.backend.service.ts +63 -8
|
@@ -10,8 +10,9 @@ import {
|
|
|
10
10
|
type SystemAccessBackendService,
|
|
11
11
|
SAAS_SERVICE_TYPES,
|
|
12
12
|
type StartApplicationBackendService, describeCaughtError } from '@wildo-ai/saas-backend-lib';
|
|
13
|
-
import { ErrorCustomMessageReference, ErrorType, type ScheduledRecoveryPolicy } from '@wildo-ai/saas-models';
|
|
14
|
-
import { assertProviderRuntimeEnvInSync } from '@wildo-ai/platform-config-lib';
|
|
13
|
+
import { ErrorCustomMessageReference, ErrorType, narrowRuntimeProviderDataResidency, type ScheduledRecoveryPolicy } from '@wildo-ai/saas-models';
|
|
14
|
+
import { assertProviderRuntimeEnvInSync, readMinionLivenessHeartbeatEnv } from '@wildo-ai/platform-config-lib';
|
|
15
|
+
import { MinionLivenessHeartbeat } from './minion-liveness-heartbeat';
|
|
15
16
|
import { MinionScopedServicesRegistryHandlerBackendService } from '../../base-classes/services/services/minion-scoped-services-registry-handler.backend.service';
|
|
16
17
|
import { SAAS_MINIONS_SERVICE_TYPES } from '../../inject/models/service-types';
|
|
17
18
|
import type { OwnScheduleReaderBackendService } from './own-schedule-reader.backend.service';
|
|
@@ -36,6 +37,7 @@ type MinionRuntimeConfig = NonNullable<NonNullable<AppConfiguration['minions']>[
|
|
|
36
37
|
@injectable()
|
|
37
38
|
export class StartMinionApplicationBackendService {
|
|
38
39
|
private tickConsumerStarted = false;
|
|
40
|
+
private livenessHeartbeat: MinionLivenessHeartbeat | undefined;
|
|
39
41
|
|
|
40
42
|
constructor(
|
|
41
43
|
@inject(SAAS_SERVICE_TYPES.StartApplicationService)
|
|
@@ -55,6 +57,14 @@ export class StartMinionApplicationBackendService {
|
|
|
55
57
|
) {}
|
|
56
58
|
|
|
57
59
|
public async initialize(config: MinionInitializationConfig): Promise<void> {
|
|
60
|
+
// Read (and so validate) the heartbeat settings before anything connects, and clear any heartbeat a PREVIOUS
|
|
61
|
+
// container left behind. In k8s `/tmp` is an `emptyDir`, which outlives a container restart inside the pod: a
|
|
62
|
+
// minion that crashed without removing its file would hand the next container a fresh-looking heartbeat, the
|
|
63
|
+
// startupProbe would pass at once, and the livenessProbe would then kill a slow boot. Cleared here, the
|
|
64
|
+
// startupProbe waits for THIS process's first beat (#1859).
|
|
65
|
+
const livenessHeartbeatSettings = readMinionLivenessHeartbeatEnv(process.env);
|
|
66
|
+
if (livenessHeartbeatSettings) await MinionLivenessHeartbeat.clear(livenessHeartbeatSettings, this.logger);
|
|
67
|
+
|
|
58
68
|
const {
|
|
59
69
|
minionName,
|
|
60
70
|
serviceIdentifier,
|
|
@@ -72,7 +82,7 @@ export class StartMinionApplicationBackendService {
|
|
|
72
82
|
//
|
|
73
83
|
// Attribution only: this grants nothing. A minion's operations are governed by its
|
|
74
84
|
// role and its `resourceAccess` whitelist; its cross-tenant reach comes from the
|
|
75
|
-
// system-access door.
|
|
85
|
+
// system-access door.
|
|
76
86
|
//
|
|
77
87
|
// assurance-control: WILDO.AUDIT.SECURITY_EVENT — privileged access by a non-human
|
|
78
88
|
// background runtime is attributable to that runtime.
|
|
@@ -185,9 +195,26 @@ export class StartMinionApplicationBackendService {
|
|
|
185
195
|
// operation but the subscription is harmless (no message ⇒ no
|
|
186
196
|
// dispatch) and keeps the runtime contract symmetric across modes.
|
|
187
197
|
await this.startTickConsumer(minionName, onTick, tickRecovery, reconcileTick);
|
|
198
|
+
await this.startLivenessHeartbeat(minionName, livenessHeartbeatSettings);
|
|
188
199
|
}
|
|
189
200
|
|
|
201
|
+
/**
|
|
202
|
+
* For a host that shuts the minion down itself. A container's SIGTERM does not come through here: it reaches
|
|
203
|
+
* `WorkerShutdownManager`, whose stop handler stops the tick consumer, and the heartbeat then stops beating on
|
|
204
|
+
* its own because `isReceiving()` turns false. The leftover file is cleared by the next container's
|
|
205
|
+
* `initialize()`.
|
|
206
|
+
*/
|
|
190
207
|
public async shutdown(): Promise<void> {
|
|
208
|
+
// First, so a stopping minion stops claiming to be alive before its consumer is torn down.
|
|
209
|
+
if (this.livenessHeartbeat) {
|
|
210
|
+
try {
|
|
211
|
+
await this.livenessHeartbeat.stop();
|
|
212
|
+
} catch (error) {
|
|
213
|
+
this.logger.warn('Failed to stop the minion liveness heartbeat cleanly', { error: describeCaughtError(error) });
|
|
214
|
+
} finally {
|
|
215
|
+
this.livenessHeartbeat = undefined;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
191
218
|
if (this.tickConsumerStarted) {
|
|
192
219
|
try {
|
|
193
220
|
await this.tickConsumer.stop();
|
|
@@ -235,12 +262,10 @@ export class StartMinionApplicationBackendService {
|
|
|
235
262
|
? async (token, scheduledAttempt) => onTick(token, { ...context, scheduledAttempt })
|
|
236
263
|
: undefined;
|
|
237
264
|
|
|
265
|
+
// Only this minion's own policy is read here. Whether a policy names an undeclared minion is the backend's diagnosis
|
|
266
|
+
// (`JobConsumerBackendService`): a minion is answered its OWN declaration only (#2954), so every sibling would read
|
|
267
|
+
// as undeclared from here.
|
|
238
268
|
const evidencePolicies = this.appConfigService.config?.scheduledEvidence?.minions;
|
|
239
|
-
for (const configuredName of Object.keys(evidencePolicies ?? {})) {
|
|
240
|
-
if (!Object.hasOwn(this.appConfigService.config.minions ?? {}, configuredName)) {
|
|
241
|
-
this.logger.warn('Scheduled evidence policy names an undeclared minion; no evidence will be collected for this key', { minionName: configuredName });
|
|
242
|
-
}
|
|
243
|
-
}
|
|
244
269
|
await this.tickConsumer.start({
|
|
245
270
|
applicationId: this.appConfigService.applicationId,
|
|
246
271
|
minionName,
|
|
@@ -254,6 +279,27 @@ export class StartMinionApplicationBackendService {
|
|
|
254
279
|
this.tickConsumerStarted = true;
|
|
255
280
|
}
|
|
256
281
|
|
|
282
|
+
/**
|
|
283
|
+
* The producer half of the file liveness probe (#1859). Started only AFTER the tick consumer subscribed, so
|
|
284
|
+
* the first heartbeat means "able to receive work", which is what the `startupProbe` waits for. The settings
|
|
285
|
+
* come from the env the container renderers stamp from `wildo.minion.config.ts > liveness`, the same parsed
|
|
286
|
+
* value the probe itself is rendered from; a host process gets no such env and writes nothing.
|
|
287
|
+
*/
|
|
288
|
+
private async startLivenessHeartbeat(
|
|
289
|
+
minionName: string,
|
|
290
|
+
settings: ReturnType<typeof readMinionLivenessHeartbeatEnv>,
|
|
291
|
+
): Promise<void> {
|
|
292
|
+
if (!settings) return;
|
|
293
|
+
const heartbeat = new MinionLivenessHeartbeat(
|
|
294
|
+
settings,
|
|
295
|
+
() => this.startApplicationService.isApplicationInitialized() && this.tickConsumer.isReceiving(),
|
|
296
|
+
this.logger,
|
|
297
|
+
);
|
|
298
|
+
await heartbeat.start();
|
|
299
|
+
this.livenessHeartbeat = heartbeat;
|
|
300
|
+
this.logger.info('Minion liveness heartbeat wired to the tick consumer', { minionName });
|
|
301
|
+
}
|
|
302
|
+
|
|
257
303
|
private getMinionRuntimeConfig(
|
|
258
304
|
appConfig: MinionAwareAppConfiguration,
|
|
259
305
|
minionName: string,
|
|
@@ -390,6 +436,9 @@ export class StartMinionApplicationBackendService {
|
|
|
390
436
|
// sibling's opt-in must not load a package into this process.
|
|
391
437
|
const projectedAugmentations = Object.entries(providerRuntime.providerRuntimeAugmentations ?? {})
|
|
392
438
|
.filter(([providerRef]) => projectedProviderRefs.includes(providerRef));
|
|
439
|
+
// Narrowed with the refs too (#3513): the region a sibling's provider is bound to is not this host's to apply.
|
|
440
|
+
const projectedProcessingRegions = Object.entries(providerRuntime.providerProcessingRegions ?? {})
|
|
441
|
+
.filter(([providerRef]) => projectedProviderRefs.includes(providerRef));
|
|
393
442
|
|
|
394
443
|
return {
|
|
395
444
|
__AUTOGENERATED__: providerRuntime.__AUTOGENERATED__,
|
|
@@ -399,6 +448,12 @@ export class StartMinionApplicationBackendService {
|
|
|
399
448
|
providerRefs: projectedProviderRefs,
|
|
400
449
|
providerRuntimeImports: projectedProviderRuntimeImports,
|
|
401
450
|
...(projectedAugmentations.length === 0 ? {} : { providerRuntimeAugmentations: Object.fromEntries(projectedAugmentations) }),
|
|
451
|
+
...(projectedProcessingRegions.length === 0 ? {} : { providerProcessingRegions: Object.fromEntries(projectedProcessingRegions) }),
|
|
452
|
+
// Narrowed with the refs too (#2294), and never dropped: a projection that lost it would load this host's providers
|
|
453
|
+
// without the residency the application declared, which is the one thing the loader's re-check exists to prevent.
|
|
454
|
+
...(providerRuntime.providerDataResidency === undefined
|
|
455
|
+
? {}
|
|
456
|
+
: { providerDataResidency: narrowRuntimeProviderDataResidency(providerRuntime.providerDataResidency, projectedProviderRefs) }),
|
|
402
457
|
providerGraphDigest: providerRuntime.providerGraphDigest,
|
|
403
458
|
lockfileSectionDigest: providerRuntime.lockfileSectionDigest,
|
|
404
459
|
artifactKind: 'backend',
|