@wildo-ai/saas-minions-lib 1.1.11 → 1.1.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/dist/esm/base-classes/services/services/minion-scoped-services-registry-handler.backend.service.d.ts.map +1 -1
  2. package/dist/esm/base-classes/services/services/minion-scoped-services-registry-handler.backend.service.js.map +1 -1
  3. package/dist/esm/index.d.ts.map +1 -1
  4. package/dist/esm/index.js.map +1 -1
  5. package/dist/esm/inject/models/container.d.ts.map +1 -1
  6. package/dist/esm/inject/models/container.js.map +1 -1
  7. package/dist/esm/inject/models/service-types.d.ts.map +1 -1
  8. package/dist/esm/inject/models/service-types.js.map +1 -1
  9. package/dist/esm/runtime/queue/minion-tick-consumer.backend.service.d.ts +10 -1
  10. package/dist/esm/runtime/queue/minion-tick-consumer.backend.service.d.ts.map +1 -1
  11. package/dist/esm/runtime/queue/minion-tick-consumer.backend.service.js +12 -1
  12. package/dist/esm/runtime/queue/minion-tick-consumer.backend.service.js.map +1 -1
  13. package/dist/esm/runtime/services/define-minion-init.d.ts +6 -3
  14. package/dist/esm/runtime/services/define-minion-init.d.ts.map +1 -1
  15. package/dist/esm/runtime/services/define-minion-init.js.map +1 -1
  16. package/dist/esm/runtime/services/minion-liveness-heartbeat.d.ts +57 -0
  17. package/dist/esm/runtime/services/minion-liveness-heartbeat.d.ts.map +1 -0
  18. package/dist/esm/runtime/services/minion-liveness-heartbeat.js +125 -0
  19. package/dist/esm/runtime/services/minion-liveness-heartbeat.js.map +1 -0
  20. package/dist/esm/runtime/services/own-schedule-reader.backend.service.d.ts.map +1 -1
  21. package/dist/esm/runtime/services/own-schedule-reader.backend.service.js.map +1 -1
  22. package/dist/esm/runtime/services/start-minion-application.backend.service.d.ts +14 -0
  23. package/dist/esm/runtime/services/start-minion-application.backend.service.d.ts.map +1 -1
  24. package/dist/esm/runtime/services/start-minion-application.backend.service.js +58 -8
  25. package/dist/esm/runtime/services/start-minion-application.backend.service.js.map +1 -1
  26. package/dist/tsconfig.build.tsbuildinfo +1 -1
  27. package/package.json +6 -6
  28. package/src/runtime/queue/__tests__/minion-tick-consumer.backend.service.test.ts +16 -1
  29. package/src/runtime/queue/minion-tick-consumer.backend.service.ts +13 -1
  30. package/src/runtime/services/__tests__/minion-liveness-heartbeat.test.ts +116 -0
  31. package/src/runtime/services/__tests__/start-minion-application.backend.service.test.ts +149 -2
  32. package/src/runtime/services/define-minion-init.ts +6 -3
  33. package/src/runtime/services/minion-liveness-heartbeat.ts +126 -0
  34. package/src/runtime/services/start-minion-application.backend.service.ts +63 -8
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@wildo-ai/saas-minions-lib",
3
- "version": "1.1.11",
3
+ "version": "1.1.13",
4
4
  "description": "Node minion runtime host layer for Wildo SaaS applications",
5
5
  "license": "SEE LICENSE IN LICENSE",
6
6
  "author": "Wildo AI",
@@ -30,19 +30,19 @@
30
30
  "test:watch": "vitest"
31
31
  },
32
32
  "dependencies": {
33
- "@wildo-ai/platform-config-lib": "1.1.11",
34
- "@wildo-ai/saas-backend-lib": "1.1.11",
35
- "@wildo-ai/saas-models": "1.1.11",
33
+ "@wildo-ai/platform-config-lib": "1.1.13",
34
+ "@wildo-ai/saas-backend-lib": "1.1.13",
35
+ "@wildo-ai/saas-models": "1.1.13",
36
36
  "reflect-metadata": "0.2.2"
37
37
  },
38
38
  "devDependencies": {
39
39
  "@types/node": "^24.0.15",
40
40
  "@types/amqplib": "^0.10.8",
41
- "rimraf": "^6.0.1",
41
+ "rimraf": "^6.1.4",
42
42
  "tsc-alias": "1.8.16",
43
43
  "tsc-watch": "7.2.1",
44
44
  "typescript": "5.9.3",
45
- "vitest": "^1.6.1"
45
+ "vitest": "^4.1.12"
46
46
  },
47
47
  "peerDependencies": {
48
48
  "inversify": "^8.1.0"
@@ -33,7 +33,7 @@ const onTime = (value: unknown) => ({ verdict: ScheduledDeliveryVerdict.ADMISSIB
33
33
  function fixture() {
34
34
  let deliver: ((message: ConsumeMessage, channel: Channel) => Promise<void>) | undefined;
35
35
  const adapter = {
36
- isConnected: () => true,
36
+ isConnected: vi.fn(() => true),
37
37
  getTopologyChannel: vi.fn().mockResolvedValue({ assertQueue: vi.fn() }),
38
38
  consume: vi.fn(async (_queue: string, callback: typeof deliver) => { deliver = callback; }),
39
39
  cancelConsumer: vi.fn(),
@@ -71,6 +71,21 @@ beforeEach(() => {
71
71
  });
72
72
 
73
73
  describe('minion scheduled coordination boundary', () => {
74
+ it('reports it can receive only while subscribed AND the broker connection is up (the liveness question, #1859)', async () => {
75
+ const f = fixture();
76
+ expect(f.service.isReceiving(), 'before start').toBe(false);
77
+ await f.service.start({ applicationId: 'app-a', minionName: 'worker' });
78
+ expect(f.service.isReceiving(), 'subscribed and connected').toBe(true);
79
+ // The adapter does not reconnect: after a broker restart the consumer is still "running" and will never
80
+ // receive another tick. That is the state the liveness heartbeat must end in a restart.
81
+ f.adapter.isConnected.mockReturnValue(false);
82
+ expect(f.service.isRunning()).toBe(true);
83
+ expect(f.service.isReceiving(), 'connection closed').toBe(false);
84
+ f.adapter.isConnected.mockReturnValue(true);
85
+ await f.service.stop();
86
+ expect(f.service.isReceiving(), 'stopped').toBe(false);
87
+ });
88
+
74
89
  it('joins the shared shutdown drain and stops admissions before draining', async () => {
75
90
  const f = fixture();
76
91
  const activity = coordinator.construct.mock.calls[0][3] as { started(): void; completed(): void };
@@ -35,7 +35,7 @@ export type MinionTickReconciler = (tick: MinionTickInvocation, attempt: Schedul
35
35
  * Minion-side consumer for the per-minion tick queue
36
36
  * (`wildo.queue.<appId>.minion-tick.<minionName>`, see `QueueNamingUtils.buildMinionTickQueueName`).
37
37
  *
38
- * Per KD-12 of `.claude/plans/minions-and-workers-implementation.md`:
38
+ * Contract:
39
39
  * - Subscribes only to the queue dedicated to THIS minion (no shared
40
40
  * `scheduled` queue subscription, so the backend-api and minion don't
41
41
  * compete on cron tick deliveries).
@@ -204,6 +204,18 @@ export class MinionTickConsumerBackendService {
204
204
  return this.isConsuming;
205
205
  }
206
206
 
207
+ /**
208
+ * Whether this consumer can still RECEIVE a tick: subscribed, and the broker connection is up.
209
+ *
210
+ * `isRunning()` alone is not enough. The RabbitMQ adapter does not reconnect, so after a broker restart the
211
+ * consumer stays "running" over a closed connection and no tick will ever arrive again. This is what the
212
+ * minion liveness heartbeat asks before each beat (#1859), so that state ends in a restart instead of a
213
+ * minion that is alive and deaf.
214
+ */
215
+ public isReceiving(): boolean {
216
+ return this.isConsuming && this.adapter.isConnected();
217
+ }
218
+
207
219
  private async handleMessage(
208
220
  msg: ConsumeMessage,
209
221
  channel: Channel,
@@ -0,0 +1,116 @@
1
+ import { afterEach, describe, expect, it, vi } from 'vitest';
2
+ import { spawnSync } from 'node:child_process';
3
+ import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
4
+ import { tmpdir } from 'node:os';
5
+ import { join } from 'node:path';
6
+ import { buildMinionLivenessProbeCommand, MinionLivenessSchema } from '@wildo-ai/platform-config-lib';
7
+ import { MinionLivenessHeartbeat } from '../minion-liveness-heartbeat';
8
+
9
+ /**
10
+ * The producer half of the minion file liveness contract (#1859). Every assertion about freshness is made by
11
+ * EXECUTING the probe command the k8s and compose lanes render, not by reading the file and agreeing with
12
+ * ourselves about what it means.
13
+ */
14
+ const tempDirs: string[] = [];
15
+ afterEach(() => {
16
+ vi.useRealTimers();
17
+ for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true });
18
+ });
19
+
20
+ function setup(intervalSec = 30) {
21
+ const dir = mkdtempSync(join(tmpdir(), 'wildo-minion-heartbeat-'));
22
+ tempDirs.push(dir);
23
+ const filePath = join(dir, 'wildo-minion-alive');
24
+ const liveness = MinionLivenessSchema.parse({ filePath, heartbeat: { intervalSec }, staleAfterSec: intervalSec * 3 });
25
+ const probe = buildMinionLivenessProbeCommand(liveness);
26
+ const probePasses = (): boolean => spawnSync(probe[0]!, probe.slice(1)).status === 0;
27
+ const state = { alive: true, skewMs: 0 };
28
+ const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
29
+ const heartbeat = new MinionLivenessHeartbeat(
30
+ { filePath, intervalSec },
31
+ () => state.alive,
32
+ logger,
33
+ () => Date.now() + state.skewMs,
34
+ );
35
+ return { filePath, probePasses, state, logger, heartbeat };
36
+ }
37
+
38
+ describe('MinionLivenessHeartbeat', () => {
39
+ it('writes a heartbeat the rendered probe accepts as soon as it starts', async () => {
40
+ const { heartbeat, probePasses, filePath } = setup();
41
+ expect(probePasses(), 'no heartbeat before start').toBe(false);
42
+ await heartbeat.start();
43
+ expect(readFileSync(filePath, 'utf8')).toMatch(/^\d+\n$/);
44
+ expect(probePasses()).toBe(true);
45
+ await heartbeat.stop();
46
+ });
47
+
48
+ it('keeps beating on its interval while the work loop is alive', async () => {
49
+ vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval'] });
50
+ const { heartbeat, filePath, state } = setup(30);
51
+ // Start with a heartbeat that is already old, as if the last beat happened 200 s ago...
52
+ state.skewMs = -200_000;
53
+ await heartbeat.start();
54
+ const first = Number(readFileSync(filePath, 'utf8'));
55
+ // ...then let the clock catch up and the interval fire: the TIMER, not a test call, refreshes it.
56
+ state.skewMs = 0;
57
+ await vi.advanceTimersByTimeAsync(29_000);
58
+ expect(Number(readFileSync(filePath, 'utf8')), 'nothing before the interval elapses').toBe(first);
59
+ await vi.advanceTimersByTimeAsync(1_000);
60
+ // The beat it fired does real file I/O, so wait for it on the REAL clock (`vi.waitFor` keeps its own timers).
61
+ await vi.waitFor(() => expect(Number(readFileSync(filePath, 'utf8'))).toBeGreaterThanOrEqual(first + 199));
62
+ await heartbeat.stop();
63
+ });
64
+
65
+ it('goes stale when the work loop stops, and fresh again when it recovers', async () => {
66
+ const { heartbeat, probePasses, state, logger } = setup(30);
67
+ // The last heartbeat the loop wrote was 95 s ago — past the 90 s window.
68
+ state.skewMs = -95_000;
69
+ await heartbeat.beat();
70
+ expect(probePasses(), 'a 95 s old heartbeat').toBe(false);
71
+
72
+ // The loop is no longer receiving (consumer stopped, or the broker connection closed): beats keep firing
73
+ // and write nothing, so the heartbeat stays stale and the kubelet restarts the minion.
74
+ state.alive = false;
75
+ state.skewMs = 0;
76
+ await heartbeat.beat();
77
+ await heartbeat.beat();
78
+ expect(probePasses(), 'beats while the loop is dead refresh nothing').toBe(false);
79
+ // Warned once on the transition, not once per beat.
80
+ expect(logger.warn).toHaveBeenCalledTimes(1);
81
+
82
+ state.alive = true;
83
+ await heartbeat.beat();
84
+ expect(probePasses(), 'the loop is back').toBe(true);
85
+ });
86
+
87
+ it('creates a missing sub-directory of its writable mount', async () => {
88
+ const dir = mkdtempSync(join(tmpdir(), 'wildo-minion-heartbeat-'));
89
+ tempDirs.push(dir);
90
+ const filePath = join(dir, 'liveness', 'alive');
91
+ const heartbeat = new MinionLivenessHeartbeat({ filePath, intervalSec: 30 }, () => true, { info: vi.fn(), warn: vi.fn(), error: vi.fn() });
92
+ await heartbeat.beat();
93
+ expect(readFileSync(filePath, 'utf8')).toMatch(/^\d+\n$/);
94
+ });
95
+
96
+ it('removes the heartbeat on stop, so a stopping minion stops reading as alive at once', async () => {
97
+ const { heartbeat, probePasses, filePath } = setup();
98
+ await heartbeat.start();
99
+ expect(probePasses()).toBe(true);
100
+ await heartbeat.stop();
101
+ expect(existsSync(filePath)).toBe(false);
102
+ expect(probePasses()).toBe(false);
103
+ expect(heartbeat.isRunning()).toBe(false);
104
+ });
105
+
106
+ it('reports an unwritable heartbeat instead of throwing, so the probe — not a crash — decides', async () => {
107
+ const dir = mkdtempSync(join(tmpdir(), 'wildo-minion-heartbeat-'));
108
+ tempDirs.push(dir);
109
+ const logger = { info: vi.fn(), warn: vi.fn(), error: vi.fn() };
110
+ // A regular FILE where the heartbeat's directory should be: neither mkdir nor the write can succeed.
111
+ writeFileSync(join(dir, 'blocker'), '');
112
+ const heartbeat = new MinionLivenessHeartbeat({ filePath: join(dir, 'blocker', 'alive'), intervalSec: 30 }, () => true, logger);
113
+ await expect(heartbeat.beat()).resolves.toBeUndefined();
114
+ expect(logger.error).toHaveBeenCalledWith('Could not write the minion liveness heartbeat', expect.anything());
115
+ });
116
+ });
@@ -4,8 +4,11 @@ import type {
4
4
  LoggerService,
5
5
  StartApplicationBackendService,
6
6
  } from '@wildo-ai/saas-backend-lib';
7
- import { ScheduledReconciliationOutcome, ScheduledReplaySafety } from '@wildo-ai/saas-models';
8
- import { PROVIDER_RUNTIME_ENV_KEYS } from '@wildo-ai/platform-config-lib';
7
+ import { ProviderDeploymentModel, ScheduledReconciliationOutcome, ScheduledReplaySafety } from '@wildo-ai/saas-models';
8
+ import { MINION_LIVENESS_HEARTBEAT_ENV_KEYS, PROVIDER_RUNTIME_ENV_KEYS } from '@wildo-ai/platform-config-lib';
9
+ import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
10
+ import { tmpdir } from 'node:os';
11
+ import { join } from 'node:path';
9
12
  import {
10
13
  defineMinionInit,
11
14
  type MinionProviderRuntimeArtifact,
@@ -34,6 +37,7 @@ function createTickConsumerStub() {
34
37
  start: vi.fn().mockResolvedValue(undefined),
35
38
  stop: vi.fn().mockResolvedValue(undefined),
36
39
  isRunning: vi.fn().mockReturnValue(false),
40
+ isReceiving: vi.fn().mockReturnValue(true),
37
41
  } as unknown as MinionTickConsumerBackendService;
38
42
  }
39
43
 
@@ -174,6 +178,57 @@ describe('StartMinionApplicationBackendService', () => {
174
178
  expect(initialize.mock.calls[0]?.[0]?.afterResourcesRegistryInitialized).toEqual(expect.any(Function));
175
179
  });
176
180
 
181
+ it('forwards the declared data residency narrowed to this minion\'s providers, so its loader re-applies it (#2294)', async () => {
182
+ const initialize = vi.fn().mockResolvedValue(undefined);
183
+ const delegate = {
184
+ initialize,
185
+ shutdown: vi.fn().mockResolvedValue(undefined),
186
+ isApplicationInitialized: vi.fn().mockReturnValue(true),
187
+ } as unknown as StartApplicationBackendService;
188
+ const { service } = instantiate(delegate);
189
+ const providerRuntime: MinionProviderRuntimeArtifact = {
190
+ __AUTOGENERATED__: 'AUTO-GENERATED FILE. DO NOT EDIT. Regenerate with `wildo config sync`.',
191
+ generatedBy: 'wildo provider-runtime-artifact',
192
+ generatedAt: '2026-11-09T00:00:00.000Z',
193
+ sourceDigest: 'sha256:test',
194
+ providerRefs: ['openai', 'anthropic'],
195
+ artifactKind: 'minions',
196
+ modules: { declaredModules: {}, moduleIds: [], moduleIdsByService: {} },
197
+ providerDataResidency: {
198
+ territories: ['eu'],
199
+ choices: {
200
+ openai: { deploymentModel: ProviderDeploymentModel.VENDOR_HOSTED, processingRegion: 'eu' },
201
+ anthropic: { deploymentModel: ProviderDeploymentModel.VENDOR_HOSTED, processingRegion: 'us' },
202
+ },
203
+ },
204
+ scopes: {
205
+ marketingScrapper: {
206
+ providers: { openai: { engineCapabilities: [], providerCapabilities: ['LLM_CHAT'], protocols: ['LLM_PROVIDER'] } },
207
+ selection: {},
208
+ },
209
+ otherMinion: {
210
+ providers: { anthropic: { engineCapabilities: [], providerCapabilities: ['LLM_CHAT'], protocols: ['LLM_PROVIDER'] } },
211
+ selection: {},
212
+ },
213
+ },
214
+ };
215
+ withMatchingProviderRuntimeEnv({ refs: ['openai'], digest: providerRuntime.sourceDigest });
216
+
217
+ await service.initialize(defineMinionInit({
218
+ minionName: 'marketingScrapper',
219
+ runtimeProfile: ApplicationRuntimeProfile.MINION,
220
+ sharedResourceConfigurations: {} as any,
221
+ sharedResourceRelationships: [],
222
+ customServiceImplementations: [],
223
+ providerRuntime,
224
+ }));
225
+
226
+ expect(initialize.mock.calls[0]?.[0]?.providerRuntime?.providerDataResidency).toEqual({
227
+ territories: ['eu'],
228
+ choices: { openai: { deploymentModel: ProviderDeploymentModel.VENDOR_HOSTED, processingRegion: 'eu' } },
229
+ });
230
+ });
231
+
177
232
  it('skips boot-time env reconciliation when the per-host scope is empty (no false-positive throw on union providerRefs)', async () => {
178
233
  // A multi-scope artifact whose union `providerRefs` includes
179
234
  // sibling refs the typed module preserves uncollapsed. The
@@ -435,4 +490,96 @@ describe('StartMinionApplicationBackendService — runtime identity (D9)', () =>
435
490
 
436
491
  expect(runtimePrincipal.describeForAudit().serviceIdentifier).toBe('minions.marketingScrapper.blue');
437
492
  });
493
+
494
+ describe('liveness heartbeat (#1859)', () => {
495
+ const dirs: string[] = [];
496
+ afterEach(() => {
497
+ vi.unstubAllEnvs();
498
+ for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true });
499
+ });
500
+ const minionInit = () => defineMinionInit({
501
+ minionName: 'marketingScrapper',
502
+ runtimeProfile: ApplicationRuntimeProfile.MINION,
503
+ sharedResourceConfigurations: {} as any,
504
+ sharedResourceRelationships: [],
505
+ customServiceImplementations: [],
506
+ });
507
+
508
+ it('starts the heartbeat the container env asks for once the tick consumer is subscribed, and removes it on shutdown', async () => {
509
+ const dir = mkdtempSync(join(tmpdir(), 'wildo-start-minion-heartbeat-'));
510
+ dirs.push(dir);
511
+ const filePath = join(dir, 'wildo-minion-alive');
512
+ vi.stubEnv(MINION_LIVENESS_HEARTBEAT_ENV_KEYS.FILE_PATH, filePath);
513
+ vi.stubEnv(MINION_LIVENESS_HEARTBEAT_ENV_KEYS.INTERVAL_SEC, '30');
514
+ const { service, tickConsumer } = instantiateWithDelegate();
515
+
516
+ await service.initialize(minionInit());
517
+
518
+ expect(tickConsumer.start).toHaveBeenCalledTimes(1);
519
+ // The heartbeat asks the CONSUMER whether it can still receive, not merely whether the process is up.
520
+ expect(tickConsumer.isReceiving).toHaveBeenCalled();
521
+ expect(readFileSync(filePath, 'utf8')).toMatch(/^\d+\n$/);
522
+
523
+ await service.shutdown();
524
+ expect(existsSync(filePath)).toBe(false);
525
+ });
526
+
527
+ it('clears a heartbeat a previous container left behind BEFORE booting, so the startupProbe waits for this process', async () => {
528
+ const dir = mkdtempSync(join(tmpdir(), 'wildo-start-minion-heartbeat-'));
529
+ dirs.push(dir);
530
+ const filePath = join(dir, 'wildo-minion-alive');
531
+ // Fresh-looking: written by a crashed predecessor a moment ago. `/tmp` is an emptyDir that survives the restart.
532
+ writeFileSync(filePath, `${Math.floor(Date.now() / 1000)}\n`);
533
+ vi.stubEnv(MINION_LIVENESS_HEARTBEAT_ENV_KEYS.FILE_PATH, filePath);
534
+ vi.stubEnv(MINION_LIVENESS_HEARTBEAT_ENV_KEYS.INTERVAL_SEC, '30');
535
+ const existedWhenBootBegan: boolean[] = [];
536
+ const delegate = {
537
+ initialize: vi.fn(async () => { existedWhenBootBegan.push(existsSync(filePath)); }),
538
+ shutdown: vi.fn().mockResolvedValue(undefined),
539
+ isApplicationInitialized: vi.fn().mockReturnValue(true),
540
+ } as unknown as StartApplicationBackendService;
541
+ const { service, tickConsumer } = instantiate(delegate);
542
+ vi.mocked(tickConsumer.isReceiving).mockReturnValue(false);
543
+
544
+ await service.initialize(minionInit());
545
+ expect(existedWhenBootBegan).toEqual([false]);
546
+ expect(existsSync(filePath), 'not re-created while the consumer cannot receive').toBe(false);
547
+ await service.shutdown();
548
+ });
549
+
550
+ it('refuses a malformed heartbeat env before connecting anything', async () => {
551
+ vi.stubEnv(MINION_LIVENESS_HEARTBEAT_ENV_KEYS.FILE_PATH, '/tmp/wildo-minion-alive');
552
+ vi.stubEnv(MINION_LIVENESS_HEARTBEAT_ENV_KEYS.INTERVAL_SEC, '30s');
553
+ const delegate = {
554
+ initialize: vi.fn().mockResolvedValue(undefined),
555
+ shutdown: vi.fn().mockResolvedValue(undefined),
556
+ isApplicationInitialized: vi.fn().mockReturnValue(true),
557
+ } as unknown as StartApplicationBackendService;
558
+ const { service } = instantiate(delegate);
559
+ await expect(service.initialize(minionInit())).rejects.toThrow(MINION_LIVENESS_HEARTBEAT_ENV_KEYS.INTERVAL_SEC);
560
+ expect(delegate.initialize).not.toHaveBeenCalled();
561
+ });
562
+
563
+ it('withholds the first heartbeat when the consumer cannot receive', async () => {
564
+ const dir = mkdtempSync(join(tmpdir(), 'wildo-start-minion-heartbeat-'));
565
+ dirs.push(dir);
566
+ const filePath = join(dir, 'wildo-minion-alive');
567
+ vi.stubEnv(MINION_LIVENESS_HEARTBEAT_ENV_KEYS.FILE_PATH, filePath);
568
+ vi.stubEnv(MINION_LIVENESS_HEARTBEAT_ENV_KEYS.INTERVAL_SEC, '30');
569
+ const { service, tickConsumer } = instantiateWithDelegate();
570
+ vi.mocked(tickConsumer.isReceiving).mockReturnValue(false);
571
+
572
+ await service.initialize(minionInit());
573
+ expect(existsSync(filePath)).toBe(false);
574
+ await service.shutdown();
575
+ });
576
+
577
+ it('writes nothing for a host process, which gets no heartbeat env', async () => {
578
+ vi.stubEnv(MINION_LIVENESS_HEARTBEAT_ENV_KEYS.FILE_PATH, '');
579
+ const { service, tickConsumer } = instantiateWithDelegate();
580
+ await service.initialize(minionInit());
581
+ expect(tickConsumer.isReceiving).not.toHaveBeenCalled();
582
+ await service.shutdown();
583
+ });
584
+ });
438
585
  });
@@ -26,6 +26,10 @@ export interface MinionProviderRuntimeArtifact {
26
26
  providerRuntimeImports?: ProviderRuntimeArtifact['providerRuntimeImports'];
27
27
  /** Mirrored for the same reason: the projection narrows and forwards it (#1672). */
28
28
  providerRuntimeAugmentations?: ProviderRuntimeArtifact['providerRuntimeAugmentations'];
29
+ /** Mirrored for the same reason: the projection narrows and forwards it (#3513). */
30
+ providerProcessingRegions?: ProviderRuntimeArtifact['providerProcessingRegions'];
31
+ /** Mirrored for the same reason: the projection narrows and forwards it (#2294), and the loader re-applies it. */
32
+ providerDataResidency?: ProviderRuntimeArtifact['providerDataResidency'];
29
33
  providerGraphDigest?: ProviderRuntimeArtifact['providerGraphDigest'];
30
34
  lockfileSectionDigest?: ProviderRuntimeArtifact['lockfileSectionDigest'];
31
35
  artifactKind: 'minions';
@@ -44,12 +48,11 @@ export interface MinionProviderRuntimeArtifact {
44
48
  * minion startup rebinds the services registry to the scoped decorator before any
45
49
  * tick is delivered.
46
50
  *
47
- * So a minion's effective authority is the INTERSECTION of two independent doors
48
- * (see `.claude/plans/minion-runtime-scope.md` D6):
51
+ * So a minion's effective authority is the INTERSECTION of two independent doors:
49
52
  *
50
53
  * - `resourceAccess` — operator-declared containment, enforced by the DI proxy;
51
54
  * - the system-access door — the sanctioned per-tenant-authorization bypass, which
52
- * for a minion is fail-closed on audit (D9 + OQ-5b): audited, or refused.
55
+ * for a minion is fail-closed on audit: audited, or refused.
53
56
  *
54
57
  * Neither subsumes the other. A resource the ACL forbids is refused even though the
55
58
  * system-access door would have allowed it, and a read the ACL permits is still
@@ -0,0 +1,126 @@
1
+ import { mkdir, rename, unlink, writeFile } from 'node:fs/promises';
2
+ import { dirname } from 'node:path';
3
+ import { describeCaughtError, type LoggerService } from '@wildo-ai/saas-backend-lib';
4
+ import { encodeMinionLivenessHeartbeat, type MinionLivenessHeartbeatSettings } from '@wildo-ai/platform-config-lib';
5
+
6
+ /**
7
+ * The producer half of a minion's file liveness contract (#1859).
8
+ *
9
+ * The probe half — the `exec` command the k8s `livenessProbe`/`startupProbe` and the compose `healthcheck`
10
+ * run, and their cadence — is rendered from the same parsed `wildo.minion.config.ts > liveness` by
11
+ * `minion-liveness.contracts.ts` in `@wildo-ai/platform-config-lib`, which also renders the env this
12
+ * writer is configured from. Read that module's header for how `heartbeat.intervalSec`, `staleAfterSec`,
13
+ * `periodSeconds` and `failureThreshold` relate.
14
+ *
15
+ * ## What a heartbeat asserts
16
+ *
17
+ * Not that the process exists: a process whose broker connection has closed is alive and useless, because
18
+ * the RabbitMQ adapter does not reconnect and the minion will never receive another tick. So every beat asks
19
+ * `isWorkLoopAlive()` first — wired by minion startup to "the tick consumer is subscribed and the broker
20
+ * connection is up" — and writes nothing when the answer is no. The file then goes stale, the probe fails,
21
+ * and the kubelet restarts the minion into a fresh connection, which is the only recovery there is.
22
+ *
23
+ * A long-running tick does not stop the beats. Ticks run detached from receipt and the beat is a timer, so
24
+ * the heartbeat stays fresh for as long as the handler awaits I/O; only a handler that blocks the event loop
25
+ * for longer than the stale window is restarted.
26
+ *
27
+ * ## The write is atomic
28
+ *
29
+ * Each beat writes a sibling temporary file and renames it over the heartbeat, so the probe never reads a
30
+ * half-written number. A torn read would fail the probe's numeric check, and a few of those in a row would
31
+ * restart a healthy minion.
32
+ */
33
+ export class MinionLivenessHeartbeat {
34
+ private timer: ReturnType<typeof setInterval> | undefined;
35
+ private inFlight: Promise<void> | undefined;
36
+ /** Logged on the transition only, so a stopped loop does not write one warning per interval. */
37
+ private reportedNotAlive = false;
38
+
39
+ constructor(
40
+ private readonly settings: MinionLivenessHeartbeatSettings,
41
+ private readonly isWorkLoopAlive: () => boolean,
42
+ private readonly logger: Pick<LoggerService, 'info' | 'warn' | 'error'>,
43
+ private readonly now: () => number = Date.now,
44
+ ) {}
45
+
46
+ /**
47
+ * Remove a heartbeat left by an earlier process, so nothing can read as alive before this one has beaten.
48
+ * Missing is the normal case.
49
+ */
50
+ public static async clear(
51
+ settings: MinionLivenessHeartbeatSettings,
52
+ logger: Pick<LoggerService, 'warn'>,
53
+ ): Promise<void> {
54
+ try {
55
+ await unlink(settings.filePath);
56
+ } catch (error) {
57
+ if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
58
+ logger.warn('Could not remove the minion liveness heartbeat', { filePath: settings.filePath, error: describeCaughtError(error) });
59
+ }
60
+ }
61
+ }
62
+
63
+ /** Writes the first beat now, then one every `intervalSec`. Idempotent. */
64
+ public async start(): Promise<void> {
65
+ if (this.timer) return;
66
+ this.timer = setInterval(() => {
67
+ void this.beat();
68
+ }, this.settings.intervalSec * 1000);
69
+ // The heartbeat must never be what keeps a process up: a minion with nothing else holding its event loop
70
+ // open has stopped working, and exiting is the honest outcome.
71
+ this.timer.unref();
72
+ await this.beat();
73
+ this.logger.info('Minion liveness heartbeat started', {
74
+ filePath: this.settings.filePath,
75
+ intervalSec: this.settings.intervalSec,
76
+ });
77
+ }
78
+
79
+ /**
80
+ * Stops beating and removes the heartbeat, so a stopping minion reads as not alive at once rather than
81
+ * one stale window later.
82
+ */
83
+ public async stop(): Promise<void> {
84
+ if (this.timer) clearInterval(this.timer);
85
+ this.timer = undefined;
86
+ await this.inFlight;
87
+ await MinionLivenessHeartbeat.clear(this.settings, this.logger);
88
+ }
89
+
90
+ public isRunning(): boolean {
91
+ return this.timer !== undefined;
92
+ }
93
+
94
+ /** One beat. Exposed for tests; the timer calls it. Never overlaps itself. */
95
+ public async beat(): Promise<void> {
96
+ if (this.inFlight) return this.inFlight;
97
+ this.inFlight = this.writeIfAlive().finally(() => {
98
+ this.inFlight = undefined;
99
+ });
100
+ return this.inFlight;
101
+ }
102
+
103
+ private async writeIfAlive(): Promise<void> {
104
+ if (!this.isWorkLoopAlive()) {
105
+ if (!this.reportedNotAlive) {
106
+ this.reportedNotAlive = true;
107
+ this.logger.warn('Minion work loop is not receiving ticks; the liveness heartbeat is withheld so the probe fails', {
108
+ filePath: this.settings.filePath,
109
+ });
110
+ }
111
+ return;
112
+ }
113
+ this.reportedNotAlive = false;
114
+ const temporaryPath = `${this.settings.filePath}.${process.pid}.tmp`;
115
+ try {
116
+ // The schema keeps the file inside a writable mount; a sub-directory of one may not exist yet.
117
+ await mkdir(dirname(this.settings.filePath), { recursive: true });
118
+ await writeFile(temporaryPath, encodeMinionLivenessHeartbeat(this.now()), 'utf8');
119
+ await rename(temporaryPath, this.settings.filePath);
120
+ } catch (error) {
121
+ // Nothing to retry here: an unwritable heartbeat (a read-only rootfs without its `/tmp` mount) fails the
122
+ // probe, and the restart that follows is the correct outcome for a minion that cannot prove it is alive.
123
+ this.logger.error('Could not write the minion liveness heartbeat', { filePath: this.settings.filePath, error: describeCaughtError(error) });
124
+ }
125
+ }
126
+ }