@serfab/cadre-host 1.10.0 → 1.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +423 -423
- package/dist/ui/assets/index-C6nELNfC.js.map +1 -1
- package/package.json +4 -4
- package/src/__tests__/global-setup.ts +58 -58
- package/src/__tests__/orchestrator-handle-contract.ts +134 -134
- package/src/__tests__/orchestrator-node-identity.test.ts +203 -203
- package/src/__tests__/orchestrator-ports.test.ts +143 -143
- package/src/__tests__/orchestrator-teardown.ts +65 -65
- package/src/__tests__/orchestrator.test.ts +1063 -1063
- package/src/bin/host.ts +1357 -1357
- package/src/donation/__tests__/fake-orchestrator.test.ts +173 -173
- package/src/donation/donation-store.ts +176 -176
- package/src/donation/grant-service.ts +252 -252
- package/src/donation/index.ts +65 -65
- package/src/donation/types.ts +260 -260
- package/src/orchestrator/host-process-orchestrator.ts +1260 -1260
- package/src/orchestrator/port-allocator.ts +73 -73
- package/src/server/__tests__/grants-admin-route.test.ts +171 -171
- package/src/server/events/types.ts +24 -24
- package/src/server/index.ts +267 -267
- package/src/server/routes/grants-admin.ts +74 -74
|
@@ -1,143 +1,143 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* The pure port-set logic behind `HostProcessOrchestrator`: the helpers that
|
|
3
|
-
* allocate, hold, release and reuse one node's set of ports, and the listen
|
|
4
|
-
* addresses a child is started with. No child processes — the behaviour these
|
|
5
|
-
* feed is exercised against a stub child in `orchestrator.test.ts`.
|
|
6
|
-
*/
|
|
7
|
-
|
|
8
|
-
import { describe, expect, it } from 'vitest';
|
|
9
|
-
|
|
10
|
-
import { PortAllocator } from '@serfab/cadre-provider';
|
|
11
|
-
|
|
12
|
-
import { childListenAddrs } from '../orchestrator/host-process-orchestrator.js';
|
|
13
|
-
import {
|
|
14
|
-
allocateNodePorts,
|
|
15
|
-
releaseNodePorts,
|
|
16
|
-
reserveNodePorts,
|
|
17
|
-
reusedNodePorts,
|
|
18
|
-
} from '../orchestrator/port-allocator.js';
|
|
19
|
-
import type { NodePorts } from '../orchestrator/types.js';
|
|
20
|
-
|
|
21
|
-
/**
|
|
22
|
-
* A port set as `state.json` hands it back from a build that predates the `ws` key:
|
|
23
|
-
* the type says every key is present, the data does not.
|
|
24
|
-
*/
|
|
25
|
-
function portsWithoutWs(ports: Omit<NodePorts, 'ws'>): NodePorts {
|
|
26
|
-
return ports as NodePorts;
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
describe('allocateNodePorts', () => {
|
|
30
|
-
it('allocates the five ports in a fixed order, the newest key last', () => {
|
|
31
|
-
const a = new PortAllocator(10000, 10010);
|
|
32
|
-
expect(allocateNodePorts(a)).toEqual({ health: 10000, metrics: 10001, p2p: 10002, admin: 10003, ws: 10004 });
|
|
33
|
-
});
|
|
34
|
-
|
|
35
|
-
// A partial set left reserved would leak from a bounded range on every failed
|
|
36
|
-
// provision — the whole reason this helper exists rather than five allocates.
|
|
37
|
-
it('is all-or-nothing: an exhausted range releases everything it took', () => {
|
|
38
|
-
const a = new PortAllocator(13000, 13002);
|
|
39
|
-
expect(() => allocateNodePorts(a)).toThrow(/No available ports/);
|
|
40
|
-
expect(a.has(13000)).toBe(false);
|
|
41
|
-
expect(a.has(13001)).toBe(false);
|
|
42
|
-
expect(a.has(13002)).toBe(false);
|
|
43
|
-
});
|
|
44
|
-
|
|
45
|
-
// Four ports used to be one node's worth. A range sized for that now fails on the
|
|
46
|
-
// fifth, and must neither hand back a set missing `ws` nor keep the four it took.
|
|
47
|
-
it('fails cleanly on a range with room for four ports but not five', () => {
|
|
48
|
-
const a = new PortAllocator(13000, 13003);
|
|
49
|
-
expect(() => allocateNodePorts(a)).toThrow(/No available ports/);
|
|
50
|
-
for (let port = 13000; port <= 13003; port++) expect(a.has(port)).toBe(false);
|
|
51
|
-
});
|
|
52
|
-
|
|
53
|
-
// The owner node's p2p port is pinned by the NAT mapping, not allocated.
|
|
54
|
-
it('honours an override and reserves it', () => {
|
|
55
|
-
const a = new PortAllocator(10000, 10010);
|
|
56
|
-
expect(allocateNodePorts(a, { p2p: 10005 })).toEqual({
|
|
57
|
-
health: 10000,
|
|
58
|
-
metrics: 10001,
|
|
59
|
-
p2p: 10005,
|
|
60
|
-
admin: 10002,
|
|
61
|
-
ws: 10003,
|
|
62
|
-
});
|
|
63
|
-
// Reserved, so no later allocation can hand it out again.
|
|
64
|
-
expect(a.has(10005)).toBe(true);
|
|
65
|
-
});
|
|
66
|
-
|
|
67
|
-
// An out-of-range override is the production case for the owner node: its
|
|
68
|
-
// libp2p port is chosen by NAT config, not by the allocator, so reserving it
|
|
69
|
-
// is a documented no-op rather than an error.
|
|
70
|
-
it('accepts an override outside the managed range without reserving it', () => {
|
|
71
|
-
const a = new PortAllocator(10000, 10010);
|
|
72
|
-
expect(allocateNodePorts(a, { p2p: 40000 })).toMatchObject({ p2p: 40000, health: 10000 });
|
|
73
|
-
expect(a.has(40000)).toBe(false);
|
|
74
|
-
});
|
|
75
|
-
|
|
76
|
-
// A re-spawn passes its previous set as overrides; every one must come back exactly.
|
|
77
|
-
it('returns a full set of overrides exactly, allocating nothing', () => {
|
|
78
|
-
const a = new PortAllocator(10000, 10010);
|
|
79
|
-
const previous = { health: 10006, metrics: 10007, p2p: 10008, admin: 10009, ws: 10010 };
|
|
80
|
-
expect(allocateNodePorts(a, previous)).toEqual(previous);
|
|
81
|
-
expect(a.allocate()).toBe(10000);
|
|
82
|
-
});
|
|
83
|
-
});
|
|
84
|
-
|
|
85
|
-
describe('reserveNodePorts / releaseNodePorts', () => {
|
|
86
|
-
it('hold and then release every key of the set', () => {
|
|
87
|
-
const a = new PortAllocator(10000, 10010);
|
|
88
|
-
const ports = { health: 10000, metrics: 10001, p2p: 10002, admin: 10003, ws: 10004 };
|
|
89
|
-
reserveNodePorts(a, ports);
|
|
90
|
-
for (const port of Object.values(ports)) expect(a.has(port)).toBe(true);
|
|
91
|
-
releaseNodePorts(a, ports);
|
|
92
|
-
for (const port of Object.values(ports)) expect(a.has(port)).toBe(false);
|
|
93
|
-
});
|
|
94
|
-
|
|
95
|
-
// Rehydrating, or dropping, a `state.json` handle written before `ws` existed.
|
|
96
|
-
it('skip a key the set lacks', () => {
|
|
97
|
-
const a = new PortAllocator(10000, 10010);
|
|
98
|
-
const ports = portsWithoutWs({ health: 10000, metrics: 10001, p2p: 10002, admin: 10003 });
|
|
99
|
-
reserveNodePorts(a, ports);
|
|
100
|
-
expect(a.has(undefined as unknown as number)).toBe(false);
|
|
101
|
-
expect(a.allocate()).toBe(10004);
|
|
102
|
-
releaseNodePorts(a, ports);
|
|
103
|
-
expect(a.has(10000)).toBe(false);
|
|
104
|
-
expect(a.has(10004)).toBe(true);
|
|
105
|
-
});
|
|
106
|
-
});
|
|
107
|
-
|
|
108
|
-
describe('reusedNodePorts', () => {
|
|
109
|
-
const previous = { health: 10005, metrics: 10006, p2p: 10007, admin: 10008, ws: 10009 };
|
|
110
|
-
|
|
111
|
-
it("returns the dropped handle's whole port set", () => {
|
|
112
|
-
expect(reusedNodePorts([{ ports: previous }])).toEqual(previous);
|
|
113
|
-
});
|
|
114
|
-
|
|
115
|
-
it('returns nothing when nothing was dropped, so every port is allocated fresh', () => {
|
|
116
|
-
expect(reusedNodePorts([])).toEqual({});
|
|
117
|
-
});
|
|
118
|
-
|
|
119
|
-
// A handle persisted before `ws` existed: its other ports come back, `ws` is new.
|
|
120
|
-
it('omits a key the dropped handle lacks', () => {
|
|
121
|
-
const legacy = portsWithoutWs({ health: 10005, metrics: 10006, p2p: 10007, admin: 10008 });
|
|
122
|
-
const reused = reusedNodePorts([{ ports: legacy }]);
|
|
123
|
-
expect(reused).toEqual({ health: 10005, metrics: 10006, p2p: 10007, admin: 10008 });
|
|
124
|
-
expect('ws' in reused).toBe(false);
|
|
125
|
-
|
|
126
|
-
const a = new PortAllocator(10000, 10010);
|
|
127
|
-
expect(allocateNodePorts(a, reused)).toEqual({ health: 10005, metrics: 10006, p2p: 10007, admin: 10008, ws: 10000 });
|
|
128
|
-
});
|
|
129
|
-
|
|
130
|
-
it('takes the last handle when a state holding duplicates dropped several', () => {
|
|
131
|
-
const older = { health: 10000, metrics: 10001, p2p: 10002, admin: 10003, ws: 10004 };
|
|
132
|
-
expect(reusedNodePorts([{ ports: older }, { ports: previous }])).toEqual(previous);
|
|
133
|
-
});
|
|
134
|
-
});
|
|
135
|
-
|
|
136
|
-
describe('childListenAddrs', () => {
|
|
137
|
-
it('names a TCP listener on the p2p port and a WebSocket listener on the ws port, on every interface', () => {
|
|
138
|
-
expect(childListenAddrs({ p2p: 10002, ws: 10004 })).toEqual([
|
|
139
|
-
'/ip4/0.0.0.0/tcp/10002',
|
|
140
|
-
'/ip4/0.0.0.0/tcp/10004/ws',
|
|
141
|
-
]);
|
|
142
|
-
});
|
|
143
|
-
});
|
|
1
|
+
/**
|
|
2
|
+
* The pure port-set logic behind `HostProcessOrchestrator`: the helpers that
|
|
3
|
+
* allocate, hold, release and reuse one node's set of ports, and the listen
|
|
4
|
+
* addresses a child is started with. No child processes — the behaviour these
|
|
5
|
+
* feed is exercised against a stub child in `orchestrator.test.ts`.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { describe, expect, it } from 'vitest';
|
|
9
|
+
|
|
10
|
+
import { PortAllocator } from '@serfab/cadre-provider';
|
|
11
|
+
|
|
12
|
+
import { childListenAddrs } from '../orchestrator/host-process-orchestrator.js';
|
|
13
|
+
import {
|
|
14
|
+
allocateNodePorts,
|
|
15
|
+
releaseNodePorts,
|
|
16
|
+
reserveNodePorts,
|
|
17
|
+
reusedNodePorts,
|
|
18
|
+
} from '../orchestrator/port-allocator.js';
|
|
19
|
+
import type { NodePorts } from '../orchestrator/types.js';
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* A port set as `state.json` hands it back from a build that predates the `ws` key:
|
|
23
|
+
* the type says every key is present, the data does not.
|
|
24
|
+
*/
|
|
25
|
+
function portsWithoutWs(ports: Omit<NodePorts, 'ws'>): NodePorts {
|
|
26
|
+
return ports as NodePorts;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
describe('allocateNodePorts', () => {
|
|
30
|
+
it('allocates the five ports in a fixed order, the newest key last', () => {
|
|
31
|
+
const a = new PortAllocator(10000, 10010);
|
|
32
|
+
expect(allocateNodePorts(a)).toEqual({ health: 10000, metrics: 10001, p2p: 10002, admin: 10003, ws: 10004 });
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
// A partial set left reserved would leak from a bounded range on every failed
|
|
36
|
+
// provision — the whole reason this helper exists rather than five allocates.
|
|
37
|
+
it('is all-or-nothing: an exhausted range releases everything it took', () => {
|
|
38
|
+
const a = new PortAllocator(13000, 13002);
|
|
39
|
+
expect(() => allocateNodePorts(a)).toThrow(/No available ports/);
|
|
40
|
+
expect(a.has(13000)).toBe(false);
|
|
41
|
+
expect(a.has(13001)).toBe(false);
|
|
42
|
+
expect(a.has(13002)).toBe(false);
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
// Four ports used to be one node's worth. A range sized for that now fails on the
|
|
46
|
+
// fifth, and must neither hand back a set missing `ws` nor keep the four it took.
|
|
47
|
+
it('fails cleanly on a range with room for four ports but not five', () => {
|
|
48
|
+
const a = new PortAllocator(13000, 13003);
|
|
49
|
+
expect(() => allocateNodePorts(a)).toThrow(/No available ports/);
|
|
50
|
+
for (let port = 13000; port <= 13003; port++) expect(a.has(port)).toBe(false);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
// The owner node's p2p port is pinned by the NAT mapping, not allocated.
|
|
54
|
+
it('honours an override and reserves it', () => {
|
|
55
|
+
const a = new PortAllocator(10000, 10010);
|
|
56
|
+
expect(allocateNodePorts(a, { p2p: 10005 })).toEqual({
|
|
57
|
+
health: 10000,
|
|
58
|
+
metrics: 10001,
|
|
59
|
+
p2p: 10005,
|
|
60
|
+
admin: 10002,
|
|
61
|
+
ws: 10003,
|
|
62
|
+
});
|
|
63
|
+
// Reserved, so no later allocation can hand it out again.
|
|
64
|
+
expect(a.has(10005)).toBe(true);
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
// An out-of-range override is the production case for the owner node: its
|
|
68
|
+
// libp2p port is chosen by NAT config, not by the allocator, so reserving it
|
|
69
|
+
// is a documented no-op rather than an error.
|
|
70
|
+
it('accepts an override outside the managed range without reserving it', () => {
|
|
71
|
+
const a = new PortAllocator(10000, 10010);
|
|
72
|
+
expect(allocateNodePorts(a, { p2p: 40000 })).toMatchObject({ p2p: 40000, health: 10000 });
|
|
73
|
+
expect(a.has(40000)).toBe(false);
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
// A re-spawn passes its previous set as overrides; every one must come back exactly.
|
|
77
|
+
it('returns a full set of overrides exactly, allocating nothing', () => {
|
|
78
|
+
const a = new PortAllocator(10000, 10010);
|
|
79
|
+
const previous = { health: 10006, metrics: 10007, p2p: 10008, admin: 10009, ws: 10010 };
|
|
80
|
+
expect(allocateNodePorts(a, previous)).toEqual(previous);
|
|
81
|
+
expect(a.allocate()).toBe(10000);
|
|
82
|
+
});
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
describe('reserveNodePorts / releaseNodePorts', () => {
|
|
86
|
+
it('hold and then release every key of the set', () => {
|
|
87
|
+
const a = new PortAllocator(10000, 10010);
|
|
88
|
+
const ports = { health: 10000, metrics: 10001, p2p: 10002, admin: 10003, ws: 10004 };
|
|
89
|
+
reserveNodePorts(a, ports);
|
|
90
|
+
for (const port of Object.values(ports)) expect(a.has(port)).toBe(true);
|
|
91
|
+
releaseNodePorts(a, ports);
|
|
92
|
+
for (const port of Object.values(ports)) expect(a.has(port)).toBe(false);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
// Rehydrating, or dropping, a `state.json` handle written before `ws` existed.
|
|
96
|
+
it('skip a key the set lacks', () => {
|
|
97
|
+
const a = new PortAllocator(10000, 10010);
|
|
98
|
+
const ports = portsWithoutWs({ health: 10000, metrics: 10001, p2p: 10002, admin: 10003 });
|
|
99
|
+
reserveNodePorts(a, ports);
|
|
100
|
+
expect(a.has(undefined as unknown as number)).toBe(false);
|
|
101
|
+
expect(a.allocate()).toBe(10004);
|
|
102
|
+
releaseNodePorts(a, ports);
|
|
103
|
+
expect(a.has(10000)).toBe(false);
|
|
104
|
+
expect(a.has(10004)).toBe(true);
|
|
105
|
+
});
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
describe('reusedNodePorts', () => {
|
|
109
|
+
const previous = { health: 10005, metrics: 10006, p2p: 10007, admin: 10008, ws: 10009 };
|
|
110
|
+
|
|
111
|
+
it("returns the dropped handle's whole port set", () => {
|
|
112
|
+
expect(reusedNodePorts([{ ports: previous }])).toEqual(previous);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
it('returns nothing when nothing was dropped, so every port is allocated fresh', () => {
|
|
116
|
+
expect(reusedNodePorts([])).toEqual({});
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
// A handle persisted before `ws` existed: its other ports come back, `ws` is new.
|
|
120
|
+
it('omits a key the dropped handle lacks', () => {
|
|
121
|
+
const legacy = portsWithoutWs({ health: 10005, metrics: 10006, p2p: 10007, admin: 10008 });
|
|
122
|
+
const reused = reusedNodePorts([{ ports: legacy }]);
|
|
123
|
+
expect(reused).toEqual({ health: 10005, metrics: 10006, p2p: 10007, admin: 10008 });
|
|
124
|
+
expect('ws' in reused).toBe(false);
|
|
125
|
+
|
|
126
|
+
const a = new PortAllocator(10000, 10010);
|
|
127
|
+
expect(allocateNodePorts(a, reused)).toEqual({ health: 10005, metrics: 10006, p2p: 10007, admin: 10008, ws: 10000 });
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
it('takes the last handle when a state holding duplicates dropped several', () => {
|
|
131
|
+
const older = { health: 10000, metrics: 10001, p2p: 10002, admin: 10003, ws: 10004 };
|
|
132
|
+
expect(reusedNodePorts([{ ports: older }, { ports: previous }])).toEqual(previous);
|
|
133
|
+
});
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
describe('childListenAddrs', () => {
|
|
137
|
+
it('names a TCP listener on the p2p port and a WebSocket listener on the ws port, on every interface', () => {
|
|
138
|
+
expect(childListenAddrs({ p2p: 10002, ws: 10004 })).toEqual([
|
|
139
|
+
'/ip4/0.0.0.0/tcp/10002',
|
|
140
|
+
'/ip4/0.0.0.0/tcp/10004/ws',
|
|
141
|
+
]);
|
|
142
|
+
});
|
|
143
|
+
});
|
|
@@ -1,65 +1,65 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Shared teardown for the test files that drive a real HostProcessOrchestrator
|
|
3
|
-
* against a stub child script.
|
|
4
|
-
*/
|
|
5
|
-
import type { HostProcessOrchestrator } from '../orchestrator/host-process-orchestrator.js';
|
|
6
|
-
import { isPidAlive } from '../orchestrator/pid-liveness.js';
|
|
7
|
-
import { decodeDockerId } from '../orchestrator/types.js';
|
|
8
|
-
|
|
9
|
-
interface StartedNode {
|
|
10
|
-
containerId: string;
|
|
11
|
-
pid: number;
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
/**
|
|
15
|
-
* Stop and remove every node the given orchestrators hold, then fail if any of
|
|
16
|
-
* their processes is still alive. Empties `orchestrators`.
|
|
17
|
-
*
|
|
18
|
-
* Nodes come from `listNodes()` rather than the state file: two orchestrators
|
|
19
|
-
* over one rootDir each rewrite the whole file from their own memory, so the
|
|
20
|
-
* file can omit a child that one of them still holds.
|
|
21
|
-
*/
|
|
22
|
-
export async function removeAllNodes(orchestrators: HostProcessOrchestrator[]): Promise<void> {
|
|
23
|
-
try {
|
|
24
|
-
const nodes: StartedNode[] = [];
|
|
25
|
-
const errors: string[] = [];
|
|
26
|
-
for (const orch of orchestrators) {
|
|
27
|
-
await removeNodesOf(orch, nodes, errors);
|
|
28
|
-
}
|
|
29
|
-
// NOTE: liveness is by pid alone, not the startup token. If the OS hands a dead child's pid to an
|
|
30
|
-
// unrelated process before this line, that process is reported and killed; if this ever produces a
|
|
31
|
-
// false failure, verify the token before counting a pid as a survivor.
|
|
32
|
-
const survivors = nodes.filter((n) => isPidAlive(n.pid));
|
|
33
|
-
if (survivors.length === 0) return;
|
|
34
|
-
survivors.forEach((n) => forceKill(n.pid));
|
|
35
|
-
throw new Error(describeSurvivors(survivors, errors));
|
|
36
|
-
} finally {
|
|
37
|
-
orchestrators.length = 0;
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
/** Keeps going past a failed removal so one failure does not strand the rest. */
|
|
42
|
-
async function removeNodesOf(orch: HostProcessOrchestrator, nodes: StartedNode[], errors: string[]): Promise<void> {
|
|
43
|
-
for (const node of orch.listNodes()) {
|
|
44
|
-
nodes.push({ containerId: node.id, pid: decodeDockerId(node.dockerId).pid });
|
|
45
|
-
try {
|
|
46
|
-
await orch.removeContainer(node.dockerId);
|
|
47
|
-
} catch (err) {
|
|
48
|
-
errors.push(`${node.id}: ${err instanceof Error ? err.message : String(err)}`);
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
function forceKill(pid: number): void {
|
|
54
|
-
try {
|
|
55
|
-
process.kill(pid, 'SIGKILL');
|
|
56
|
-
} catch (err) {
|
|
57
|
-
if ((err as NodeJS.ErrnoException).code !== 'ESRCH') throw err;
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
|
|
61
|
-
function describeSurvivors(survivors: StartedNode[], errors: string[]): string {
|
|
62
|
-
const names = survivors.map((n) => `${n.containerId} (pid ${n.pid})`).join(', ');
|
|
63
|
-
const causes = errors.length > 0 ? `; removeContainer errors: ${errors.join(' | ')}` : '';
|
|
64
|
-
return `test teardown left child processes running (now force-killed): ${names}${causes}`;
|
|
65
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* Shared teardown for the test files that drive a real HostProcessOrchestrator
|
|
3
|
+
* against a stub child script.
|
|
4
|
+
*/
|
|
5
|
+
import type { HostProcessOrchestrator } from '../orchestrator/host-process-orchestrator.js';
|
|
6
|
+
import { isPidAlive } from '../orchestrator/pid-liveness.js';
|
|
7
|
+
import { decodeDockerId } from '../orchestrator/types.js';
|
|
8
|
+
|
|
9
|
+
interface StartedNode {
|
|
10
|
+
containerId: string;
|
|
11
|
+
pid: number;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Stop and remove every node the given orchestrators hold, then fail if any of
|
|
16
|
+
* their processes is still alive. Empties `orchestrators`.
|
|
17
|
+
*
|
|
18
|
+
* Nodes come from `listNodes()` rather than the state file: two orchestrators
|
|
19
|
+
* over one rootDir each rewrite the whole file from their own memory, so the
|
|
20
|
+
* file can omit a child that one of them still holds.
|
|
21
|
+
*/
|
|
22
|
+
export async function removeAllNodes(orchestrators: HostProcessOrchestrator[]): Promise<void> {
|
|
23
|
+
try {
|
|
24
|
+
const nodes: StartedNode[] = [];
|
|
25
|
+
const errors: string[] = [];
|
|
26
|
+
for (const orch of orchestrators) {
|
|
27
|
+
await removeNodesOf(orch, nodes, errors);
|
|
28
|
+
}
|
|
29
|
+
// NOTE: liveness is by pid alone, not the startup token. If the OS hands a dead child's pid to an
|
|
30
|
+
// unrelated process before this line, that process is reported and killed; if this ever produces a
|
|
31
|
+
// false failure, verify the token before counting a pid as a survivor.
|
|
32
|
+
const survivors = nodes.filter((n) => isPidAlive(n.pid));
|
|
33
|
+
if (survivors.length === 0) return;
|
|
34
|
+
survivors.forEach((n) => forceKill(n.pid));
|
|
35
|
+
throw new Error(describeSurvivors(survivors, errors));
|
|
36
|
+
} finally {
|
|
37
|
+
orchestrators.length = 0;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Keeps going past a failed removal so one failure does not strand the rest. */
|
|
42
|
+
async function removeNodesOf(orch: HostProcessOrchestrator, nodes: StartedNode[], errors: string[]): Promise<void> {
|
|
43
|
+
for (const node of orch.listNodes()) {
|
|
44
|
+
nodes.push({ containerId: node.id, pid: decodeDockerId(node.dockerId).pid });
|
|
45
|
+
try {
|
|
46
|
+
await orch.removeContainer(node.dockerId);
|
|
47
|
+
} catch (err) {
|
|
48
|
+
errors.push(`${node.id}: ${err instanceof Error ? err.message : String(err)}`);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function forceKill(pid: number): void {
|
|
54
|
+
try {
|
|
55
|
+
process.kill(pid, 'SIGKILL');
|
|
56
|
+
} catch (err) {
|
|
57
|
+
if ((err as NodeJS.ErrnoException).code !== 'ESRCH') throw err;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function describeSurvivors(survivors: StartedNode[], errors: string[]): string {
|
|
62
|
+
const names = survivors.map((n) => `${n.containerId} (pid ${n.pid})`).join(', ');
|
|
63
|
+
const causes = errors.length > 0 ? `; removeContainer errors: ${errors.join(' | ')}` : '';
|
|
64
|
+
return `test teardown left child processes running (now force-killed): ${names}${causes}`;
|
|
65
|
+
}
|