@serfab/cadre-host 0.9.0 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/dist/auth/index.d.ts +2 -0
  2. package/dist/auth/index.js +1 -0
  3. package/dist/auth/index.js.map +1 -1
  4. package/dist/auth/self-label.d.ts +36 -0
  5. package/dist/auth/self-label.js +57 -0
  6. package/dist/auth/self-label.js.map +1 -0
  7. package/dist/auth/trust-circle.d.ts +32 -5
  8. package/dist/auth/trust-circle.js +31 -11
  9. package/dist/auth/trust-circle.js.map +1 -1
  10. package/dist/bin/host.js +41 -5
  11. package/dist/bin/host.js.map +1 -1
  12. package/dist/donation/__tests__/fake-orchestrator.d.ts +103 -0
  13. package/dist/donation/__tests__/fake-orchestrator.js +180 -0
  14. package/dist/donation/__tests__/fake-orchestrator.js.map +1 -0
  15. package/dist/donation/donation-service.d.ts +237 -12
  16. package/dist/donation/donation-service.js +491 -34
  17. package/dist/donation/donation-service.js.map +1 -1
  18. package/dist/donation/donation-store.d.ts +7 -0
  19. package/dist/donation/donation-store.js +7 -0
  20. package/dist/donation/donation-store.js.map +1 -1
  21. package/dist/donation/donation-supervisor.d.ts +167 -0
  22. package/dist/donation/donation-supervisor.js +341 -0
  23. package/dist/donation/donation-supervisor.js.map +1 -0
  24. package/dist/donation/index.d.ts +4 -2
  25. package/dist/donation/index.js +2 -1
  26. package/dist/donation/index.js.map +1 -1
  27. package/dist/donation/types.d.ts +26 -0
  28. package/dist/donation/types.js.map +1 -1
  29. package/dist/index.d.ts +3 -1
  30. package/dist/index.js +3 -1
  31. package/dist/index.js.map +1 -1
  32. package/dist/orchestrator/host-process-orchestrator.d.ts +109 -5
  33. package/dist/orchestrator/host-process-orchestrator.js +341 -98
  34. package/dist/orchestrator/host-process-orchestrator.js.map +1 -1
  35. package/dist/orchestrator/node-identity.d.ts +26 -0
  36. package/dist/orchestrator/node-identity.js +36 -0
  37. package/dist/orchestrator/node-identity.js.map +1 -0
  38. package/dist/orchestrator/port-allocator.d.ts +14 -0
  39. package/dist/orchestrator/port-allocator.js +45 -0
  40. package/dist/orchestrator/port-allocator.js.map +1 -1
  41. package/dist/owner/owner-node-client.d.ts +20 -5
  42. package/dist/owner/owner-node-client.js +21 -4
  43. package/dist/owner/owner-node-client.js.map +1 -1
  44. package/dist/server/error-handler.d.ts +7 -1
  45. package/dist/server/error-handler.js +23 -1
  46. package/dist/server/error-handler.js.map +1 -1
  47. package/dist/server/events/types.d.ts +3 -0
  48. package/dist/server/index.d.ts +7 -0
  49. package/dist/server/index.js +9 -3
  50. package/dist/server/index.js.map +1 -1
  51. package/dist/server/routes/grants.js +22 -3
  52. package/dist/server/routes/grants.js.map +1 -1
  53. package/dist/server/routes/strands.d.ts +20 -0
  54. package/dist/server/routes/strands.js +38 -0
  55. package/dist/server/routes/strands.js.map +1 -0
  56. package/dist/strands/index.d.ts +7 -0
  57. package/dist/strands/index.js +6 -0
  58. package/dist/strands/index.js.map +1 -0
  59. package/dist/strands/strand-service.d.ts +50 -0
  60. package/dist/strands/strand-service.js +103 -0
  61. package/dist/strands/strand-service.js.map +1 -0
  62. package/dist/strands/types.d.ts +93 -0
  63. package/dist/strands/types.js +23 -0
  64. package/dist/strands/types.js.map +1 -0
  65. package/dist/ui/assets/index-BVoqLjyX.css +1 -0
  66. package/dist/ui/assets/index-C-iks41M.js +17 -0
  67. package/dist/ui/assets/index-C-iks41M.js.map +1 -0
  68. package/dist/ui/index.html +2 -2
  69. package/dist/update/release-key.d.ts +1 -1
  70. package/dist/update/release-key.js +1 -1
  71. package/package.json +5 -5
  72. package/src/__tests__/build-targets.test.ts +26 -0
  73. package/src/__tests__/global-setup.ts +51 -0
  74. package/src/__tests__/orchestrator-env-scrub.test.ts +140 -0
  75. package/src/__tests__/orchestrator-node-identity.test.ts +204 -0
  76. package/src/__tests__/orchestrator.test.ts +387 -6
  77. package/src/auth/__tests__/self-label.test.ts +85 -0
  78. package/src/auth/__tests__/trust-circle-integration.test.ts +152 -147
  79. package/src/auth/__tests__/trust-circle.test.ts +45 -1
  80. package/src/auth/index.ts +2 -0
  81. package/src/auth/self-label.ts +74 -0
  82. package/src/auth/trust-circle.ts +45 -13
  83. package/src/bin/host.ts +1299 -1258
  84. package/src/donation/__tests__/donation-service.test.ts +836 -58
  85. package/src/donation/__tests__/donation-supervisor.test.ts +510 -0
  86. package/src/donation/__tests__/fake-orchestrator.test.ts +168 -0
  87. package/src/donation/__tests__/fake-orchestrator.ts +225 -0
  88. package/src/donation/donation-service.ts +611 -41
  89. package/src/donation/donation-store.ts +7 -0
  90. package/src/donation/donation-supervisor.ts +388 -0
  91. package/src/donation/index.ts +15 -0
  92. package/src/donation/types.ts +23 -0
  93. package/src/index.ts +14 -0
  94. package/src/orchestrator/__tests__/node-identity.test.ts +62 -0
  95. package/src/orchestrator/host-process-orchestrator.ts +336 -97
  96. package/src/orchestrator/node-identity.ts +40 -0
  97. package/src/orchestrator/port-allocator.ts +48 -0
  98. package/src/owner/owner-node-client.ts +31 -5
  99. package/src/server/__tests__/grants-route.test.ts +80 -6
  100. package/src/server/__tests__/strands-route.test.ts +233 -0
  101. package/src/server/error-handler.ts +24 -1
  102. package/src/server/events/types.ts +1 -0
  103. package/src/server/index.ts +16 -3
  104. package/src/server/routes/grants.ts +27 -3
  105. package/src/server/routes/strands.ts +54 -0
  106. package/src/strands/__tests__/strand-service.test.ts +165 -0
  107. package/src/strands/index.ts +14 -0
  108. package/src/strands/strand-service.ts +135 -0
  109. package/src/strands/types.ts +102 -0
  110. package/src/update/release-key.ts +1 -1
  111. package/dist/ui/assets/index-BlMXQPSw.js +0 -14
  112. package/dist/ui/assets/index-BlMXQPSw.js.map +0 -1
  113. package/dist/ui/assets/index-BwU5mJGF.css +0 -1
@@ -0,0 +1,103 @@
1
+ /**
2
+ * Shared fake orchestrator for the donation unit tests — no real child
3
+ * processes. Records every create / stop / remove, hands out deterministic
4
+ * unique spawn results, and tracks per-`dockerId` liveness so a test can crash
5
+ * one node and let the supervisor observe exactly that one as down.
6
+ *
7
+ * This file holds no tests of its own; its contract is pinned by the sibling
8
+ * `fake-orchestrator.test.ts`, which exists so a future agent cannot make a
9
+ * failing donation test go green by relaxing the fake.
10
+ */
11
+ import type { Orchestrator, OrchestratorCreateRequest, OrchestratorCreateResult, OrchestratorStats } from '@serfab/cadre-provider';
12
+ import type { ManagedNodeInfo, NodeStateListener } from '../../orchestrator/types.js';
13
+ /**
14
+ * Mirrors the parts of `HostProcessOrchestrator`'s handle lifecycle the donation
15
+ * code depends on:
16
+ *
17
+ * - a **successful** `createContainer` drops every prior handle for the same
18
+ * `containerId` (the real `dropStaleHandle`), so an old `dockerId` stops
19
+ * resolving the moment its container re-spawns;
20
+ * - a **failed** `createContainer` leaves those handles exactly as it found them
21
+ * (the real `restoreDroppedHandles`);
22
+ * - `stopContainer` / `removeContainer` throw `Container not found: <dockerId>`
23
+ * for a handle this orchestrator no longer knows (the real `requireHandle`);
24
+ * - `reclaimWorkdir` refuses a containerId that still resolves to a live
25
+ * handle (the real guard — that directory belongs to `removeContainer`).
26
+ *
27
+ * Consequently `stopped` / `removed` mean **"handles this orchestrator actually
28
+ * acted on"**, not "calls attempted" — a stop aimed at a dropped handle is
29
+ * rejected and never appears. `onStop` still fires for those rejected calls, so
30
+ * a test can observe the attempt separately from its outcome.
31
+ */
32
+ export declare class FakeOrchestrator implements Orchestrator {
33
+ createCalls: OrchestratorCreateRequest[];
34
+ stopped: string[];
35
+ removed: string[];
36
+ /**
37
+ * Container ids whose workdir this orchestrator actually removed. Same
38
+ * "acted on, not attempted" rule as `stopped` / `removed`: a refused reclaim
39
+ * leaves no entry.
40
+ */
41
+ reclaimedWorkdirs: string[];
42
+ failCreate: boolean;
43
+ createDelayMs: number;
44
+ /**
45
+ * Observation hook — lets a test inspect the world as the stop happens. Fires
46
+ * even when the handle is unknown and the stop is then rejected.
47
+ */
48
+ onStop?: (dockerId: string) => void;
49
+ /**
50
+ * Observation hook — fires before the spawn resolves or fails, modelling the
51
+ * real class's *pre-drop* `await` window (`ensureNodeIdentity`, `resolvePush`).
52
+ * Work driven from here sees the previous handle still alive. Contrast
53
+ * {@link onSpawned}.
54
+ */
55
+ onCreate?: (request: OrchestratorCreateRequest) => void;
56
+ /**
57
+ * Observation hook — fires once the drop has landed and the new child is
58
+ * registered, immediately before `createContainer` resolves. Models the
59
+ * microtask gap between the real `createContainer` returning and its caller's
60
+ * `await` resuming: the real drop → launch → return runs synchronously, so
61
+ * this is the earliest point other code can observe the post-drop world.
62
+ * Contrast {@link onCreate}.
63
+ */
64
+ onSpawned?: (dockerId: string) => void;
65
+ /** Observation hook — fires before the liveness answer is computed. */
66
+ onIsRunning?: (dockerId: string) => void;
67
+ private readonly children;
68
+ private readonly listeners;
69
+ private counter;
70
+ createContainer(request: OrchestratorCreateRequest): Promise<OrchestratorCreateResult>;
71
+ stopContainer(dockerId: string): Promise<void>;
72
+ removeContainer(dockerId: string): Promise<void>;
73
+ /**
74
+ * NOTE: this and {@link getLogs} take an unknown `dockerId` where the real
75
+ * class throws (`requireHandle`). No donation path calls either, so the gap is
76
+ * unobservable — route them through `requireChild` if one starts to.
77
+ */
78
+ getStats(): Promise<OrchestratorStats>;
79
+ /** Unknown handles read as not running, matching the real orchestrator. */
80
+ isRunning(dockerId: string): Promise<boolean>;
81
+ /** Resolve a spawn's friendly containerId back to its dockerId. */
82
+ resolveDockerId(containerId: string): string | undefined;
83
+ /**
84
+ * Remove the workdir of a container no handle owns, mirroring the real
85
+ * `reclaimWorkdir`. Refuses (returns `false`, records nothing) while a child
86
+ * for that containerId still resolves — that directory belongs to
87
+ * `removeContainer`, which stops the child first.
88
+ *
89
+ * There is no directory here to check for existence, so a first reclaim of an
90
+ * unresolvable id always reports `true`; the real class returns `false` when
91
+ * the path is already gone. Nothing in the donation layer branches on the
92
+ * return value, so the gap is unobservable — model it if one starts to.
93
+ */
94
+ reclaimWorkdir(containerId: string): boolean;
95
+ getLogs(): Promise<string>;
96
+ onStateChange(listener: NodeStateListener): () => void;
97
+ /** Kill a child out from under the host — the crash the supervisor exists for. */
98
+ crash(dockerId: string): void;
99
+ /** Emit an arbitrary state change (e.g. the owner node stopping). */
100
+ emit(info: ManagedNodeInfo): void;
101
+ /** Mirrors `HostProcessOrchestrator.requireHandle` — unknown handles throw. */
102
+ private requireChild;
103
+ }
@@ -0,0 +1,180 @@
1
+ /**
2
+ * Shared fake orchestrator for the donation unit tests — no real child
3
+ * processes. Records every create / stop / remove, hands out deterministic
4
+ * unique spawn results, and tracks per-`dockerId` liveness so a test can crash
5
+ * one node and let the supervisor observe exactly that one as down.
6
+ *
7
+ * This file holds no tests of its own; its contract is pinned by the sibling
8
+ * `fake-orchestrator.test.ts`, which exists so a future agent cannot make a
9
+ * failing donation test go green by relaxing the fake.
10
+ */
11
+ function sleep(ms) {
12
+ return new Promise((res) => setTimeout(res, ms));
13
+ }
14
+ /**
15
+ * Mirrors the parts of `HostProcessOrchestrator`'s handle lifecycle the donation
16
+ * code depends on:
17
+ *
18
+ * - a **successful** `createContainer` drops every prior handle for the same
19
+ * `containerId` (the real `dropStaleHandle`), so an old `dockerId` stops
20
+ * resolving the moment its container re-spawns;
21
+ * - a **failed** `createContainer` leaves those handles exactly as it found them
22
+ * (the real `restoreDroppedHandles`);
23
+ * - `stopContainer` / `removeContainer` throw `Container not found: <dockerId>`
24
+ * for a handle this orchestrator no longer knows (the real `requireHandle`);
25
+ * - `reclaimWorkdir` refuses a containerId that still resolves to a live
26
+ * handle (the real guard — that directory belongs to `removeContainer`).
27
+ *
28
+ * Consequently `stopped` / `removed` mean **"handles this orchestrator actually
29
+ * acted on"**, not "calls attempted" — a stop aimed at a dropped handle is
30
+ * rejected and never appears. `onStop` still fires for those rejected calls, so
31
+ * a test can observe the attempt separately from its outcome.
32
+ */
33
+ export class FakeOrchestrator {
34
+ constructor() {
35
+ this.createCalls = [];
36
+ this.stopped = [];
37
+ this.removed = [];
38
+ /**
39
+ * Container ids whose workdir this orchestrator actually removed. Same
40
+ * "acted on, not attempted" rule as `stopped` / `removed`: a refused reclaim
41
+ * leaves no entry.
42
+ */
43
+ this.reclaimedWorkdirs = [];
44
+ this.failCreate = false;
45
+ this.createDelayMs = 0;
46
+ this.children = new Map();
47
+ this.listeners = new Set();
48
+ this.counter = 0;
49
+ }
50
+ async createContainer(request) {
51
+ this.createCalls.push(request);
52
+ this.onCreate?.(request);
53
+ if (this.createDelayMs)
54
+ await sleep(this.createDelayMs);
55
+ if (this.failCreate)
56
+ throw new Error('spawn boom');
57
+ // Drop the prior handles for this container, mirroring `dropStaleHandle`.
58
+ // Success-only, and placed here on purpose: the real drop happens after the
59
+ // spawn's every `await` and cannot fail afterwards, and a create that throws
60
+ // puts the handles back (`restoreDroppedHandles`) — so a failed create must
61
+ // leave them untouched. `DonationSupervisor`'s give-up test depends on that.
62
+ for (const [id, child] of this.children) {
63
+ if (child.containerId === request.containerId)
64
+ this.children.delete(id);
65
+ }
66
+ const n = ++this.counter;
67
+ const dockerId = `dock_${n}`;
68
+ this.children.set(dockerId, {
69
+ containerId: request.containerId,
70
+ partyId: request.partyId,
71
+ profile: request.profile,
72
+ running: true,
73
+ });
74
+ this.onSpawned?.(dockerId);
75
+ return {
76
+ dockerId,
77
+ healthEndpoint: `http://127.0.0.1:${9000 + n}/health`,
78
+ metricsEndpoint: `http://127.0.0.1:${9000 + n}/metrics`,
79
+ seedEndpoint: `http://127.0.0.1:${9000 + n}/seed`,
80
+ seedToken: `seed-token-${n}`,
81
+ p2pPort: 4000 + n,
82
+ };
83
+ }
84
+ async stopContainer(dockerId) {
85
+ // Before the check: a test may want to observe that a stop was *attempted*
86
+ // even when this orchestrator rejects it.
87
+ this.onStop?.(dockerId);
88
+ const child = this.requireChild(dockerId);
89
+ this.stopped.push(dockerId);
90
+ child.running = false;
91
+ }
92
+ async removeContainer(dockerId) {
93
+ // NOTE: the real `removeContainer` stops a still-live child first; this one
94
+ // only deletes, so a remove aimed at a running child records no `stopped`
95
+ // entry and emits no state change. Every donation caller stops before it
96
+ // reclaims, so nothing sees the difference today — if one stops doing that,
97
+ // stop the child here too.
98
+ this.requireChild(dockerId);
99
+ this.removed.push(dockerId);
100
+ this.children.delete(dockerId);
101
+ }
102
+ /**
103
+ * NOTE: this and {@link getLogs} take an unknown `dockerId` where the real
104
+ * class throws (`requireHandle`). No donation path calls either, so the gap is
105
+ * unobservable — route them through `requireChild` if one starts to.
106
+ */
107
+ async getStats() {
108
+ return { cpuPercent: 0, memoryBytes: 0, networkRxBytes: 0, networkTxBytes: 0 };
109
+ }
110
+ /** Unknown handles read as not running, matching the real orchestrator. */
111
+ async isRunning(dockerId) {
112
+ this.onIsRunning?.(dockerId);
113
+ return this.children.get(dockerId)?.running ?? false;
114
+ }
115
+ /** Resolve a spawn's friendly containerId back to its dockerId. */
116
+ resolveDockerId(containerId) {
117
+ for (const [dockerId, child] of this.children) {
118
+ if (child.containerId === containerId)
119
+ return dockerId;
120
+ }
121
+ return undefined;
122
+ }
123
+ /**
124
+ * Remove the workdir of a container no handle owns, mirroring the real
125
+ * `reclaimWorkdir`. Refuses (returns `false`, records nothing) while a child
126
+ * for that containerId still resolves — that directory belongs to
127
+ * `removeContainer`, which stops the child first.
128
+ *
129
+ * There is no directory here to check for existence, so a first reclaim of an
130
+ * unresolvable id always reports `true`; the real class returns `false` when
131
+ * the path is already gone. Nothing in the donation layer branches on the
132
+ * return value, so the gap is unobservable — model it if one starts to.
133
+ */
134
+ reclaimWorkdir(containerId) {
135
+ if (this.resolveDockerId(containerId))
136
+ return false;
137
+ this.reclaimedWorkdirs.push(containerId);
138
+ return true;
139
+ }
140
+ async getLogs() {
141
+ return '';
142
+ }
143
+ onStateChange(listener) {
144
+ this.listeners.add(listener);
145
+ return () => { this.listeners.delete(listener); };
146
+ }
147
+ /** Kill a child out from under the host — the crash the supervisor exists for. */
148
+ crash(dockerId) {
149
+ const child = this.children.get(dockerId);
150
+ if (!child?.running)
151
+ return;
152
+ child.running = false;
153
+ this.emit(toNodeInfo(dockerId, child));
154
+ }
155
+ /** Emit an arbitrary state change (e.g. the owner node stopping). */
156
+ emit(info) {
157
+ for (const listener of this.listeners)
158
+ listener(info);
159
+ }
160
+ /** Mirrors `HostProcessOrchestrator.requireHandle` — unknown handles throw. */
161
+ requireChild(dockerId) {
162
+ const child = this.children.get(dockerId);
163
+ if (!child)
164
+ throw new Error(`Container not found: ${dockerId}`);
165
+ return child;
166
+ }
167
+ }
168
+ function toNodeInfo(dockerId, child) {
169
+ return {
170
+ id: child.containerId,
171
+ dockerId,
172
+ partyId: child.partyId,
173
+ profile: child.profile,
174
+ status: child.running ? 'running' : 'stopped',
175
+ spawnedAt: new Date(0).toISOString(),
176
+ workdir: `/fake/${child.containerId}`,
177
+ ports: { health: 0, metrics: 0, p2p: 0, admin: 0 },
178
+ };
179
+ }
180
+ //# sourceMappingURL=fake-orchestrator.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"fake-orchestrator.js","sourceRoot":"","sources":["../../../src/donation/__tests__/fake-orchestrator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAWH,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;AACnD,CAAC;AAUD;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,OAAO,gBAAgB;IAA7B;QACE,gBAAW,GAAgC,EAAE,CAAC;QAC9C,YAAO,GAAa,EAAE,CAAC;QACvB,YAAO,GAAa,EAAE,CAAC;QACvB;;;;WAIG;QACH,sBAAiB,GAAa,EAAE,CAAC;QACjC,eAAU,GAAG,KAAK,CAAC;QACnB,kBAAa,GAAG,CAAC,CAAC;QAyBD,aAAQ,GAAG,IAAI,GAAG,EAAqB,CAAC;QACxC,cAAS,GAAG,IAAI,GAAG,EAAqB,CAAC;QAClD,YAAO,GAAG,CAAC,CAAC;IA0HtB,CAAC;IAxHC,KAAK,CAAC,eAAe,CAAC,OAAkC;QACtD,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/B,IAAI,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,CAAC;QACzB,IAAI,IAAI,CAAC,aAAa;YAAE,MAAM,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACxD,IAAI,IAAI,CAAC,UAAU;YAAE,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC;QACnD,0EAA0E;QAC1E,4EAA4E;QAC5E,6EAA6E;QAC7E,4EAA4E;QAC5E,6EAA6E;QAC7E,KAAK,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACxC,IAAI,KAAK,CAAC,WAAW,KAAK,OAAO,CAAC,WAAW;gBAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC;QACzB,MAAM,QAAQ,GAAG,QAAQ,CAAC,EAAE,CAAC;QAC7B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE;YAC1B,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,OAAO,EAAE,IAAI;SACd,CAAC,CAAC;QACH,IAAI,CAAC,SAAS,EAAE,CAAC,QAAQ,CAAC,CAAC;QAC3B,OAAO;YACL,QAAQ;YACR,cAAc,EAAE,oBAAoB,IAAI,GAAG,CAAC,SAAS;YACrD,eAAe,EAAE,oBAAoB,IAAI,GAAG,CAAC,UAAU;YACvD,YAAY,EAAE,oBAAoB,IAAI,GAAG,CAAC,OAAO;YACjD,SAAS,EAAE,cAAc,CAAC,EAAE;YAC5B,OAAO,EAAE,IAAI,GAAG,CAAC;SAClB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,QAAgB;QAClC,2EAA2E;QAC3E,0CAA0C;QAC1C,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,CAAC;QACxB,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;QAC1C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5B,KAAK,CAAC,OAAO,GAAG,KAAK,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,QAAgB;QACpC,4EAA4E;QAC5E,0EAA0E;QAC1E,yEAAyE;QACzE,4EAA4E;QAC5E,2BAA2B;QAC3B,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;QAC5B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC5B,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACjC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,QAAQ;QACZ,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,CAAC;IACjF,CAAC;IAED,2EAA2E;IAC3E,KAAK,CAAC,SAAS,CAAC,QAAgB;QAC9B,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC;QAC7B,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,OAAO,IAAI,KAAK,CAAC;IACvD,CAAC;IAED,mEAAmE;IACnE,eAAe,CAAC,WAAmB;QACjC,KAAK,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC9C,IAAI,KAAK,CAAC,WAAW,KAAK,WAAW;gBAAE,OAAO,QAAQ,CAAC;QACzD,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED;;;;;;;;;;OAUG;IACH,cAAc,CAAC,WAAmB;QAChC,IAAI,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC;YAAE,OAAO,KAAK,CAAC;QACpD,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACzC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,OAAO;QACX,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,aAAa,CAAC,QAA2B;QACvC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC7B,OAAO,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IACpD,CAAC;IAED,kFAAkF;IAClF,KAAK,CAAC,QAAgB;QACpB,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,EAAE,OAAO;YAAE,OAAO;QAC5B,KAAK,CAAC,OAAO,GAAG,KAAK,CAAC;QACtB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;IACzC,CAAC;IAED,qEAAqE;IACrE,IAAI,CAAC,IAAqB;QACxB,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS;YAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;IACxD,CAAC;IAED,+EAA+E;IACvE,YAAY,CAAC,QAAgB;QACnC,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,QAAQ,EAAE,CAAC,CAAC;QAChE,OAAO,KAAK,CAAC;IACf,CAAC;CACF;AAED,SAAS,UAAU,CAAC,QAAgB,EAAE,KAAgB;IACpD,OAAO;QACL,EAAE,EAAE,KAAK,CAAC,WAAW;QACrB,QAAQ;QACR,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;QAC7C,SAAS,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE;QACpC,OAAO,EAAE,SAAS,KAAK,CAAC,WAAW,EAAE;QACrC,KAAK,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE;KACnD,CAAC;AACJ,CAAC"}
@@ -1,6 +1,6 @@
1
1
  import type { Orchestrator } from '@serfab/cadre-provider';
2
2
  import type { DonationStore } from './donation-store.js';
3
- import type { DonationView, GrantValidator } from './types.js';
3
+ import type { DonationStatus, DonationView, GrantValidator } from './types.js';
4
4
  /**
5
5
  * Default age after which a donation still stuck in `awaiting_seed` is
6
6
  * auto-terminated: the requester provisioned a node but never presented a seed
@@ -11,6 +11,16 @@ import type { DonationView, GrantValidator } from './types.js';
11
11
  * to tune the grace period, promote this to `host.config.json`.
12
12
  */
13
13
  export declare const DONATION_AWAITING_SEED_TTL_MS: number;
14
+ /**
15
+ * Default age after which a donation still stuck in `provisioning` is
16
+ * auto-terminated: the host wrote the row before starting the child, and a
17
+ * crash/kill in that window (or during the spawn itself) leaves nothing to
18
+ * advance it across a restart. A real spawn (identity key read/gen, port
19
+ * allocation, process spawn) completes in well under a second normally, so 5
20
+ * minutes is generously past "any plausible spawn" without risking a false
21
+ * reap of one still genuinely in flight under heavy load.
22
+ */
23
+ export declare const DONATION_PROVISIONING_TTL_MS: number;
14
24
  /** How often the reap sweep runs while cadre-host is up. 5 minutes. */
15
25
  export declare const DONATION_REAP_SWEEP_MS: number;
16
26
  /** A grantee-driven request to provision one donated node. */
@@ -24,28 +34,94 @@ export interface DonationProvisionRequest {
24
34
  /**
25
35
  * The requester's owner public key(s), base64url. Pinned as cold-start
26
36
  * seed-trust anchors on the donated node so it accepts the requester-signed
27
- * seed presented in {@link DonationService.applySeed}. Without these the node
28
- * defaults to a db-anchored policy that rejects every seed.
37
+ * seed presented in {@link DonationService.applySeed}. Without these the node's
38
+ * node-local trusted-owner anchor holds no key for the requester's party, and
39
+ * the default anchored policy rejects every seed.
40
+ *
41
+ * Shape-validated on the way in ({@link DonationService.provision}): every entry
42
+ * must decode as a base64url 32-byte Ed25519 public key, the same rule the node
43
+ * itself applies to `CADRE_OWNER_KEYS`. So everything downstream — the persisted
44
+ * record, the child's environment, a later respawn replaying it — carries values
45
+ * the node can actually start with. Curve membership is NOT checked (see
46
+ * `requireEd25519PublicKeyB64`): a well-formed key naming the wrong signer still
47
+ * fails at seed time, as any wrong key would.
29
48
  */
30
49
  ownerKeys: string[];
31
50
  /** Node profile; defaults to `storage` so the node participates and is dialable. */
32
51
  profile?: 'storage' | 'transaction';
33
52
  }
34
- /** Result of presenting a seed to a donated node's `POST /seed`. */
35
- export interface DonationSeedResult {
36
- success: boolean;
37
- peersAdded?: number;
53
+ /**
54
+ * Outcome of {@link DonationService.applySeed}.
55
+ *
56
+ * - `seeded` — the node accepted the seed and the record now says so.
57
+ * - `rejected` — the node refused the seed (its own seed-trust policy). Nothing
58
+ * written.
59
+ * - `abandoned` — the seed reached the node, but the loan ended while the
60
+ * request was in flight. The ending wins: the record is left exactly as the
61
+ * ending wrote it. `status` is the status that won, absent when the row is
62
+ * gone entirely.
63
+ */
64
+ export type DonationSeedResult = {
65
+ outcome: 'seeded';
66
+ peersAdded: number;
67
+ } | {
68
+ outcome: 'rejected';
38
69
  error?: string;
39
- }
70
+ } | {
71
+ outcome: 'abandoned';
72
+ status?: DonationStatus;
73
+ };
40
74
  /** Live peer identity of a donated node. */
41
75
  export interface DonationPeerInfo {
42
76
  peerId: string;
43
77
  multiaddrs: string[];
44
78
  }
79
+ /**
80
+ * Outcome of {@link DonationService.respawn}. Three distinct non-throwing ends,
81
+ * which a bare `DonationView | undefined` conflated:
82
+ *
83
+ * - `respawned` — a new child is up and the record now names its handles.
84
+ * - `not_respawnable` — the record predates persisted spawn inputs, so there is
85
+ * nothing to replay. Permanent; no child was spawned.
86
+ * - `abandoned` — the loan ended (or the record vanished) while the spawn was in
87
+ * flight. The ending wins: the record is left exactly as the ending wrote it
88
+ * and the new child is cleaned up. `status` is the status that won, absent
89
+ * when the row is gone entirely.
90
+ */
91
+ export type RespawnResult = {
92
+ outcome: 'respawned';
93
+ donation: DonationView;
94
+ } | {
95
+ outcome: 'not_respawnable';
96
+ } | {
97
+ outcome: 'abandoned';
98
+ status?: DonationStatus;
99
+ };
100
+ /**
101
+ * Orchestrator capabilities the donation cleanup paths need beyond the base
102
+ * `Orchestrator`. Both are optional — only `HostProcessOrchestrator` implements
103
+ * them; absent on a test double or a future orchestrator, the cleanup degrades
104
+ * to terminalizing the record with nothing to reclaim.
105
+ */
106
+ export interface DonationOrchestrator extends Orchestrator {
107
+ /**
108
+ * Resolve a spawn's friendly containerId (== a donation's id) back to its
109
+ * current dockerId, so a stuck-`provisioning` reap can find and reclaim a
110
+ * child that was actually spawned before the host died.
111
+ */
112
+ resolveDockerId?(containerId: string): string | undefined;
113
+ /**
114
+ * Remove the working directory a spawn created for `containerId` when no
115
+ * handle owns it — the cleanup of last resort for a record that never got a
116
+ * `dockerId`. Returns whether anything was removed. Refuses when a handle
117
+ * still resolves (that child's directory belongs to `removeContainer`).
118
+ */
119
+ reclaimWorkdir?(containerId: string): boolean;
120
+ }
45
121
  /** Constructor options. */
46
122
  export interface DonationServiceOptions {
47
123
  /** Orchestrator that spawns/stops the donated node child process. */
48
- orchestrator: Orchestrator;
124
+ orchestrator: DonationOrchestrator;
49
125
  /** Grant validator (identity/expiry/revocation + quota). */
50
126
  grants: GrantValidator;
51
127
  /** Persistent donation store (`donations.json`). */
@@ -89,9 +165,27 @@ export declare class DonationService {
89
165
  * Provision a donated node for a validated grant. Serialized per grant token
90
166
  * so the quota check and record-create are atomic. On any orchestrator failure
91
167
  * the reserved resources are reclaimed and the record is marked `error`.
168
+ *
169
+ * The owner-key pins are shape-checked FIRST — ahead of the per-grant lock, the
170
+ * quota check and the record write — so a malformed request costs nothing: no
171
+ * waiting behind another provision, no grant quota slot burned, no workdir or
172
+ * port taken, and no `error` record to explain later. Without it the request is
173
+ * answered as provisioned and the child dies at boot, which is the one place the
174
+ * caller cannot act on the typo.
92
175
  */
93
176
  provision(request: DonationProvisionRequest): Promise<DonationView>;
94
177
  private provisionLocked;
178
+ /**
179
+ * Mark a still-`provisioning` record as failed — and ONLY such a record. A row
180
+ * that went `terminated` while the spawn was in flight is the borrower's own
181
+ * ending and must not be rewritten as a host fault; a row that vanished must
182
+ * not be recreated.
183
+ *
184
+ * Best-effort (mirroring {@link storeRespawnAttempt}): we are already unwinding a
185
+ * provision failure, and a store error here must not mask the error being
186
+ * reported to the caller.
187
+ */
188
+ private markProvisionFailed;
95
189
  /**
96
190
  * Live peer identity of a donated node, read fresh from its `/status` every
97
191
  * call (no cache) so a re-keyed identity / remapped multiaddr is never served
@@ -102,12 +196,79 @@ export declare class DonationService {
102
196
  * Present the requester's phone-signed seed to the donated node's `POST /seed`,
103
197
  * authenticated with the host↔node `seedToken`. On success the donation moves
104
198
  * to `seeded`. The node still enforces its own seed-trust policy, so a seed
105
- * signed by a key the node did not pin is rejected here (`success: false`).
199
+ * signed by a key the node did not pin comes back `rejected`.
200
+ *
201
+ * **An ending that lands mid-seed wins.** The record is re-read after the node
202
+ * answers; if the loan ended (or the row vanished) while the request was in
203
+ * flight, that write stands and we report `abandoned` rather than marking a
204
+ * dead loan `seeded`. Unlike an abandoned respawn there is nothing to clean up
205
+ * — the record named its `dockerId` the whole time, so the ending's own
206
+ * `terminate` already stopped and reclaimed the child.
106
207
  */
107
208
  applySeed(id: string, encodedSeed: string): Promise<DonationSeedResult>;
108
209
  /**
109
- * Terminate a donation: stop + remove the child process, then mark the record
110
- * `terminated` (kept for audit, excluded from the live-node tally).
210
+ * Re-spawn a donated node that is no longer running, replaying the spawn
211
+ * inputs persisted on the record. The node keeps its workdir — and with it its
212
+ * identity key, trusted-owner anchor, and retained bootstrap peers — so it
213
+ * comes back as the *same* peer and rejoins from its own durable node-local
214
+ * stores. Only the host-side handles change: `dockerId`, `seedEndpoint`,
215
+ * `seedToken` are written back fresh.
216
+ *
217
+ * **Status is deliberately unchanged.** A `seeded` node stays `seeded` (it
218
+ * needs no re-seeding); an `awaiting_seed` node stays `awaiting_seed` and the
219
+ * borrower's later seed goes to the new endpoint/token.
220
+ *
221
+ * **An ending that lands mid-spawn wins.** The record is re-read after the
222
+ * orchestrator returns; if it went `terminated` (or `error`, or vanished)
223
+ * while the child was starting, that write stands and the new child is
224
+ * abandoned — see {@link abandonRespawn}. Both non-`respawned` outcomes are
225
+ * skips, not failures, so a sweep over the store keeps going. Orchestrator
226
+ * failures DO throw; the caller owns backoff/give-up.
227
+ *
228
+ * NOTE: not serialized. Two overlapping calls for the same id both spawn a
229
+ * child, and the second spawn drops the first's orchestrator handle — leaving
230
+ * an unmanaged process. A supervisor with more than one trigger (timer +
231
+ * exit event) must not let its passes overlap on one id.
232
+ */
233
+ respawn(id: string): Promise<RespawnResult>;
234
+ /**
235
+ * Give up on a respawn whose loan ended while the child was starting: stop the
236
+ * new child and, unless the record went `error`, reclaim it.
237
+ *
238
+ * Reclaim, not merely stop. The ending's own cleanup already ran and found
239
+ * nothing: `HostProcessOrchestrator.createContainer` calls `dropStaleHandle`
240
+ * *while* spawning, so by the time the concurrent `terminate` reached its
241
+ * stop/reclaim the old handle was gone and both calls were swallowed as
242
+ * best-effort no-ops. The new handle is now the only thing holding that
243
+ * spawn's ports and workdir, so stopping alone would leak both. On a
244
+ * `terminated` record the loan is over and the workdir (identity key +
245
+ * node-local stores) goes with it — which is what `terminate` intended.
246
+ *
247
+ * `error` is the deliberate exception: `removeContainer` deletes the workdir,
248
+ * and `DonationSupervisor.giveUp` keeps it on purpose so the identity key the
249
+ * borrower's cadre approved survives. Since both spawns share one workdir
250
+ * (`<rootDir>/<containerId>`), reclaiming here would delete exactly what
251
+ * `giveUp` meant to keep.
252
+ *
253
+ * NOTE: that skipped reclaim leaks the new spawn's four ports — the record
254
+ * still names the *previous* `dockerId`, so a later `terminate` cleans up the
255
+ * old handle, not this one. Unreachable today (`giveUp` is only called from
256
+ * the supervisor's serialized pass, so it cannot overlap a respawn); if a
257
+ * second `respawn` caller ever appears, write the new `dockerId` onto the
258
+ * `error` record here so that later `terminate` reclaims the right child.
259
+ */
260
+ private abandonRespawn;
261
+ /**
262
+ * Terminate a donation: mark the record `terminated` FIRST, then stop + remove
263
+ * the child process (kept in the store for audit, excluded from the live-node
264
+ * tally).
265
+ *
266
+ * Order matters: stopping the child fires the orchestrator's `onStateChange`,
267
+ * and a respawn supervisor listening there must already see a terminal record
268
+ * — otherwise it observes "node gone, record still `seeded`" and resurrects a
269
+ * loan the borrower just ended. The tradeoff: a crash between the write and
270
+ * the stop leaves a `terminated` record with a live child (reaped on the next
271
+ * host start), which is strictly better than resurrecting an ended loan.
111
272
  */
112
273
  terminate(id: string): Promise<void>;
113
274
  /**
@@ -119,6 +280,39 @@ export declare class DonationService {
119
280
  * logged and the sweep continues. Returns the reaped donation ids.
120
281
  */
121
282
  reapStaleAwaitingSeed(ttlMs?: number): Promise<string[]>;
283
+ /**
284
+ * Auto-terminate donations stuck in `provisioning` past `ttlMs` — the record
285
+ * is written before the child spawn starts, so a host crash/kill anywhere in
286
+ * that window (including mid-spawn) leaves nothing to advance it across a
287
+ * restart: there is no in-flight `provisionLocked` call left to reach its
288
+ * post-spawn re-read. Run on the same periodic sweep and startup pass as
289
+ * {@link reapStaleAwaitingSeed}. Age is measured from `updatedAt`, which for a
290
+ * genuinely-stuck row equals `createdAt` — `provisionLocked` never touches a
291
+ * `provisioning` record again except to advance it out of this predicate.
292
+ * Best-effort per record: a failed reap is logged and the sweep continues.
293
+ * Returns the reaped donation ids.
294
+ */
295
+ reapStaleProvisioning(ttlMs?: number): Promise<string[]>;
296
+ /**
297
+ * Terminalize a stuck-`provisioning` record as `error` and reclaim any child
298
+ * the orchestrator can still find for it. Status is written FIRST (same
299
+ * ordering rule as {@link terminate}): reclaiming fires `onStateChange`, and
300
+ * anything listening must already see a terminal record.
301
+ *
302
+ * NOTE: the stop only reaches a child the orchestrator has a handle for.
303
+ * `HostProcessOrchestrator.launchChild` spawns the process and persists the
304
+ * handle in the same synchronous step, so the only child this misses is one
305
+ * whose host died between the OS spawn and that write — an orphan *process*
306
+ * this reap terminalizes the record for but cannot kill (its ports stay held
307
+ * until a reboot). Its working directory is no longer out of reach, though:
308
+ * with no handle to resolve we fall back to reclaiming by container name,
309
+ * which also covers the far commoner case of a host that died before any
310
+ * child existed at all. If the spawn→persist window ever widens (an async
311
+ * step added between the two), the orchestrator still needs its own orphan
312
+ * *process* sweep — a record that names no handle cannot get one after the
313
+ * fact.
314
+ */
315
+ private reclaimStuckProvisioning;
122
316
  /** One donation (redacted), or undefined when unknown. */
123
317
  get(id: string): DonationView | undefined;
124
318
  /** All donations (redacted), optionally scoped to one grant token. */
@@ -126,8 +320,39 @@ export declare class DonationService {
126
320
  private requireDonation;
127
321
  /** Best-effort stop; logs but never throws (cleanup path). */
128
322
  private safeStop;
323
+ /**
324
+ * Persist a failed respawn's attempt counters — plus, when the spawn itself
325
+ * succeeded and it was the record write that failed, the new child's handles.
326
+ * Merged onto whatever is on disk now, never written wholesale: the caller's
327
+ * copy predates the orchestrator round-trip and `store.put` replaces the whole
328
+ * row, so writing it back would undo a `terminate` that landed while the spawn
329
+ * was in flight and resurrect a loan the borrower just ended.
330
+ *
331
+ * `status` and `updatedAt` are deliberately untouched — the attempt did not
332
+ * succeed, and `updatedAt` is what defers the stale-`awaiting_seed` reap.
333
+ *
334
+ * Recording `spawned` is what keeps the record pointing at the child that
335
+ * actually exists: `createContainer` already dropped the handle the record
336
+ * previously named, so leaving the old `dockerId` there would send every later
337
+ * stop/reclaim at an id the orchestrator cannot resolve — stranding the node's
338
+ * workdir on disk forever.
339
+ *
340
+ * Best-effort: we are already unwinding a respawn failure, and a store error
341
+ * here must not mask it. Losing the write costs the caller one extra attempt
342
+ * before backoff; the supervisor's next pass re-reads the record, finds the
343
+ * named child not running, and respawns again — which drops whatever handle is
344
+ * current, so the ports self-heal either way.
345
+ */
346
+ private storeRespawnAttempt;
129
347
  /** Best-effort remove; logs but never throws (cleanup path). */
130
348
  private safeReclaim;
349
+ /**
350
+ * Best-effort removal of the working directory of a donation whose spawn
351
+ * never produced a `dockerId`; logs but never throws (cleanup path). A no-op
352
+ * on an orchestrator without the capability, and on one that refuses because
353
+ * a live handle still owns the directory.
354
+ */
355
+ private safeReclaimWorkdir;
131
356
  /**
132
357
  * Run `fn` after any in-flight provision for the same grant token, so the
133
358
  * quota check + record-create pair is atomic per grant.