@quolu/lattice 0.52.3 → 0.52.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/artifact-contracts-v2.mjs +325 -325
- package/src/artifact-contracts.mjs +895 -895
- package/src/boundary-compiler.mjs +712 -712
- package/src/boundary-observation-compiler-v2.mjs +344 -344
- package/src/bounded-seam.mjs +230 -230
- package/src/bridge-address.mjs +107 -107
- package/src/bridge-cli.mjs +297 -297
- package/src/bridge-config.mjs +346 -346
- package/src/bridge-daemon.mjs +378 -378
- package/src/bridge-hub-protocol.mjs +198 -0
- package/src/bridge-launch-agent.mjs +323 -323
- package/src/bridge-registrar.mjs +102 -102
- package/src/bridge-server.mjs +376 -376
- package/src/cli-help.mjs +308 -308
- package/src/cli-stdio.mjs +40 -40
- package/src/control-compiler.mjs +532 -532
- package/src/dag-chain.mjs +261 -261
- package/src/factory-diagnostics.mjs +188 -188
- package/src/git-process.mjs +72 -0
- package/src/hash-chain.mjs +76 -76
- package/src/hooks-cli.mjs +1057 -1057
- package/src/isolation-runner.mjs +409 -409
- package/src/node-version-guard.mjs +44 -44
- package/src/project-cli.mjs +3 -3
- package/src/project-identity.mjs +130 -130
- package/src/rc1-black-box-oracle.mjs +906 -906
- package/src/rc1-comparison.mjs +154 -154
- package/src/rc1-evidence-bundle.mjs +456 -456
- package/src/rc1-v4-campaign.mjs +708 -708
- package/src/rc1-v4-transform.mjs +467 -467
- package/src/rc1-v5-artifact-set.mjs +855 -855
- package/src/rc1-v5-behavior-evidence.mjs +594 -594
- package/src/rc1-v5-campaign.mjs +797 -797
- package/src/rc1-v5-transform.mjs +421 -421
- package/src/rc1-v6-artifact-set.mjs +807 -807
- package/src/rc1-v6-behavior-evidence.mjs +273 -273
- package/src/rc1-v6-campaign.mjs +623 -623
- package/src/rc1-v6-causal-binding.mjs +473 -473
- package/src/rc1-v6-measurement.mjs +313 -313
- package/src/rc2-artifact-set.mjs +1584 -1584
- package/src/rc2-campaign.mjs +1506 -1506
- package/src/rc2-delivery-policy-front-end.mjs +1079 -1079
- package/src/rc2-delivery-policy-oracle.mjs +134 -134
- package/src/rc2-delivery-policy-transform.mjs +1127 -1127
- package/src/rc2-rc1-transfer-front-end.mjs +511 -511
- package/src/rc3-actual-dogfood.mjs +652 -652
- package/src/rc3-dogfood-scaffold.mjs +315 -315
- package/src/rc3-scripted-campaign.mjs +1383 -1383
- package/src/rc4-stage1-dogfood.mjs +673 -673
- package/src/runtime-adapter-registry.mjs +524 -524
- package/src/runtime-cli.mjs +4588 -4588
- package/src/runtime-contracts.mjs +824 -824
- package/src/runtime-control-store.mjs +604 -604
- package/src/runtime-controller-protocol.mjs +587 -587
- package/src/runtime-decision-verifier.mjs +701 -701
- package/src/runtime-diff-observer.mjs +361 -361
- package/src/runtime-direct-os-observer.mjs +301 -301
- package/src/runtime-driver-state.mjs +166 -166
- package/src/runtime-engine.mjs +779 -779
- package/src/runtime-errors.mjs +356 -356
- package/src/runtime-event-store.mjs +189 -189
- package/src/runtime-front-end.mjs +925 -925
- package/src/runtime-gate-store.mjs +481 -481
- package/src/runtime-hold-recompile.mjs +916 -916
- package/src/runtime-io-sentinel.mjs +391 -391
- package/src/runtime-lifecycle-lock.mjs +294 -294
- package/src/runtime-managed-supervisor.mjs +1490 -1490
- package/src/runtime-multi-epoch-store.mjs +838 -838
- package/src/runtime-projection.mjs +269 -269
- package/src/runtime-pull-intake.mjs +1192 -1192
- package/src/runtime-scripted-adapter-controller.mjs +1160 -1160
- package/src/runtime-scripted-executor.mjs +163 -163
- package/src/runtime-scripted-worktree.mjs +104 -104
- package/src/runtime-seam-resolve.mjs +428 -428
- package/src/runtime-seam-treatment.mjs +173 -173
- package/src/runtime-socket-owner.mjs +125 -125
- package/src/runtime-work-order-contracts.mjs +91 -91
- package/src/runtime-work-order-controller.mjs +1171 -1171
- package/src/runtime-worktree-executor.mjs +199 -199
- package/src/schedulability-compiler-v2.mjs +303 -303
- package/src/schedulability-verifier-v2.mjs +317 -317
- package/src/seam-apply.mjs +549 -549
- package/src/seam-commit-shared.mjs +22 -22
- package/src/seam-commit-transform.mjs +81 -81
- package/src/seam-commit.mjs +18 -18
- package/src/seam-cost.mjs +322 -322
- package/src/seam-derivation.mjs +188 -188
- package/src/seam-gate.mjs +146 -146
- package/src/seam-proposal-contracts.mjs +446 -446
- package/src/seam-proposal-queries.mjs +521 -521
- package/src/seam-proposal.mjs +2011 -2011
- package/src/seam-ref.mjs +33 -33
- package/src/seam-rewrite.mjs +286 -286
- package/src/seam-transform.mjs +554 -554
- package/src/seam-verification.mjs +260 -260
- package/src/sensor-adapter.mjs +432 -432
- package/src/sensor-cli.mjs +139 -139
- package/src/sensor-diff.mjs +661 -661
- package/src/sensor-node-runtime.mjs +53 -53
- package/src/sensor-runtime.mjs +52 -52
- package/src/timestamp-contract.mjs +8 -8
- package/src/todo-audit-pending.mjs +91 -91
- package/src/todo-chain.mjs +178 -178
- package/src/todo-cli.mjs +8 -8
- package/src/todo-contracts.mjs +728 -728
- package/src/todo-dashboard-registry.mjs +573 -573
- package/src/todo-dispatch-shape.mjs +190 -190
- package/src/todo-gantt-html-independence.mjs +239 -239
- package/src/todo-gantt-html-shared.mjs +226 -226
- package/src/todo-gantt-html-style.mjs +131 -131
- package/src/todo-gantt-html.mjs +248 -248
- package/src/todo-gantt-layout.mjs +974 -974
- package/src/todo-gantt-live.mjs +361 -361
- package/src/todo-gantt-nested.mjs +263 -263
- package/src/todo-gantt-presentation.mjs +217 -217
- package/src/todo-gantt-scope.mjs +123 -123
- package/src/todo-gantt-svg.mjs +353 -353
- package/src/todo-independence-contracts.mjs +595 -595
- package/src/todo-independence-guidance.mjs +322 -322
- package/src/todo-independence.mjs +640 -640
- package/src/todo-markdown-renderer.mjs +260 -260
- package/src/todo-migration.mjs +448 -448
- package/src/todo-narrative-anchor.mjs +130 -130
- package/src/todo-note-store.mjs +629 -629
- package/src/todo-parallel-candidates.mjs +114 -114
- package/src/todo-revision.mjs +995 -995
- package/src/todo-split.mjs +472 -472
- package/src/todo-status.mjs +690 -690
- package/src/todo-store-git-transaction.mjs +418 -418
- package/src/todo-store.mjs +42 -21
- package/src/treatment-compiler.mjs +728 -728
- package/src/treatment-runner.mjs +656 -656
- package/src/witness-scaffold.mjs +180 -180
|
@@ -1,1490 +1,1490 @@
|
|
|
1
|
-
import net from 'node:net';
|
|
2
|
-
import { createHash, randomBytes, randomUUID } from 'node:crypto';
|
|
3
|
-
import { execFile, spawn } from 'node:child_process';
|
|
4
|
-
import { chmod, lstat, mkdir, readFile, realpath, rm } from 'node:fs/promises';
|
|
5
|
-
import path from 'node:path';
|
|
6
|
-
import { fileURLToPath } from 'node:url';
|
|
7
|
-
import { promisify } from 'node:util';
|
|
8
|
-
import { canonicalizeArtifact, digestArtifact } from './artifact-contracts.mjs';
|
|
9
|
-
import { selfDigest } from './runtime-contracts.mjs';
|
|
10
|
-
import { captureWorktreeDiff } from './runtime-diff-observer.mjs';
|
|
11
|
-
import { pidsOwningSocketPath, socketPathsOwnedByPid } from './runtime-socket-owner.mjs';
|
|
12
|
-
import { createDirectOsProcessObserver as createDirectOsProcessObserverV2 } from './runtime-direct-os-observer.mjs';
|
|
13
|
-
import {
|
|
14
|
-
armStagedWriteLease,
|
|
15
|
-
createControllerBootstrap,
|
|
16
|
-
createControllerRequest,
|
|
17
|
-
createWriteGate,
|
|
18
|
-
validateArmedWriteLease,
|
|
19
|
-
validateAdapterLaunchDescriptor,
|
|
20
|
-
validateAdapterRegistry,
|
|
21
|
-
validateControllerDescriptor,
|
|
22
|
-
validateControllerError,
|
|
23
|
-
validateControllerHeartbeat,
|
|
24
|
-
validateControllerRegistration,
|
|
25
|
-
validateControllerResponse,
|
|
26
|
-
validateControllerHandshakeResponse,
|
|
27
|
-
validateEpochRebindAck,
|
|
28
|
-
validateProcessStartIdentity,
|
|
29
|
-
validateQuiescenceAck,
|
|
30
|
-
validateReleaseAck,
|
|
31
|
-
validateRuntimeControlRequest,
|
|
32
|
-
validateRuntimeControlResponse,
|
|
33
|
-
validateStagedWriteLease,
|
|
34
|
-
verifyCentralWriteGate,
|
|
35
|
-
} from './runtime-controller-protocol.mjs';
|
|
36
|
-
|
|
37
|
-
const SHA256 = /^[0-9a-f]{64}$/;
|
|
38
|
-
const ID = /^[0-9A-Za-z](?:[0-9A-Za-z._-]{0,127})$/;
|
|
39
|
-
|
|
40
|
-
export class ManagedRuntimeError extends Error {
|
|
41
|
-
constructor(code, detail) {
|
|
42
|
-
super(`${code}: ${detail}`);
|
|
43
|
-
this.name = 'ManagedRuntimeError';
|
|
44
|
-
this.code = code;
|
|
45
|
-
this.detail = detail;
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
function fail(code, detail) { throw new ManagedRuntimeError(code, detail); }
|
|
50
|
-
function digest(value) { return typeof value === 'string' && SHA256.test(value); }
|
|
51
|
-
function identifier(value) { return typeof value === 'string' && ID.test(value); }
|
|
52
|
-
function monotonicDefault() { return Number(process.hrtime.bigint() / 1_000_000n); }
|
|
53
|
-
function sha256Bytes(bytes) { return createHash('sha256').update(bytes).digest('hex'); }
|
|
54
|
-
const execFileAsync = promisify(execFile);
|
|
55
|
-
|
|
56
|
-
export async function observeManagedProcessStartIdentity(pid) {
|
|
57
|
-
let startedIdentity;
|
|
58
|
-
try {
|
|
59
|
-
const { stdout } = await execFileAsync('/bin/ps', ['-o', 'lstart=', '-p', String(pid)], { encoding: 'utf8' });
|
|
60
|
-
startedIdentity = stdout.trim();
|
|
61
|
-
} catch { fail('ADAPTER_CONTROLLER_UNAVAILABLE', `process start identity観測失敗: ${pid}`); }
|
|
62
|
-
if (!startedIdentity) fail('ADAPTER_CONTROLLER_UNAVAILABLE', `process不在: ${pid}`);
|
|
63
|
-
const identity = { schema: 'lattice.process_start_identity.v1', platform: process.platform, pid, started_identity: startedIdentity, identity_digest: '' };
|
|
64
|
-
identity.identity_digest = selfDigest(identity, 'identity_digest');
|
|
65
|
-
return identity;
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
export async function observeMacosBinaryIdentity(binaryPath) {
|
|
69
|
-
if (process.platform !== 'darwin') fail('ADAPTER_BINARY_IDENTITY_MISMATCH', 'macOS codesign identityはdarwin専用');
|
|
70
|
-
let output;
|
|
71
|
-
try {
|
|
72
|
-
const observed = await execFileAsync('/usr/bin/codesign',
|
|
73
|
-
['-d', '--verbose=4', '--requirements', '-', binaryPath], { encoding: 'utf8' });
|
|
74
|
-
output = `${observed.stdout ?? ''}\n${observed.stderr ?? ''}`;
|
|
75
|
-
} catch (error) {
|
|
76
|
-
output = `${error?.stdout ?? ''}\n${error?.stderr ?? ''}`;
|
|
77
|
-
if (!output.trim()) fail('ADAPTER_BINARY_IDENTITY_MISMATCH', 'codesign identity観測失敗');
|
|
78
|
-
}
|
|
79
|
-
const field = (name) => output.match(new RegExp(`(?:^|\\n)${name}=([^\\n]+)`, 'u'))?.[1]?.trim() ?? '';
|
|
80
|
-
const requirement = output.match(/(?:^|\n)#?\s*designated => ([^\n]+)/u)?.[1]?.trim() ?? '';
|
|
81
|
-
const rawCdhash = (output.match(/(?:^|\n)CandidateCDHashFull sha256=([0-9a-f]+)/u)?.[1]
|
|
82
|
-
?? field('CDHash')).toLowerCase();
|
|
83
|
-
if (!/^[0-9a-f]+$/u.test(rawCdhash) || !field('Identifier') || !requirement) {
|
|
84
|
-
fail('ADAPTER_BINARY_IDENTITY_MISMATCH', 'codesign出力の必須identity不足');
|
|
85
|
-
}
|
|
86
|
-
const identity = {
|
|
87
|
-
schema: 'lattice.macos_binary_identity.v1', kind: 'macos_codesign',
|
|
88
|
-
cdhash: rawCdhash, signing_identifier: field('Identifier'),
|
|
89
|
-
team_identifier: field('TeamIdentifier'),
|
|
90
|
-
designated_requirement_digest: sha256Bytes(Buffer.from(requirement)), identity_digest: '',
|
|
91
|
-
};
|
|
92
|
-
identity.identity_digest = selfDigest(identity, 'identity_digest');
|
|
93
|
-
return identity;
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
/**
|
|
97
|
-
* 起動済みprocessが実際に実行しているimageのcanonical pathをOSから観測する。
|
|
98
|
-
* bytesの同一性はcallerがlaunch descriptorのSHA-256と照合する。
|
|
99
|
-
*/
|
|
100
|
-
export async function observeProcessExecutablePath(pid) {
|
|
101
|
-
if (!Number.isSafeInteger(pid) || pid < 1) {
|
|
102
|
-
fail('ADAPTER_BINARY_IDENTITY_MISMATCH', 'exec後image PID不正');
|
|
103
|
-
}
|
|
104
|
-
if (process.platform === 'linux') {
|
|
105
|
-
return realpath(`/proc/${pid}/exe`);
|
|
106
|
-
}
|
|
107
|
-
if (process.platform === 'darwin') {
|
|
108
|
-
const { stdout } = await execFileAsync(
|
|
109
|
-
'/bin/ps',
|
|
110
|
-
['-o', 'comm=', '-p', String(pid)],
|
|
111
|
-
{ encoding: 'utf8' },
|
|
112
|
-
);
|
|
113
|
-
return realpath(stdout.trim());
|
|
114
|
-
}
|
|
115
|
-
fail('ADAPTER_BINARY_IDENTITY_MISMATCH', `exec後image path観測未対応platform: ${process.platform}`);
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
/** storeが解決したimmutable bindingからprocess/worktree/checkpointをDirect OSで再観測する。 */
|
|
119
|
-
/**
|
|
120
|
-
* 本repositoryの現在状態を1つのdigestへ畳む。
|
|
121
|
-
*
|
|
122
|
-
* HEAD・作業ツリー状態(untracked/ignoredを含む)・全refを見る。workerがworktreeの外へ
|
|
123
|
-
* 書いてもworktreeのdiffには映らないので、ここだけが本repositoryへの書き込みを捕まえる。
|
|
124
|
-
*/
|
|
125
|
-
export async function canonicalRepositoryFingerprint(repoRoot) {
|
|
126
|
-
const parts = [];
|
|
127
|
-
for (const args of [
|
|
128
|
-
['rev-parse', 'HEAD'],
|
|
129
|
-
['status', '--porcelain=v1', '--untracked-files=all', '--ignored=matching'],
|
|
130
|
-
['for-each-ref', '--format=%(refname) %(objectname)'],
|
|
131
|
-
]) {
|
|
132
|
-
const { stdout } = await execFileAsync('git', args, {
|
|
133
|
-
cwd: repoRoot, encoding: 'utf8', maxBuffer: 32 * 1024 * 1024,
|
|
134
|
-
});
|
|
135
|
-
parts.push(stdout);
|
|
136
|
-
}
|
|
137
|
-
return createHash('sha256').update(parts.join('\u0000'), 'utf8').digest('hex');
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
export function createDirectOsProcessObserver({ resolveObservationBinding }) {
|
|
141
|
-
if (typeof resolveObservationBinding !== 'function') fail('SUPERVISOR_CONFIGURATION_INVALID', 'observation binding resolverなし');
|
|
142
|
-
return async ({ kind, binding, ack }) => {
|
|
143
|
-
if (!['quiescence', 'rebind'].includes(kind)) fail('HOLD_ACKS_INCOMPLETE', '未知のDirect OS observation kind');
|
|
144
|
-
const resolved = await resolveObservationBinding({ binding: structuredClone(binding), ack: structuredClone(ack) });
|
|
145
|
-
if (resolved === null || typeof resolved !== 'object' || Array.isArray(resolved)
|
|
146
|
-
|| !Number.isSafeInteger(resolved.process_pid) || resolved.process_pid < 1
|
|
147
|
-
|| !Number.isSafeInteger(resolved.process_group_id) || resolved.process_group_id < 1
|
|
148
|
-
|| typeof resolved.worktree_path !== 'string' || !path.isAbsolute(resolved.worktree_path)
|
|
149
|
-
|| typeof resolved.base_sha !== 'string' || !/^[0-9a-f]{40}$/.test(resolved.base_sha)
|
|
150
|
-
|| !validateProcessStartIdentity(resolved.process_start_identity)
|
|
151
|
-
|| resolved.process_start_identity.pid !== resolved.process_pid) fail('HOLD_ACKS_INCOMPLETE', 'observation binding不正');
|
|
152
|
-
// 本repositoryを見る材料は対で渡す。片方だけでは照合できず、片方だけを受けると
|
|
153
|
-
// 「検査した」と読める記録が検査なしで作れてしまう。
|
|
154
|
-
const canonicalRoot = resolved.canonical_root ?? null;
|
|
155
|
-
const canonicalBaseline = resolved.canonical_fingerprint_digest ?? null;
|
|
156
|
-
if ((canonicalRoot === null) !== (canonicalBaseline === null)
|
|
157
|
-
|| (canonicalRoot !== null && !path.isAbsolute(canonicalRoot))
|
|
158
|
-
|| (canonicalBaseline !== null && !/^[0-9a-f]{64}$/u.test(canonicalBaseline))) {
|
|
159
|
-
fail('HOLD_ACKS_INCOMPLETE', 'canonical repository観測bindingが不正');
|
|
160
|
-
}
|
|
161
|
-
let processState = 'exited';
|
|
162
|
-
let observedIdentity = resolved.process_start_identity;
|
|
163
|
-
let processGroupId = resolved.process_group_id;
|
|
164
|
-
try {
|
|
165
|
-
const { stdout } = await execFileAsync('/bin/ps', ['-o', 'lstart=,pgid=,state=', '-p', String(resolved.process_pid)], { encoding: 'utf8' });
|
|
166
|
-
const line = stdout.trim();
|
|
167
|
-
if (line) {
|
|
168
|
-
const match = line.match(/^(.*\d{4})\s+(\d+)\s+(\S+)$/);
|
|
169
|
-
if (!match) fail('HOLD_ACKS_INCOMPLETE', 'process observation parse失敗');
|
|
170
|
-
observedIdentity = { schema: 'lattice.process_start_identity.v1', platform: process.platform, pid: resolved.process_pid, started_identity: match[1].trim(), identity_digest: '' };
|
|
171
|
-
observedIdentity.identity_digest = selfDigest(observedIdentity, 'identity_digest');
|
|
172
|
-
if (observedIdentity.identity_digest !== resolved.process_start_identity.identity_digest) fail('HOLD_ACKS_INCOMPLETE', 'PID再利用又はstart identity差替え');
|
|
173
|
-
processGroupId = Number(match[2]);
|
|
174
|
-
if (processGroupId !== resolved.process_group_id) fail('HOLD_ACKS_INCOMPLETE', 'process group差替え');
|
|
175
|
-
processState = match[3].startsWith('T') ? 'stopped' : 'running';
|
|
176
|
-
}
|
|
177
|
-
} catch (error) {
|
|
178
|
-
if (error instanceof ManagedRuntimeError) throw error;
|
|
179
|
-
processState = 'exited';
|
|
180
|
-
}
|
|
181
|
-
if (!['stopped', 'exited'].includes(processState)) fail('HOLD_ACKS_INCOMPLETE', 'executor processがquiescedでない');
|
|
182
|
-
try {
|
|
183
|
-
const { stdout: groupStdout } = await execFileAsync('/bin/ps', ['-o', 'pid=,state=', '-g', String(resolved.process_group_id)], { encoding: 'utf8' });
|
|
184
|
-
for (const line of groupStdout.trim().split('\n').filter(Boolean)) {
|
|
185
|
-
const match = line.trim().match(/^(\d+)\s+(\S+)$/);
|
|
186
|
-
if (!match || (!match[2].startsWith('T') && !match[2].startsWith('Z'))) fail('HOLD_ACKS_INCOMPLETE', 'process group childがquiescedでない');
|
|
187
|
-
}
|
|
188
|
-
} catch (error) {
|
|
189
|
-
if (error instanceof ManagedRuntimeError) throw error;
|
|
190
|
-
if (!(error?.code === 1 && String(error?.stdout ?? '').trim() === '')) {
|
|
191
|
-
fail('HOLD_ACKS_INCOMPLETE', 'process group直接観測失敗');
|
|
192
|
-
}
|
|
193
|
-
// exit 1かつ出力空だけをgroup消滅済みと解釈する。
|
|
194
|
-
}
|
|
195
|
-
const worktreeInfo = await lstat(resolved.worktree_path);
|
|
196
|
-
if (!worktreeInfo.isDirectory() || worktreeInfo.isSymbolicLink()) fail('HOLD_ACKS_INCOMPLETE', 'worktreeがreal directoryでない');
|
|
197
|
-
const worktreeRealpath = await realpath(resolved.worktree_path);
|
|
198
|
-
if (worktreeRealpath !== resolved.worktree_path) fail('HOLD_ACKS_INCOMPLETE', 'worktree path差替え');
|
|
199
|
-
const checkpoint = await captureWorktreeDiff({ worktreePath: worktreeRealpath, baseSha: resolved.base_sha });
|
|
200
|
-
const processObservation = { schema: 'lattice.direct_process_observation.v1', pid: resolved.process_pid, process_start_identity_digest: observedIdentity.identity_digest, process_group_id: processGroupId, state: processState };
|
|
201
|
-
const processObservationDigest = digestArtifact(processObservation);
|
|
202
|
-
// worktreeの外——本repository——への書き込みは、worktreeのdiffにはまったく映らない。
|
|
203
|
-
// 見ていないことを「変更が無かった」と読ませないため、検査したかどうかを記録へ残す
|
|
204
|
-
// (ADR 0140)。渡されていなければ`null`=未検査であり、無変更の主張ではない。
|
|
205
|
-
let canonicalDigest = null;
|
|
206
|
-
if (canonicalRoot !== null) {
|
|
207
|
-
canonicalDigest = await canonicalRepositoryFingerprint(canonicalRoot);
|
|
208
|
-
if (canonicalDigest !== canonicalBaseline) {
|
|
209
|
-
fail('HOLD_ACKS_INCOMPLETE', 'canonical repositoryがworker実行中に変化した');
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
const worktreeFingerprint = { schema: 'lattice.direct_worktree_fingerprint.v2', worktree_id: binding?.worktree_id ?? ack.worktree_id, worktree_realpath: worktreeRealpath, checkpoint_digest: checkpoint.checkpoint_digest, canonical_fingerprint_digest: canonicalDigest };
|
|
213
|
-
const worktreeFingerprintDigest = digestArtifact(worktreeFingerprint);
|
|
214
|
-
return {
|
|
215
|
-
quiesced: true,
|
|
216
|
-
process_observation_digest: processObservationDigest,
|
|
217
|
-
worktree_fingerprint_digest: worktreeFingerprintDigest,
|
|
218
|
-
final_checkpoint_digest: checkpoint.checkpoint_digest,
|
|
219
|
-
observation: processObservation,
|
|
220
|
-
worktree_fingerprint: worktreeFingerprint,
|
|
221
|
-
checkpoint,
|
|
222
|
-
write_enabled: false,
|
|
223
|
-
};
|
|
224
|
-
};
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
async function readCanonicalRegular(filePath) {
|
|
228
|
-
const info = await lstat(filePath);
|
|
229
|
-
if (!info.isFile() || info.isSymbolicLink()) fail('ADAPTER_LAUNCH_INVALID', `regular fileでない: ${filePath}`);
|
|
230
|
-
const bytes = await readFile(filePath);
|
|
231
|
-
let value;
|
|
232
|
-
try { value = JSON.parse(bytes); } catch { fail('ADAPTER_LAUNCH_INVALID', `JSON不正: ${filePath}`); }
|
|
233
|
-
if (bytes.toString('utf8') !== `${canonicalizeArtifact(value)}\n`) fail('ADAPTER_LAUNCH_INVALID', `canonical bytesでない: ${filePath}`);
|
|
234
|
-
return { value, bytes };
|
|
235
|
-
}
|
|
236
|
-
|
|
237
|
-
function validateRunningBinding(value) {
|
|
238
|
-
const fields = ['todo_id', 'executor_handle', 'worktree_id', 'plan_epoch', 'packet_digest', 'write_lease_id', 'controller_registration_digest'];
|
|
239
|
-
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
|
240
|
-
&& Object.keys(value).sort().join('\0') === fields.sort().join('\0')
|
|
241
|
-
&& identifier(value.todo_id) && identifier(value.executor_handle) && identifier(value.worktree_id)
|
|
242
|
-
&& Number.isSafeInteger(value.plan_epoch) && value.plan_epoch > 0
|
|
243
|
-
&& digest(value.packet_digest) && identifier(value.write_lease_id)
|
|
244
|
-
&& digest(value.controller_registration_digest);
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
/**
|
|
248
|
-
* ADR 0064のcontrol plane本体。永続化はcallerが渡すjournal/gateWriterへ明示委譲し、
|
|
249
|
-
* controller transportとOS再観測なしには停止・再認可を成功扱いしない。
|
|
250
|
-
*/
|
|
251
|
-
export class RuntimeManagedSupervisor {
|
|
252
|
-
#runId;
|
|
253
|
-
#sessionNonceDigest;
|
|
254
|
-
#clock;
|
|
255
|
-
#processObserver;
|
|
256
|
-
#runningBindingResolver;
|
|
257
|
-
#journal;
|
|
258
|
-
#gateWriter;
|
|
259
|
-
#shutdownFinalizer;
|
|
260
|
-
#controllers = new Map();
|
|
261
|
-
#registrationToController = new Map();
|
|
262
|
-
#leases = new Map();
|
|
263
|
-
#gate = null;
|
|
264
|
-
#previousGate = null;
|
|
265
|
-
#releaseAcks = [];
|
|
266
|
-
#gateRecords = new Map();
|
|
267
|
-
#frozen = false;
|
|
268
|
-
#gateGeneration = 0;
|
|
269
|
-
|
|
270
|
-
constructor({ runId, sessionNonceDigest, clock = monotonicDefault, processObserver, runningBindingResolver, journal, gateWriter, shutdownFinalizer = async () => {} }) {
|
|
271
|
-
if (!identifier(runId) || !digest(sessionNonceDigest) || typeof clock !== 'function'
|
|
272
|
-
|| typeof processObserver !== 'function' || typeof runningBindingResolver !== 'function'
|
|
273
|
-
|| typeof journal?.append !== 'function'
|
|
274
|
-
|| typeof gateWriter?.commit !== 'function' || typeof shutdownFinalizer !== 'function') fail('SUPERVISOR_CONFIGURATION_INVALID', 'durable dependencyが不足');
|
|
275
|
-
this.#runId = runId;
|
|
276
|
-
this.#sessionNonceDigest = sessionNonceDigest;
|
|
277
|
-
this.#clock = clock;
|
|
278
|
-
this.#processObserver = processObserver;
|
|
279
|
-
this.#runningBindingResolver = runningBindingResolver;
|
|
280
|
-
this.#journal = journal;
|
|
281
|
-
this.#gateWriter = gateWriter;
|
|
282
|
-
this.#shutdownFinalizer = shutdownFinalizer;
|
|
283
|
-
}
|
|
284
|
-
|
|
285
|
-
async hydrateCommittedGate() {
|
|
286
|
-
if (typeof this.#gateWriter.read !== 'function') return null;
|
|
287
|
-
const committed = await this.#gateWriter.read();
|
|
288
|
-
if (committed === null) return null;
|
|
289
|
-
if (committed.gate.run_id !== this.#runId
|
|
290
|
-
|| !Number.isSafeInteger(committed.gate.gate_generation)
|
|
291
|
-
|| committed.gate.gate_generation < 1) {
|
|
292
|
-
fail('EPOCH_ACTIVATION_INCOMPLETE', 'persisted gate generation binding不正');
|
|
293
|
-
}
|
|
294
|
-
this.#gate = structuredClone(committed.gate);
|
|
295
|
-
this.#gateGeneration = committed.gate.gate_generation;
|
|
296
|
-
return structuredClone(committed.gate);
|
|
297
|
-
}
|
|
298
|
-
|
|
299
|
-
get runId() { return this.#runId; }
|
|
300
|
-
get frozen() { return this.#frozen; }
|
|
301
|
-
get gate() { return this.#gate === null ? null : structuredClone(this.#gate); }
|
|
302
|
-
|
|
303
|
-
async registerController({ descriptor, registration, transport }) {
|
|
304
|
-
if (!validateControllerDescriptor(descriptor) || !validateControllerRegistration(registration)
|
|
305
|
-
|| registration.run_id !== this.#runId
|
|
306
|
-
|| registration.supervisor_session_nonce_digest !== this.#sessionNonceDigest
|
|
307
|
-
|| registration.controller_descriptor_digest !== descriptor.descriptor_digest
|
|
308
|
-
|| typeof transport?.request !== 'function') fail('ADAPTER_CONTROLLER_UNAVAILABLE', 'controller registration binding不正');
|
|
309
|
-
if (this.#controllers.has(descriptor.controller_id)
|
|
310
|
-
|| this.#registrationToController.has(registration.registration_digest)) fail('ADAPTER_CONTROLLER_UNAVAILABLE', 'controller二重登録');
|
|
311
|
-
const now = this.#clock();
|
|
312
|
-
const record = { descriptor: structuredClone(descriptor), registration: structuredClone(registration),
|
|
313
|
-
transport, lastHeartbeat: now, lastHeartbeatSequence: 0,
|
|
314
|
-
lastRecordedLeaseSetDigest: null, connected: true, revoked: false };
|
|
315
|
-
this.#controllers.set(descriptor.controller_id, record);
|
|
316
|
-
this.#registrationToController.set(registration.registration_digest, descriptor.controller_id);
|
|
317
|
-
await this.#append('controller_registered', { controller_id: descriptor.controller_id, registration_digest: registration.registration_digest });
|
|
318
|
-
return structuredClone(registration);
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
async heartbeat({ controllerId, registrationDigest, sequence, leaseSetDigest, sessionNonceDigest }) {
|
|
322
|
-
const record = this.#requireController(controllerId);
|
|
323
|
-
if (!record.connected || sessionNonceDigest !== this.#sessionNonceDigest
|
|
324
|
-
|| registrationDigest !== record.registration.registration_digest
|
|
325
|
-
|| !Number.isSafeInteger(sequence) || sequence <= record.lastHeartbeatSequence || !digest(leaseSetDigest)) {
|
|
326
|
-
await this.#failClosed(record, 'CONTROLLER_HEARTBEAT_EXPIRED', 'heartbeat binding不正');
|
|
327
|
-
}
|
|
328
|
-
// prepare/release応答とsupervisor投影の間には短い非同期窓がある。heartbeatは
|
|
329
|
-
// livenessだけを担い、lease集合はwrite認可時のcentral gate full-chainで照合する。
|
|
330
|
-
record.lastHeartbeat = this.#clock();
|
|
331
|
-
record.lastHeartbeatSequence = sequence;
|
|
332
|
-
// heartbeatはlivenessの更新であり、同じlease集合のtickを全件journalへ積む必要はない。
|
|
333
|
-
// 初回とlease集合の変化だけを耐久記録し、壁時計に比例するjournal成長を止める。
|
|
334
|
-
if (record.lastRecordedLeaseSetDigest !== leaseSetDigest) {
|
|
335
|
-
await this.#append('controller_heartbeat', { controller_id: controllerId,
|
|
336
|
-
registration_digest: registrationDigest, sequence, lease_set_digest: leaseSetDigest });
|
|
337
|
-
record.lastRecordedLeaseSetDigest = leaseSetDigest;
|
|
338
|
-
}
|
|
339
|
-
}
|
|
340
|
-
|
|
341
|
-
async disconnect(controllerId) {
|
|
342
|
-
const record = this.#requireController(controllerId);
|
|
343
|
-
record.connected = false;
|
|
344
|
-
await this.#failClosed(record, 'CONTROLLER_HEARTBEAT_EXPIRED', 'controller socket disconnected');
|
|
345
|
-
}
|
|
346
|
-
|
|
347
|
-
async assertControllerHealth() {
|
|
348
|
-
for (const record of this.#controllers.values()) {
|
|
349
|
-
const age = this.#clock() - record.lastHeartbeat;
|
|
350
|
-
if (!record.connected || age < 0 || age > record.descriptor.heartbeat.ttl_ms) {
|
|
351
|
-
await this.#failClosed(record, 'CONTROLLER_HEARTBEAT_EXPIRED', 'heartbeat TTL超過');
|
|
352
|
-
}
|
|
353
|
-
}
|
|
354
|
-
}
|
|
355
|
-
|
|
356
|
-
async route(operation, controllerId, fields) {
|
|
357
|
-
await this.assertControllerHealth();
|
|
358
|
-
if (this.#frozen && ['dispatch', 'observe'].includes(operation)) fail('RUN_FROZEN', 'freeze中のroute');
|
|
359
|
-
const record = this.#requireController(controllerId);
|
|
360
|
-
const request = createControllerRequest(operation, {
|
|
361
|
-
request_id: fields.request_id ?? randomUUID(),
|
|
362
|
-
registration_digest: record.registration.registration_digest,
|
|
363
|
-
...Object.fromEntries(Object.entries(fields).filter(([key]) => !['request_id', 'registration_digest'].includes(key))),
|
|
364
|
-
});
|
|
365
|
-
const response = await record.transport.request(operation, structuredClone(request));
|
|
366
|
-
if (!validateControllerResponse(operation, response, request.request_id)) {
|
|
367
|
-
await this.#failClosed(record, 'ADAPTER_CONTROLLER_UNAVAILABLE', `${operation} response不正`);
|
|
368
|
-
}
|
|
369
|
-
// running pollは状態遷移ではない。全tickを耐久化すると長寿命workerほどcontrol journalを
|
|
370
|
-
// 膨らませ、artifact document上限でsupervisor自身を落とす。dispatchと非running観測だけを残す。
|
|
371
|
-
if (operation === 'dispatch' || response.observation?.state !== 'running') {
|
|
372
|
-
await this.#append(operation === 'dispatch' ? 'dispatch_routed' : 'observation_routed', {
|
|
373
|
-
controller_id: controllerId, request_digest: request.request_digest,
|
|
374
|
-
response_digest: response.response_digest,
|
|
375
|
-
});
|
|
376
|
-
}
|
|
377
|
-
return structuredClone(response);
|
|
378
|
-
}
|
|
379
|
-
|
|
380
|
-
/** 全running bindingを一件も省略せずbarrierし、controller ackと独立OS観測を照合する。 */
|
|
381
|
-
async barrierAll({ barrierId, reason, frozenEventDigest }) {
|
|
382
|
-
return this.#barrier({ barrierId, reason, frozenEventDigest, targetTodoIds: null });
|
|
383
|
-
}
|
|
384
|
-
|
|
385
|
-
/** findingの影響群だけを静止し、無関係なrunning bindingはそのまま通す。 */
|
|
386
|
-
async barrierSelected({ barrierId, reason, frozenEventDigest, todoIds }) {
|
|
387
|
-
if (!Array.isArray(todoIds) || todoIds.length === 0
|
|
388
|
-
|| todoIds.some((todoId) => !identifier(todoId))
|
|
389
|
-
|| new Set(todoIds).size !== todoIds.length) {
|
|
390
|
-
fail('HOLD_ACKS_INCOMPLETE', '対象barrierのTODO集合が不正');
|
|
391
|
-
}
|
|
392
|
-
return this.#barrier({ barrierId, reason, frozenEventDigest,
|
|
393
|
-
targetTodoIds: new Set(todoIds) });
|
|
394
|
-
}
|
|
395
|
-
|
|
396
|
-
async #barrier({ barrierId, reason, frozenEventDigest, targetTodoIds }) {
|
|
397
|
-
const durableBindings = await this.#runningBindingResolver({ runId: this.#runId, frozenEventDigest });
|
|
398
|
-
if (!identifier(barrierId) || typeof reason !== 'string' || reason.length === 0
|
|
399
|
-
|| !digest(frozenEventDigest) || !Array.isArray(durableBindings)
|
|
400
|
-
|| !durableBindings.every(validateRunningBinding)) fail('HOLD_ACKS_INCOMPLETE', 'durable running binding不正');
|
|
401
|
-
await this.assertControllerHealth();
|
|
402
|
-
this.#frozen = true;
|
|
403
|
-
const allRunningBindings = await this.#resolveRunningUnion({ durableBindings, frozenEventDigest });
|
|
404
|
-
const runningBindings = targetTodoIds === null ? allRunningBindings
|
|
405
|
-
: allRunningBindings.filter((binding) => targetTodoIds.has(binding.todo_id));
|
|
406
|
-
const unique = new Set(runningBindings.map((binding) => binding.todo_id));
|
|
407
|
-
if (unique.size !== runningBindings.length) fail('HOLD_ACKS_INCOMPLETE', 'running TODO重複');
|
|
408
|
-
const barrierControlDigest = await this.#append('barrier_requested', { barrier_id: barrierId,
|
|
409
|
-
reason, running_count: runningBindings.length,
|
|
410
|
-
running_todo_ids: runningBindings.map((binding) => binding.todo_id).sort(),
|
|
411
|
-
frozen_event_digest: frozenEventDigest });
|
|
412
|
-
if (!digest(barrierControlDigest)) fail('HOLD_ACKS_INCOMPLETE', 'barrier control event未耐久化');
|
|
413
|
-
const acknowledgements = [];
|
|
414
|
-
for (const record of this.#controllers.values()) {
|
|
415
|
-
const owned = runningBindings.filter((binding) => binding.controller_registration_digest === record.registration.registration_digest);
|
|
416
|
-
if (owned.length === 0) continue;
|
|
417
|
-
const response = await this.routeWhileFrozen('barrier', record, {
|
|
418
|
-
request_id: randomUUID(), barrier_id: barrierId, reason,
|
|
419
|
-
running_bindings: owned, frozen_event_digest: frozenEventDigest,
|
|
420
|
-
barrier_control_digest: barrierControlDigest,
|
|
421
|
-
});
|
|
422
|
-
if (response.barrier_id !== barrierId || !Array.isArray(response.quiescence_acks)
|
|
423
|
-
|| response.quiescence_acks.length !== owned.length) fail('HOLD_ACKS_INCOMPLETE', 'barrier ack件数不一致');
|
|
424
|
-
for (const binding of owned) {
|
|
425
|
-
const ack = response.quiescence_acks.find((candidate) => candidate.todo_id === binding.todo_id);
|
|
426
|
-
if (!validateQuiescenceAck(ack) || ack.run_id !== this.#runId
|
|
427
|
-
|| ack.executor_handle !== binding.executor_handle || ack.worktree_id !== binding.worktree_id
|
|
428
|
-
|| ack.plan_epoch !== binding.plan_epoch || ack.packet_digest !== binding.packet_digest
|
|
429
|
-
|| ack.write_lease_id !== binding.write_lease_id
|
|
430
|
-
|| ack.barrier_control_digest !== barrierControlDigest
|
|
431
|
-
|| ack.supervisor_session_nonce_digest !== this.#sessionNonceDigest) fail('HOLD_ACKS_INCOMPLETE', `ack binding不一致: ${binding.todo_id}`);
|
|
432
|
-
const observed = await this.#processObserver({ kind: 'quiescence', binding: structuredClone(binding), ack: structuredClone(ack), descriptor: structuredClone(record.descriptor) });
|
|
433
|
-
if (observed?.process_observation_digest !== ack.process_observation_digest
|
|
434
|
-
|| observed?.worktree_fingerprint_digest !== ack.worktree_fingerprint_digest
|
|
435
|
-
|| observed?.final_checkpoint_digest !== ack.final_checkpoint_digest
|
|
436
|
-
|| observed?.quiesced !== true) fail('HOLD_ACKS_INCOMPLETE', `直接再観測不一致: ${binding.todo_id}`);
|
|
437
|
-
acknowledgements.push(ack);
|
|
438
|
-
const directObservation = structuredClone(observed);
|
|
439
|
-
await this.#append('executor_quiesced', { barrier_id: barrierId,
|
|
440
|
-
barrier_control_digest: barrierControlDigest, todo_id: binding.todo_id,
|
|
441
|
-
ack: structuredClone(ack), direct_observation: directObservation,
|
|
442
|
-
evidence_digest: digestArtifact({ ack, direct_observation: directObservation }) });
|
|
443
|
-
}
|
|
444
|
-
}
|
|
445
|
-
if (acknowledgements.length !== runningBindings.length) fail('HOLD_ACKS_INCOMPLETE', '未登録controller所有bindingあり');
|
|
446
|
-
const observedAfterBarrier = await this.#collectControllerRunning({ frozenEventDigest });
|
|
447
|
-
const residualBindings = targetTodoIds === null ? observedAfterBarrier
|
|
448
|
-
: observedAfterBarrier.filter((binding) => targetTodoIds.has(binding.todo_id));
|
|
449
|
-
if (residualBindings.length !== 0) {
|
|
450
|
-
fail('HOLD_ACKS_INCOMPLETE', `barrier後もrunning executor残存: ${residualBindings.map((binding) => binding.todo_id).join(',')}`);
|
|
451
|
-
}
|
|
452
|
-
return acknowledgements.map((ack) => structuredClone(ack));
|
|
453
|
-
}
|
|
454
|
-
|
|
455
|
-
/** durable storeと全controllerの直接inventoryをexact binding単位で和集合にする。 */
|
|
456
|
-
async #resolveRunningUnion({ durableBindings, frozenEventDigest }) {
|
|
457
|
-
const observedBindings = await this.#collectControllerRunning({ frozenEventDigest });
|
|
458
|
-
const union = new Map();
|
|
459
|
-
const executorOwners = new Map();
|
|
460
|
-
for (const binding of [...durableBindings, ...observedBindings]) {
|
|
461
|
-
const controllerId = this.#registrationToController.get(binding.controller_registration_digest);
|
|
462
|
-
if (controllerId === undefined) fail('HOLD_ACKS_INCOMPLETE', `未登録controller binding: ${binding.todo_id}`);
|
|
463
|
-
const canonical = canonicalizeArtifact(binding);
|
|
464
|
-
const prior = union.get(binding.todo_id);
|
|
465
|
-
if (prior !== undefined && prior.canonical !== canonical) {
|
|
466
|
-
fail('HOLD_ACKS_INCOMPLETE', `durable/controller running binding不一致: ${binding.todo_id}`);
|
|
467
|
-
}
|
|
468
|
-
const executorOwner = executorOwners.get(binding.executor_handle);
|
|
469
|
-
if (executorOwner !== undefined && executorOwner !== binding.todo_id) {
|
|
470
|
-
fail('HOLD_ACKS_INCOMPLETE', `executor handle重複: ${binding.executor_handle}`);
|
|
471
|
-
}
|
|
472
|
-
union.set(binding.todo_id, { canonical, binding: structuredClone(binding) });
|
|
473
|
-
executorOwners.set(binding.executor_handle, binding.todo_id);
|
|
474
|
-
}
|
|
475
|
-
return [...union.values()].map(({ binding }) => binding)
|
|
476
|
-
.sort((left, right) => left.todo_id < right.todo_id ? -1 : left.todo_id > right.todo_id ? 1 : 0);
|
|
477
|
-
}
|
|
478
|
-
|
|
479
|
-
async #collectControllerRunning({ frozenEventDigest }) {
|
|
480
|
-
const observed = [];
|
|
481
|
-
for (const record of this.#controllers.values()) {
|
|
482
|
-
const response = await this.routeWhileFrozen('inventory', record, {
|
|
483
|
-
request_id: randomUUID(), frozen_event_digest: frozenEventDigest,
|
|
484
|
-
});
|
|
485
|
-
if (response.inventory_digest !== digestArtifact(response.running_bindings)
|
|
486
|
-
|| response.running_bindings.some((binding) => (
|
|
487
|
-
binding.controller_registration_digest !== record.registration.registration_digest))) {
|
|
488
|
-
fail('HOLD_ACKS_INCOMPLETE', `controller inventory binding不一致: ${record.descriptor.controller_id}`);
|
|
489
|
-
}
|
|
490
|
-
observed.push(...response.running_bindings.map((binding) => structuredClone(binding)));
|
|
491
|
-
}
|
|
492
|
-
return observed;
|
|
493
|
-
}
|
|
494
|
-
|
|
495
|
-
/** finding/store永続化後に呼ぶtyped hold閉路。対象群のquiescence完了後だけheldを返す。 */
|
|
496
|
-
async holdConflict({ findingDigest, frozenEventDigest, barrierId, reason, recordedAt,
|
|
497
|
-
todoIds = null }) {
|
|
498
|
-
if (!digest(findingDigest) || typeof recordedAt !== 'string' || Number.isNaN(Date.parse(recordedAt))) fail('FINDING_UNRESOLVED', 'hold finding binding不正');
|
|
499
|
-
const acknowledgements = todoIds === null
|
|
500
|
-
? await this.barrierAll({ barrierId, reason, frozenEventDigest })
|
|
501
|
-
: await this.barrierSelected({ barrierId, reason, frozenEventDigest,
|
|
502
|
-
todoIds });
|
|
503
|
-
const result = {
|
|
504
|
-
schema: 'lattice.runtime_hold_result.v1', run_id: this.#runId,
|
|
505
|
-
finding_digest: findingDigest, barrier_id: barrierId,
|
|
506
|
-
quiescence_ack_digests: acknowledgements.map((ack) => ack.ack_digest).sort(),
|
|
507
|
-
outcome: 'held', recorded_at: recordedAt, result_digest: '',
|
|
508
|
-
};
|
|
509
|
-
result.result_digest = selfDigest(result, 'result_digest');
|
|
510
|
-
return result;
|
|
511
|
-
}
|
|
512
|
-
|
|
513
|
-
async shutdownManaged({ mode, reason, barrierId, frozenEventDigest, recordedAt }) {
|
|
514
|
-
if (!['close', 'abandon'].includes(mode) || typeof reason !== 'string' || reason.length === 0
|
|
515
|
-
|| typeof recordedAt !== 'string' || Number.isNaN(Date.parse(recordedAt))) fail('MANAGED_SHUTDOWN_INCOMPLETE', 'shutdown入力不正');
|
|
516
|
-
const acknowledgements = await this.barrierAll({ barrierId, reason: `managed_${mode}:${reason}`, frozenEventDigest });
|
|
517
|
-
for (const record of this.#controllers.values()) {
|
|
518
|
-
const owned = [...this.#leases.values()].filter((entry) => entry.controllerId === record.descriptor.controller_id && !entry.revoked);
|
|
519
|
-
const leaseDigests = owned.map((entry) => entry.lease.lease_digest).sort();
|
|
520
|
-
const response = await this.routeWhileFrozen('revoke', record, { request_id: randomUUID(), reason, lease_digests: leaseDigests });
|
|
521
|
-
if (JSON.stringify(response.revoked_lease_digests) !== JSON.stringify(leaseDigests)
|
|
522
|
-
|| response.residual_processes.length !== 0) fail('MANAGED_SHUTDOWN_INCOMPLETE', 'controller revoke不完全');
|
|
523
|
-
for (const entry of owned) entry.revoked = true;
|
|
524
|
-
await this.#append('lease_revoked', { controller_id: record.descriptor.controller_id, reason });
|
|
525
|
-
}
|
|
526
|
-
await this.#shutdownFinalizer();
|
|
527
|
-
const result = { schema: 'lattice.runtime_shutdown_result.v1', run_id: this.#runId, mode, outcome: 'quiesced', quiescence_ack_digests: acknowledgements.map((ack) => ack.ack_digest).sort(), recorded_at: recordedAt, result_digest: '' };
|
|
528
|
-
result.result_digest = selfDigest(result, 'result_digest');
|
|
529
|
-
return result;
|
|
530
|
-
}
|
|
531
|
-
|
|
532
|
-
async rebindController({ controllerId, rebindPacket, stagedLease, expected }) {
|
|
533
|
-
const record = this.#requireController(controllerId);
|
|
534
|
-
if (!this.#frozen) fail('EPOCH_REBIND_INCOMPLETE', 'freeze外rebind');
|
|
535
|
-
const response = await this.routeWhileFrozen('rebind', record, {
|
|
536
|
-
request_id: randomUUID(), rebind_packet: rebindPacket, staged_lease: stagedLease,
|
|
537
|
-
});
|
|
538
|
-
if (response.staged_lease_digest !== stagedLease.lease_digest) fail('EPOCH_REBIND_INCOMPLETE', 'staged lease response不一致');
|
|
539
|
-
const controlEventDigest = await this.#acceptRebindAck({ controllerId,
|
|
540
|
-
ack: response.rebind_ack, stagedLease, expected });
|
|
541
|
-
return { ack: structuredClone(response.rebind_ack), control_event_digest: controlEventDigest };
|
|
542
|
-
}
|
|
543
|
-
|
|
544
|
-
/** affected successorを実行不能なstaged leaseのままprepareする。 */
|
|
545
|
-
async prepareController({ controllerId, executorPacket, stagedLease }) {
|
|
546
|
-
const record = this.#requireController(controllerId);
|
|
547
|
-
if (!this.#frozen || !validateStagedWriteLease(stagedLease)
|
|
548
|
-
|| stagedLease.run_id !== this.#runId || stagedLease.state !== 'staged'
|
|
549
|
-
|| stagedLease.todo_id !== executorPacket?.todo_id
|
|
550
|
-
|| stagedLease.packet_digest !== executorPacket?.packet_digest
|
|
551
|
-
|| stagedLease.controller_registration_digest !== record.registration.registration_digest) {
|
|
552
|
-
fail('EPOCH_REBIND_INCOMPLETE', 'prepare staged binding不正');
|
|
553
|
-
}
|
|
554
|
-
const response = await this.routeWhileFrozen('prepare', record, {
|
|
555
|
-
request_id: randomUUID(), executor_packet: executorPacket, staged_lease: stagedLease,
|
|
556
|
-
});
|
|
557
|
-
const ack = response.prepare_ack;
|
|
558
|
-
if (response.staged_lease_digest !== stagedLease.lease_digest
|
|
559
|
-
|| ack.run_id !== this.#runId || ack.todo_id !== executorPacket.todo_id
|
|
560
|
-
|| ack.plan_epoch !== executorPacket.plan_epoch
|
|
561
|
-
|| ack.packet_digest !== executorPacket.packet_digest
|
|
562
|
-
|| ack.staged_lease_digest !== stagedLease.lease_digest
|
|
563
|
-
|| ack.registration_digest !== record.registration.registration_digest
|
|
564
|
-
|| ack.controller_id !== controllerId
|
|
565
|
-
|| ack.supervisor_session_nonce_digest !== this.#sessionNonceDigest) {
|
|
566
|
-
fail('EPOCH_REBIND_INCOMPLETE', 'direct prepare ack不一致');
|
|
567
|
-
}
|
|
568
|
-
this.#leases.set(stagedLease.lease_digest, { lease: structuredClone(stagedLease),
|
|
569
|
-
controllerId, issuedAt: this.#clock(), revoked: false });
|
|
570
|
-
return structuredClone(ack);
|
|
571
|
-
}
|
|
572
|
-
|
|
573
|
-
async #acceptRebindAck({ controllerId, ack, stagedLease, expected }) {
|
|
574
|
-
const record = this.#requireController(controllerId);
|
|
575
|
-
if (!this.#frozen || !validateEpochRebindAck(ack) || !validateStagedWriteLease(stagedLease)
|
|
576
|
-
|| ack.run_id !== this.#runId || ack.supervisor_session_nonce_digest !== this.#sessionNonceDigest
|
|
577
|
-
|| stagedLease.run_id !== this.#runId || stagedLease.state !== 'staged'
|
|
578
|
-
|| stagedLease.controller_registration_digest !== record.registration.registration_digest
|
|
579
|
-
|| ack.new_write_lease_id !== stagedLease.lease_id
|
|
580
|
-
|| ack.todo_id !== expected.todo_id || ack.executor_handle !== expected.executor_handle
|
|
581
|
-
|| ack.worktree_id !== expected.worktree_id || ack.predecessor_packet_digest !== expected.predecessor_packet_digest
|
|
582
|
-
|| ack.rebind_packet_digest !== expected.rebind_packet_digest) fail('EPOCH_REBIND_INCOMPLETE', 'direct rebind ack不一致');
|
|
583
|
-
const observed = await this.#processObserver({ kind: 'rebind', ack: structuredClone(ack), stagedLease: structuredClone(stagedLease), descriptor: structuredClone(record.descriptor) });
|
|
584
|
-
if (observed?.write_enabled !== false) fail('EPOCH_REBIND_INCOMPLETE', 'rebind直接再観測不一致');
|
|
585
|
-
this.#leases.set(stagedLease.lease_digest, { lease: structuredClone(stagedLease), controllerId, issuedAt: this.#clock(), revoked: false });
|
|
586
|
-
return this.#append('epoch_rebind_acknowledged', { todo_id: ack.todo_id,
|
|
587
|
-
ack_digest: ack.ack_digest, staged_lease_digest: stagedLease.lease_digest });
|
|
588
|
-
}
|
|
589
|
-
|
|
590
|
-
/** 全release ackを回収した後だけ、中央gateを一回commitしてstaged v1→armed v2を有効化する。 */
|
|
591
|
-
async commitWriteGate({ planEpoch, committedEpochDigest, activationDigest,
|
|
592
|
-
releaseBarrierDigest = null, commitReleaseBarrier = null,
|
|
593
|
-
afterControllerRelease = null, committedAt }) {
|
|
594
|
-
await this.assertControllerHealth();
|
|
595
|
-
if (!this.#frozen || !Number.isSafeInteger(planEpoch) || planEpoch < 1
|
|
596
|
-
|| !(releaseBarrierDigest === null || digest(releaseBarrierDigest))
|
|
597
|
-
|| !digest(committedEpochDigest) || !digest(activationDigest)) fail('EPOCH_ACTIVATION_INCOMPLETE', 'gate入力不正');
|
|
598
|
-
const staged = [...this.#leases.values()].filter((entry) => !entry.revoked && entry.lease.plan_epoch === planEpoch && validateStagedWriteLease(entry.lease));
|
|
599
|
-
const nextGeneration = this.#gateGeneration + 1;
|
|
600
|
-
const readyAcks = [];
|
|
601
|
-
let releaseIndex = 0;
|
|
602
|
-
for (const record of this.#controllers.values()) {
|
|
603
|
-
const ownedStaged = staged.filter((entry) => entry.lease.controller_registration_digest === record.registration.registration_digest).map((entry) => entry.lease.lease_digest).sort();
|
|
604
|
-
const ready = await this.routeWhileFrozen('activate', record, {
|
|
605
|
-
request_id: randomUUID(), committed_epoch_digest: committedEpochDigest,
|
|
606
|
-
activation_digest: activationDigest, staged_lease_digests: ownedStaged,
|
|
607
|
-
});
|
|
608
|
-
if (ready.observed_pointer_digest !== committedEpochDigest
|
|
609
|
-
|| ready.ready_ack.registration_digest !== record.registration.registration_digest
|
|
610
|
-
|| ready.ready_ack.controller_id !== record.descriptor.controller_id
|
|
611
|
-
|| ready.ready_ack.run_id !== this.#runId || ready.ready_ack.plan_epoch !== planEpoch
|
|
612
|
-
|| ready.ready_ack.activation_digest !== activationDigest
|
|
613
|
-
|| ready.ready_ack.supervisor_session_nonce_digest !== this.#sessionNonceDigest
|
|
614
|
-
|| JSON.stringify(ready.ready_ack.staged_lease_digests) !== JSON.stringify(ownedStaged)) fail('EPOCH_ACTIVATION_INCOMPLETE', 'ready pointer/binding不一致');
|
|
615
|
-
readyAcks.push(ready.ready_ack);
|
|
616
|
-
}
|
|
617
|
-
if (typeof commitReleaseBarrier === 'function') {
|
|
618
|
-
const barrier = { schema: 'lattice.release_epoch_barrier.v1', run_id: this.#runId,
|
|
619
|
-
committed_epoch_pointer_digest: committedEpochDigest, plan_epoch: planEpoch,
|
|
620
|
-
activation_digest: activationDigest,
|
|
621
|
-
controller_ready_ack_digests: readyAcks.map((ack) => ack.ack_digest).sort(),
|
|
622
|
-
staged_lease_digests: staged.map((entry) => entry.lease.lease_digest).sort(),
|
|
623
|
-
gate_generation: nextGeneration, release_digest: '' };
|
|
624
|
-
barrier.release_digest = selfDigest(barrier, 'release_digest');
|
|
625
|
-
const receipt = await commitReleaseBarrier(structuredClone(barrier));
|
|
626
|
-
if (receipt?.release_digest !== barrier.release_digest) fail('EPOCH_ACTIVATION_INCOMPLETE', 'release barrier commit不一致');
|
|
627
|
-
releaseBarrierDigest = barrier.release_digest;
|
|
628
|
-
}
|
|
629
|
-
if (!digest(releaseBarrierDigest)) fail('EPOCH_ACTIVATION_INCOMPLETE', 'release barrier未commit');
|
|
630
|
-
const armed = staged.map((entry) => armStagedWriteLease(entry.lease, { releaseBarrierDigest, gateGeneration: nextGeneration }));
|
|
631
|
-
const releaseAcks = [];
|
|
632
|
-
for (const record of this.#controllers.values()) {
|
|
633
|
-
const ownedStaged = staged.filter((entry) => entry.lease.controller_registration_digest === record.registration.registration_digest).map((entry) => entry.lease.lease_digest).sort();
|
|
634
|
-
const released = await this.routeWhileFrozen('release', record, {
|
|
635
|
-
request_id: randomUUID(), release_barrier_digest: releaseBarrierDigest,
|
|
636
|
-
activation_digest: activationDigest, gate_generation: nextGeneration,
|
|
637
|
-
staged_lease_digests: ownedStaged,
|
|
638
|
-
});
|
|
639
|
-
const ownedArmed = armed.filter((lease) => lease.controller_registration_digest === record.registration.registration_digest).map((lease) => lease.lease_digest).sort();
|
|
640
|
-
if (released.observed_gate_generation !== nextGeneration
|
|
641
|
-
|| JSON.stringify(released.armed_lease_digests) !== JSON.stringify(ownedArmed)
|
|
642
|
-
|| !validateReleaseAck(released.release_ack)) fail('EPOCH_ACTIVATION_INCOMPLETE', 'direct release ack不一致');
|
|
643
|
-
releaseAcks.push(released.release_ack);
|
|
644
|
-
releaseIndex += 1;
|
|
645
|
-
if (typeof afterControllerRelease === 'function') await afterControllerRelease({
|
|
646
|
-
release_index: releaseIndex, controller_id: record.descriptor.controller_id,
|
|
647
|
-
release_ack_digest: released.release_ack.ack_digest,
|
|
648
|
-
});
|
|
649
|
-
}
|
|
650
|
-
const registrations = [...this.#controllers.values()].map((entry) => entry.registration);
|
|
651
|
-
const controllers = [...this.#controllers.values()].map((entry) => entry.descriptor);
|
|
652
|
-
const gate = createWriteGate({ runId: this.#runId, planEpoch, gateGeneration: nextGeneration, releaseBarrierDigest, releaseAcks, armedLeases: armed, previousGateDigest: this.#gate?.gate_digest ?? null, committedAt });
|
|
653
|
-
const verified = verifyCentralWriteGate({ gate, runId: this.#runId, planEpoch, releaseBarrierDigest, sessionNonceDigest: this.#sessionNonceDigest, registrations, controllers, releaseAcks, armedLeases: armed, previousGate: this.#gate });
|
|
654
|
-
if (!verified.valid) fail('EPOCH_ACTIVATION_INCOMPLETE', verified.reason);
|
|
655
|
-
const atomicActivation = {
|
|
656
|
-
gate: structuredClone(gate),
|
|
657
|
-
control_events: [
|
|
658
|
-
{ kind: 'write_gate_committed', payload: { gate_digest: gate.gate_digest, gate_generation: nextGeneration } },
|
|
659
|
-
{ kind: 'epoch_activated', payload: { plan_epoch: planEpoch, gate_digest: gate.gate_digest } },
|
|
660
|
-
{ kind: 'intake_resumed', payload: { plan_epoch: planEpoch, gate_digest: gate.gate_digest } },
|
|
661
|
-
],
|
|
662
|
-
};
|
|
663
|
-
const commitReceipt = await this.#gateWriter.commit(atomicActivation);
|
|
664
|
-
if (!digest(commitReceipt?.control_head_digest) || commitReceipt.gate_digest !== gate.gate_digest) fail('EPOCH_ACTIVATION_INCOMPLETE', 'gate/event atomic commit証拠不正');
|
|
665
|
-
// Durable central commit成功後にだけlocal projectionをarmedへ置換する。
|
|
666
|
-
for (const armedLease of armed) {
|
|
667
|
-
const old = staged.find((entry) => entry.lease.lease_id === armedLease.lease_id);
|
|
668
|
-
this.#leases.delete(old.lease.lease_digest);
|
|
669
|
-
this.#leases.set(armedLease.lease_digest, { ...old, lease: armedLease, issuedAt: this.#clock() });
|
|
670
|
-
}
|
|
671
|
-
this.#previousGate = this.#gate;
|
|
672
|
-
this.#gate = gate;
|
|
673
|
-
this.#releaseAcks = releaseAcks.map((ack) => structuredClone(ack));
|
|
674
|
-
this.#gateRecords.set(gate.gate_generation, {
|
|
675
|
-
gate: structuredClone(gate),
|
|
676
|
-
releaseAcks: releaseAcks.map((ack) => structuredClone(ack)),
|
|
677
|
-
});
|
|
678
|
-
this.#gateGeneration = nextGeneration;
|
|
679
|
-
this.#frozen = false;
|
|
680
|
-
return { gate: structuredClone(gate), armedLeases: armed.map((lease) => structuredClone(lease)) };
|
|
681
|
-
}
|
|
682
|
-
|
|
683
|
-
/** write/process開始の度にfull chainを再検証する。generationだけのshort pathは持たない。 */
|
|
684
|
-
async authorizeWrite({ leaseDigest }) {
|
|
685
|
-
await this.assertControllerHealth();
|
|
686
|
-
const entry = this.#leases.get(leaseDigest);
|
|
687
|
-
if (this.#frozen || !entry || entry.revoked || !validateArmedWriteLease(entry.lease)) fail('RUN_FROZEN', '有効なarmed leaseなし');
|
|
688
|
-
const gateRecord = this.#gateRecords.get(entry.lease.gate_generation);
|
|
689
|
-
const gate = gateRecord?.gate ?? (this.#gate?.gate_generation === entry.lease.gate_generation
|
|
690
|
-
? this.#gate : null);
|
|
691
|
-
const releaseAcks = gateRecord?.releaseAcks ?? (gate === this.#gate ? this.#releaseAcks : []);
|
|
692
|
-
if (gate === null) fail('RUN_FROZEN', 'leaseのorigin gateが無い');
|
|
693
|
-
const previousGate = this.#gateRecords.get(gate.gate_generation - 1)?.gate
|
|
694
|
-
?? (gate === this.#gate ? this.#previousGate : null);
|
|
695
|
-
const gateLeases = [...this.#leases.values()]
|
|
696
|
-
.filter((item) => !item.revoked && validateArmedWriteLease(item.lease)
|
|
697
|
-
&& item.lease.gate_generation === gate.gate_generation)
|
|
698
|
-
.map((item) => item.lease);
|
|
699
|
-
const verified = verifyCentralWriteGate({
|
|
700
|
-
gate, runId: this.#runId, planEpoch: entry.lease.plan_epoch,
|
|
701
|
-
releaseBarrierDigest: entry.lease.release_barrier_digest, sessionNonceDigest: this.#sessionNonceDigest,
|
|
702
|
-
registrations: [...this.#controllers.values()].map((record) => record.registration),
|
|
703
|
-
controllers: [...this.#controllers.values()].map((record) => record.descriptor),
|
|
704
|
-
releaseAcks, armedLeases: gateLeases,
|
|
705
|
-
previousGate,
|
|
706
|
-
});
|
|
707
|
-
// gate commit時にarmした後続frontierのleaseを壁時計で失効させると、正しく直列待ちした
|
|
708
|
-
// taskほどdispatch不能になる。freshnessはactive gate chainとrevokeで決める。
|
|
709
|
-
if (!verified.valid) {
|
|
710
|
-
entry.revoked = true;
|
|
711
|
-
this.#frozen = true;
|
|
712
|
-
fail('RUN_FROZEN', verified.reason);
|
|
713
|
-
}
|
|
714
|
-
return structuredClone(entry.lease);
|
|
715
|
-
}
|
|
716
|
-
|
|
717
|
-
/** restart後は旧nonce leaseを無効化し、観測集合の和を全件barrierする。 */
|
|
718
|
-
async recoveryBarrier({ barrierId, frozenEventDigest }) {
|
|
719
|
-
this.#frozen = true;
|
|
720
|
-
for (const entry of this.#leases.values()) entry.revoked = true;
|
|
721
|
-
await this.#append('supervisor_recovery_barrier', { barrier_id: barrierId });
|
|
722
|
-
return this.barrierAll({ barrierId, reason: 'supervisor_restart', frozenEventDigest });
|
|
723
|
-
}
|
|
724
|
-
|
|
725
|
-
async routeWhileFrozen(operation, record, fields) {
|
|
726
|
-
const request = createControllerRequest(operation, { request_id: fields.request_id ?? randomUUID(), registration_digest: record.registration.registration_digest, ...Object.fromEntries(Object.entries(fields).filter(([key]) => !['request_id', 'registration_digest'].includes(key))) });
|
|
727
|
-
const response = await record.transport.request(operation, structuredClone(request));
|
|
728
|
-
if (!validateControllerResponse(operation, response, request.request_id)) fail('ADAPTER_CONTROLLER_UNAVAILABLE', `${operation} response不正`);
|
|
729
|
-
return response;
|
|
730
|
-
}
|
|
731
|
-
|
|
732
|
-
#requireController(controllerId) {
|
|
733
|
-
const record = this.#controllers.get(controllerId);
|
|
734
|
-
if (!record) fail('ADAPTER_CONTROLLER_UNAVAILABLE', `unknown controller: ${controllerId}`);
|
|
735
|
-
return record;
|
|
736
|
-
}
|
|
737
|
-
|
|
738
|
-
async #failClosed(record, code, detail) {
|
|
739
|
-
this.#frozen = true;
|
|
740
|
-
record.revoked = true;
|
|
741
|
-
for (const entry of this.#leases.values()) {
|
|
742
|
-
if (entry.controllerId === record.descriptor.controller_id) entry.revoked = true;
|
|
743
|
-
}
|
|
744
|
-
const leaseDigests = [...this.#leases.values()].filter((entry) => entry.controllerId === record.descriptor.controller_id).map((entry) => entry.lease.lease_digest).sort();
|
|
745
|
-
if (record.connected) {
|
|
746
|
-
let response;
|
|
747
|
-
try { response = await this.routeWhileFrozen('revoke', record, { request_id: randomUUID(), reason: detail, lease_digests: leaseDigests }); }
|
|
748
|
-
catch { fail(code, `${detail}; controller revoke未確認`); }
|
|
749
|
-
if (JSON.stringify(response.revoked_lease_digests) !== JSON.stringify(leaseDigests)
|
|
750
|
-
|| response.residual_processes.length !== 0) fail(code, `${detail}; controller revoke不完全`);
|
|
751
|
-
await this.#append('lease_revoked', { controller_id: record.descriptor.controller_id, reason: detail, response_digest: response.response_digest });
|
|
752
|
-
}
|
|
753
|
-
fail(code, detail);
|
|
754
|
-
}
|
|
755
|
-
|
|
756
|
-
async #append(kind, payload) {
|
|
757
|
-
return this.#journal.append({ run_id: this.#runId, kind, session_nonce_digest: this.#sessionNonceDigest, payload });
|
|
758
|
-
}
|
|
759
|
-
}
|
|
760
|
-
|
|
761
|
-
async function exchangeControllerHandshake({ socketPath, runId, supervisorSessionNonce,
|
|
762
|
-
controllerSocketRef, timeoutMs }) {
|
|
763
|
-
const requestId = randomUUID();
|
|
764
|
-
const challenge = randomBytes(32).toString('hex');
|
|
765
|
-
const request = { schema: 'lattice.adapter_controller_handshake_request.v1', request_id: requestId,
|
|
766
|
-
run_id: runId, supervisor_session_nonce: supervisorSessionNonce,
|
|
767
|
-
controller_socket_ref: controllerSocketRef, challenge, request_digest: '' };
|
|
768
|
-
request.request_digest = selfDigest(request, 'request_digest');
|
|
769
|
-
return new Promise((resolve, reject) => {
|
|
770
|
-
const socket = net.createConnection({ path: socketPath });
|
|
771
|
-
let buffer = '';
|
|
772
|
-
const timer = setTimeout(() => { socket.destroy(); reject(new ManagedRuntimeError('ADAPTER_CONTROLLER_UNAVAILABLE', 'controller handshake timeout')); }, timeoutMs);
|
|
773
|
-
const done = (error, value) => { clearTimeout(timer); socket.destroy(); if (error) reject(error); else resolve(value); };
|
|
774
|
-
socket.setEncoding('utf8');
|
|
775
|
-
socket.once('connect', () => socket.write(`${canonicalizeArtifact(request)}\n`));
|
|
776
|
-
socket.on('data', (chunk) => {
|
|
777
|
-
buffer += chunk;
|
|
778
|
-
const newline = buffer.indexOf('\n');
|
|
779
|
-
if (newline < 0) return;
|
|
780
|
-
try {
|
|
781
|
-
if (buffer.slice(newline + 1).length !== 0) throw new Error('multiple documents');
|
|
782
|
-
const response = JSON.parse(buffer.slice(0, newline));
|
|
783
|
-
if (!validateControllerHandshakeResponse(response, { requestId, challenge, runId })) throw new Error('handshake binding不正');
|
|
784
|
-
done(null, response);
|
|
785
|
-
} catch (error) { done(new ManagedRuntimeError('ADAPTER_CONTROLLER_UNAVAILABLE', error.message)); }
|
|
786
|
-
});
|
|
787
|
-
socket.once('error', (error) => done(new ManagedRuntimeError('ADAPTER_CONTROLLER_UNAVAILABLE', error.message)));
|
|
788
|
-
});
|
|
789
|
-
}
|
|
790
|
-
|
|
791
|
-
function createControllerSocketTransport(
|
|
792
|
-
socketPath,
|
|
793
|
-
timeoutMs,
|
|
794
|
-
dispatchLivenessTimeoutMs = timeoutMs,
|
|
795
|
-
) {
|
|
796
|
-
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1
|
|
797
|
-
|| !Number.isSafeInteger(dispatchLivenessTimeoutMs) || dispatchLivenessTimeoutMs < 1) {
|
|
798
|
-
fail('SUPERVISOR_CONFIGURATION_INVALID', 'controller transport timeout不正');
|
|
799
|
-
}
|
|
800
|
-
const socket = net.createConnection({ path: socketPath });
|
|
801
|
-
const pending = new Map();
|
|
802
|
-
let buffer = '';
|
|
803
|
-
let connected = false;
|
|
804
|
-
let heartbeatHandler = null;
|
|
805
|
-
let disconnectHandler = null;
|
|
806
|
-
const ready = new Promise((resolve, reject) => {
|
|
807
|
-
socket.once('connect', () => { connected = true; resolve(); });
|
|
808
|
-
socket.once('error', reject);
|
|
809
|
-
});
|
|
810
|
-
socket.setEncoding('utf8');
|
|
811
|
-
const armRequestTimer = (requestId, entry) => {
|
|
812
|
-
clearTimeout(entry.timer);
|
|
813
|
-
const waitMs = entry.operation === 'dispatch' ? dispatchLivenessTimeoutMs : timeoutMs;
|
|
814
|
-
entry.timer = setTimeout(() => {
|
|
815
|
-
pending.delete(requestId);
|
|
816
|
-
entry.reject(new ManagedRuntimeError(
|
|
817
|
-
'ADAPTER_CONTROLLER_UNAVAILABLE',
|
|
818
|
-
`${entry.operation} timeout`,
|
|
819
|
-
));
|
|
820
|
-
}, waitMs);
|
|
821
|
-
};
|
|
822
|
-
const refreshDispatchLiveness = () => {
|
|
823
|
-
for (const [requestId, entry] of pending.entries()) {
|
|
824
|
-
if (entry.operation === 'dispatch') armRequestTimer(requestId, entry);
|
|
825
|
-
}
|
|
826
|
-
};
|
|
827
|
-
const failPending = (detail) => {
|
|
828
|
-
connected = false;
|
|
829
|
-
for (const entry of pending.values()) { clearTimeout(entry.timer); entry.reject(new ManagedRuntimeError('ADAPTER_CONTROLLER_UNAVAILABLE', detail)); }
|
|
830
|
-
pending.clear();
|
|
831
|
-
Promise.resolve(disconnectHandler?.()).catch(() => {});
|
|
832
|
-
};
|
|
833
|
-
socket.on('close', () => failPending('controller persistent socket disconnected'));
|
|
834
|
-
socket.on('error', (error) => failPending(error.message));
|
|
835
|
-
socket.on('data', (chunk) => {
|
|
836
|
-
buffer += chunk;
|
|
837
|
-
while (buffer.includes('\n')) {
|
|
838
|
-
const newline = buffer.indexOf('\n');
|
|
839
|
-
const line = buffer.slice(0, newline); buffer = buffer.slice(newline + 1);
|
|
840
|
-
let document;
|
|
841
|
-
try { document = JSON.parse(line); } catch { failPending('controller document JSON不正'); socket.destroy(); return; }
|
|
842
|
-
const errorEntry = pending.get(document?.request_id);
|
|
843
|
-
if (errorEntry && validateControllerError(document, document.request_id)) {
|
|
844
|
-
pending.delete(document.request_id);
|
|
845
|
-
clearTimeout(errorEntry.timer);
|
|
846
|
-
const detail = Object.keys(document.detail).length === 0
|
|
847
|
-
? ''
|
|
848
|
-
: `: ${canonicalizeArtifact(document.detail)}`;
|
|
849
|
-
errorEntry.reject(new ManagedRuntimeError(
|
|
850
|
-
'ADAPTER_CONTROLLER_UNAVAILABLE',
|
|
851
|
-
`${document.code}: ${document.message}${detail}`,
|
|
852
|
-
));
|
|
853
|
-
socket.destroy();
|
|
854
|
-
return;
|
|
855
|
-
}
|
|
856
|
-
if (validateControllerHeartbeat(document)) {
|
|
857
|
-
if (heartbeatHandler === null) {
|
|
858
|
-
socket.destroy();
|
|
859
|
-
continue;
|
|
860
|
-
}
|
|
861
|
-
Promise.resolve()
|
|
862
|
-
.then(() => heartbeatHandler(structuredClone(document)))
|
|
863
|
-
.then(() => refreshDispatchLiveness())
|
|
864
|
-
.catch(() => socket.destroy());
|
|
865
|
-
continue;
|
|
866
|
-
}
|
|
867
|
-
const entry = pending.get(document.request_id);
|
|
868
|
-
if (!entry || !validateControllerResponse(entry.operation, document, document.request_id)) { failPending('unsolicited/invalid controller response'); socket.destroy(); return; }
|
|
869
|
-
pending.delete(document.request_id); clearTimeout(entry.timer); entry.resolve(document);
|
|
870
|
-
}
|
|
871
|
-
});
|
|
872
|
-
return Object.freeze({
|
|
873
|
-
async ensureConnected() { await ready; if (!connected) fail('ADAPTER_CONTROLLER_UNAVAILABLE', 'persistent controller socket不達'); },
|
|
874
|
-
setHeartbeatHandler(handler) { heartbeatHandler = handler; },
|
|
875
|
-
setDisconnectHandler(handler) { disconnectHandler = handler; },
|
|
876
|
-
close() { disconnectHandler = null; socket.destroy(); },
|
|
877
|
-
async request(operation, request) {
|
|
878
|
-
await ready;
|
|
879
|
-
if (!connected) fail('ADAPTER_CONTROLLER_UNAVAILABLE', 'controller persistent socket不達');
|
|
880
|
-
return new Promise((resolve, reject) => {
|
|
881
|
-
const entry = { operation, resolve, reject, timer: null };
|
|
882
|
-
pending.set(request.request_id, entry);
|
|
883
|
-
armRequestTimer(request.request_id, entry);
|
|
884
|
-
socket.write(`${canonicalizeArtifact(request)}\n`);
|
|
885
|
-
});
|
|
886
|
-
},
|
|
887
|
-
});
|
|
888
|
-
}
|
|
889
|
-
|
|
890
|
-
export const runtimeManagedSupervisorInternal = Object.freeze({
|
|
891
|
-
createControllerSocketTransport,
|
|
892
|
-
});
|
|
893
|
-
|
|
894
|
-
/**
|
|
895
|
-
* durable registryをpreflightし、実controller hostとのnonce challenge後にだけactivation証拠を返す。
|
|
896
|
-
* callerは成功返却前にrun storeへ何もpublishしてはならない。
|
|
897
|
-
*/
|
|
898
|
-
async function activateManagedSupervisorController({ repoRoot, runDir, runId, adapterKind, supervisorSessionNonce, timeoutMs = 5_000, binaryIdentityObserver = observeMacosBinaryIdentity }) {
|
|
899
|
-
if (!path.isAbsolute(repoRoot) || !path.isAbsolute(runDir) || !identifier(runId) || !identifier(adapterKind)
|
|
900
|
-
|| typeof supervisorSessionNonce !== 'string' || supervisorSessionNonce.length < 32) fail('SUPERVISOR_CONFIGURATION_INVALID', 'activation入力不正');
|
|
901
|
-
const canonicalRepo = await realpath(repoRoot);
|
|
902
|
-
const registryPath = path.join(canonicalRepo, '.lattice/runtime/adapter-registry/registry.json');
|
|
903
|
-
let registryArtifact;
|
|
904
|
-
try { registryArtifact = await readCanonicalRegular(registryPath); } catch (error) {
|
|
905
|
-
if (error?.code === 'ENOENT') fail('ADAPTER_NOT_REGISTERED', adapterKind);
|
|
906
|
-
throw error;
|
|
907
|
-
}
|
|
908
|
-
if (!validateAdapterRegistry(registryArtifact.value)) fail('ADAPTER_LAUNCH_INVALID', 'registry不正');
|
|
909
|
-
const entry = registryArtifact.value.entries.find((candidate) => candidate.adapter_kind === adapterKind);
|
|
910
|
-
if (!entry) fail('ADAPTER_NOT_REGISTERED', adapterKind);
|
|
911
|
-
const descriptorPath = path.join(canonicalRepo, entry.launch_descriptor_ref);
|
|
912
|
-
if (!descriptorPath.startsWith(`${canonicalRepo}${path.sep}`)) fail('ADAPTER_LAUNCH_INVALID', 'descriptor repo外');
|
|
913
|
-
const launchArtifact = await readCanonicalRegular(descriptorPath);
|
|
914
|
-
const launch = launchArtifact.value;
|
|
915
|
-
if (!validateAdapterLaunchDescriptor(launch) || launch.adapter_kind !== adapterKind
|
|
916
|
-
|| launch.descriptor_digest !== entry.launch_descriptor_digest) fail('ADAPTER_LAUNCH_INVALID', 'launch descriptor binding不正');
|
|
917
|
-
const supervisorDir = path.join(runDir, 'supervisor');
|
|
918
|
-
const controllerDir = path.join(supervisorDir, 'controllers');
|
|
919
|
-
const controllerId = `${adapterKind}-${randomUUID()}`;
|
|
920
|
-
const controllerSocketRef = `supervisor/controllers/${controllerId}.sock`;
|
|
921
|
-
const controllerSocketPath = path.join(runDir, controllerSocketRef);
|
|
922
|
-
const supervisorSocketRef = 'supervisor/control.sock';
|
|
923
|
-
let child = null;
|
|
924
|
-
let childStderr = '';
|
|
925
|
-
try {
|
|
926
|
-
await mkdir(controllerDir, { recursive: true, mode: 0o700 });
|
|
927
|
-
let handshakeSocket = launch.endpoint;
|
|
928
|
-
let handshakeConnectPath = launch.endpoint;
|
|
929
|
-
let endpointOwnerPids = null;
|
|
930
|
-
if (launch.launch_kind === 'host_binary') {
|
|
931
|
-
const binaryInfo = await lstat(launch.binary_path);
|
|
932
|
-
if (!binaryInfo.isFile() || binaryInfo.isSymbolicLink() || (binaryInfo.mode & 0o111) === 0) fail('ADAPTER_LAUNCH_INVALID', 'binaryが実行可能regular fileでない');
|
|
933
|
-
const binaryReal = await realpath(launch.binary_path);
|
|
934
|
-
if (binaryReal !== launch.binary_path || sha256Bytes(await readFile(binaryReal)) !== launch.binary_digest) fail('ADAPTER_BINARY_IDENTITY_MISMATCH', 'binary pre-exec identity不一致');
|
|
935
|
-
if (launch.binary_identity !== null) {
|
|
936
|
-
if (typeof binaryIdentityObserver !== 'function'
|
|
937
|
-
|| canonicalizeArtifact(await binaryIdentityObserver(binaryReal)) !== canonicalizeArtifact(launch.binary_identity)) fail('ADAPTER_BINARY_IDENTITY_MISMATCH', 'macOS identity不一致');
|
|
938
|
-
}
|
|
939
|
-
const configPath = path.resolve(canonicalRepo, launch.config_ref);
|
|
940
|
-
if (!configPath.startsWith(`${canonicalRepo}${path.sep}`)) fail('ADAPTER_LAUNCH_INVALID', 'config repo外');
|
|
941
|
-
const config = await lstat(configPath);
|
|
942
|
-
if (!config.isFile() || config.isSymbolicLink() || sha256Bytes(await readFile(configPath)) !== launch.config_digest) fail('ADAPTER_LAUNCH_INVALID', 'config digest不一致');
|
|
943
|
-
const bootstrap = createControllerBootstrap({ requestId: randomUUID(), runId, controllerSocketRef, supervisorSocketRef, supervisorSessionNonce });
|
|
944
|
-
// controller hostのcwdはrun store。bootstrapの固定relative socket refを任意absolute pathへ拡張しない。
|
|
945
|
-
child = spawn(binaryReal, launch.argv, {
|
|
946
|
-
cwd: runDir,
|
|
947
|
-
detached: true,
|
|
948
|
-
stdio: ['ignore', 'ignore', 'pipe', 'pipe'],
|
|
949
|
-
});
|
|
950
|
-
child.stderr.setEncoding('utf8');
|
|
951
|
-
child.stderr.on('data', (chunk) => {
|
|
952
|
-
childStderr = `${childStderr}${chunk}`.slice(-8_192);
|
|
953
|
-
});
|
|
954
|
-
child.stdio[3].write(`${canonicalizeArtifact(bootstrap)}\n`);
|
|
955
|
-
child.stdio[3].end();
|
|
956
|
-
handshakeSocket = controllerSocketPath;
|
|
957
|
-
handshakeConnectPath = controllerSocketRef;
|
|
958
|
-
const deadline = Date.now() + timeoutMs;
|
|
959
|
-
while (Date.now() < deadline) {
|
|
960
|
-
try { if ((await lstat(handshakeSocket)).isSocket()) break; } catch (error) { if (error?.code !== 'ENOENT') throw error; }
|
|
961
|
-
if (child.exitCode !== null) {
|
|
962
|
-
fail(
|
|
963
|
-
'ADAPTER_CONTROLLER_UNAVAILABLE',
|
|
964
|
-
`controller exited: ${child.exitCode}${childStderr ? `: ${childStderr.trim()}` : ''}`,
|
|
965
|
-
);
|
|
966
|
-
}
|
|
967
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
968
|
-
}
|
|
969
|
-
let socketInfo;
|
|
970
|
-
try {
|
|
971
|
-
socketInfo = await lstat(handshakeSocket);
|
|
972
|
-
} catch {
|
|
973
|
-
fail(
|
|
974
|
-
'ADAPTER_CONTROLLER_UNAVAILABLE',
|
|
975
|
-
`controller socket未生成${childStderr ? `: ${childStderr.trim()}` : ''}`,
|
|
976
|
-
);
|
|
977
|
-
}
|
|
978
|
-
if (!socketInfo.isSocket()) fail('ADAPTER_CONTROLLER_UNAVAILABLE', 'controller endpointがsocketでない');
|
|
979
|
-
// exec後にも同じ実行image bytesを再検証する。PID生存も同時に要求する。
|
|
980
|
-
try { process.kill(child.pid, 0); } catch { fail('ADAPTER_CONTROLLER_UNAVAILABLE', 'controller process不達'); }
|
|
981
|
-
let executedImage;
|
|
982
|
-
try {
|
|
983
|
-
executedImage = await observeProcessExecutablePath(child.pid);
|
|
984
|
-
} catch { fail('ADAPTER_BINARY_IDENTITY_MISMATCH', 'exec後image path観測失敗'); }
|
|
985
|
-
if (executedImage !== binaryReal || sha256Bytes(await readFile(executedImage)) !== launch.binary_digest) {
|
|
986
|
-
fail('ADAPTER_BINARY_IDENTITY_MISMATCH', 'exec後image path/bytes不一致');
|
|
987
|
-
}
|
|
988
|
-
if (launch.binary_identity !== null
|
|
989
|
-
&& canonicalizeArtifact(await binaryIdentityObserver(executedImage)) !== canonicalizeArtifact(launch.binary_identity)) {
|
|
990
|
-
fail('ADAPTER_BINARY_IDENTITY_MISMATCH', 'exec後macOS identity不一致');
|
|
991
|
-
}
|
|
992
|
-
} else {
|
|
993
|
-
if (!path.isAbsolute(handshakeSocket) || await realpath(path.dirname(handshakeSocket)) !== path.dirname(handshakeSocket)) {
|
|
994
|
-
fail('ADAPTER_LAUNCH_INVALID', 'existing endpoint pathがcanonical absoluteでない');
|
|
995
|
-
}
|
|
996
|
-
const endpointInfo = await lstat(handshakeSocket);
|
|
997
|
-
if (!endpointInfo.isSocket() || endpointInfo.isSymbolicLink()) fail('ADAPTER_LAUNCH_INVALID', 'existing endpointがsocketでない');
|
|
998
|
-
let ownerPids;
|
|
999
|
-
try { ownerPids = await pidsOwningSocketPath(handshakeSocket); }
|
|
1000
|
-
catch { fail('ADAPTER_LAUNCH_INVALID', 'existing endpoint ownerを観測できない'); }
|
|
1001
|
-
endpointOwnerPids = new Set(ownerPids);
|
|
1002
|
-
if (endpointOwnerPids.size !== 1 || [...endpointOwnerPids].some((pid) => !Number.isSafeInteger(pid) || pid < 1)) {
|
|
1003
|
-
fail('ADAPTER_LAUNCH_INVALID', 'existing endpoint ownerが一意でない');
|
|
1004
|
-
}
|
|
1005
|
-
}
|
|
1006
|
-
const handshake = await exchangeControllerHandshake({ socketPath: handshakeConnectPath, runId,
|
|
1007
|
-
supervisorSessionNonce, controllerSocketRef, timeoutMs });
|
|
1008
|
-
const controllerDescriptor = handshake.descriptor;
|
|
1009
|
-
if (controllerDescriptor.adapter_kind !== adapterKind || controllerDescriptor.socket_ref !== controllerSocketRef
|
|
1010
|
-
|| controllerDescriptor.capabilities.capabilities_digest !== launch.capabilities_digest
|
|
1011
|
-
|| (child !== null && controllerDescriptor.pid !== child.pid)) fail('ADAPTER_CONTROLLER_UNAVAILABLE', 'controller descriptor launch binding不一致');
|
|
1012
|
-
if (endpointOwnerPids !== null) {
|
|
1013
|
-
if (!endpointOwnerPids.has(controllerDescriptor.pid)) {
|
|
1014
|
-
fail('ADAPTER_CONTROLLER_UNAVAILABLE', 'existing endpoint inode ownerとdescriptor PID不一致');
|
|
1015
|
-
}
|
|
1016
|
-
const deadline = Date.now() + timeoutMs;
|
|
1017
|
-
while (Date.now() < deadline) {
|
|
1018
|
-
try { if ((await lstat(controllerSocketPath)).isSocket()) break; } catch (error) { if (error?.code !== 'ENOENT') throw error; }
|
|
1019
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
1020
|
-
}
|
|
1021
|
-
let controllerOwnerPids;
|
|
1022
|
-
try { controllerOwnerPids = await pidsOwningSocketPath(controllerSocketPath); }
|
|
1023
|
-
catch { fail('ADAPTER_CONTROLLER_UNAVAILABLE', 'existing controller socket ownerを観測できない'); }
|
|
1024
|
-
const persistentOwners = new Set(controllerOwnerPids);
|
|
1025
|
-
if (persistentOwners.size !== 1 || !persistentOwners.has(controllerDescriptor.pid)) {
|
|
1026
|
-
fail('ADAPTER_CONTROLLER_UNAVAILABLE', 'persistent controller socket ownerとdescriptor PID不一致');
|
|
1027
|
-
}
|
|
1028
|
-
handshakeConnectPath = controllerSocketRef;
|
|
1029
|
-
}
|
|
1030
|
-
const observedControllerIdentity = await observeManagedProcessStartIdentity(controllerDescriptor.pid);
|
|
1031
|
-
if (canonicalizeArtifact(observedControllerIdentity) !== canonicalizeArtifact(controllerDescriptor.process_start_identity)) fail('ADAPTER_CONTROLLER_UNAVAILABLE', `controller process start identity不一致: expected=${controllerDescriptor.process_start_identity.identity_digest} observed=${observedControllerIdentity.identity_digest}`);
|
|
1032
|
-
const sessionNonceDigest = digestArtifact(supervisorSessionNonce);
|
|
1033
|
-
const registration = {
|
|
1034
|
-
schema: 'lattice.runtime_adapter_registration.v1', registration_id: `registration-${controllerId}`,
|
|
1035
|
-
run_id: runId, supervisor_session_nonce_digest: sessionNonceDigest,
|
|
1036
|
-
controller_descriptor_digest: controllerDescriptor.descriptor_digest,
|
|
1037
|
-
registered_operations: [...controllerDescriptor.capabilities.operations],
|
|
1038
|
-
registered_at: new Date().toISOString(), registration_digest: '',
|
|
1039
|
-
};
|
|
1040
|
-
registration.registration_digest = selfDigest(registration, 'registration_digest');
|
|
1041
|
-
if (!validateControllerRegistration(registration)) fail('ADAPTER_CONTROLLER_UNAVAILABLE', 'registration生成不正');
|
|
1042
|
-
const processIdentity = await observeManagedProcessStartIdentity(process.pid);
|
|
1043
|
-
const supervisorDescriptor = { schema: 'lattice.runtime_supervisor_descriptor.v1', run_id: runId, pid: process.pid, process_start_identity: processIdentity, socket_ref: supervisorSocketRef, session_nonce_digest: sessionNonceDigest, protocol_version: 'v1', activated_at: new Date().toISOString(), descriptor_digest: '' };
|
|
1044
|
-
supervisorDescriptor.descriptor_digest = selfDigest(supervisorDescriptor, 'descriptor_digest');
|
|
1045
|
-
const activationControlEvent = { schema: 'lattice.runtime_control_event.v1', run_id: runId, sequence: 1, previous_digest: null, kind: 'supervisor_activated', session_nonce_digest: sessionNonceDigest, payload: { supervisor_descriptor_digest: supervisorDescriptor.descriptor_digest, controller_descriptor_digest: controllerDescriptor.descriptor_digest, registration_digest: registration.registration_digest }, recorded_at: supervisorDescriptor.activated_at, event_digest: '' };
|
|
1046
|
-
activationControlEvent.event_digest = selfDigest(activationControlEvent, 'event_digest');
|
|
1047
|
-
const controllerTransport = createControllerSocketTransport(
|
|
1048
|
-
handshakeConnectPath,
|
|
1049
|
-
timeoutMs,
|
|
1050
|
-
controllerDescriptor.heartbeat.ttl_ms,
|
|
1051
|
-
);
|
|
1052
|
-
let disposed = false;
|
|
1053
|
-
const disposeController = async () => {
|
|
1054
|
-
if (disposed) return;
|
|
1055
|
-
disposed = true;
|
|
1056
|
-
controllerTransport.close();
|
|
1057
|
-
if (child?.pid) {
|
|
1058
|
-
try { process.kill(-child.pid, 'SIGTERM'); } catch { /* already exited */ }
|
|
1059
|
-
const deadline = Date.now() + 2_000;
|
|
1060
|
-
while (Date.now() < deadline) {
|
|
1061
|
-
try { process.kill(child.pid, 0); } catch { break; }
|
|
1062
|
-
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
1063
|
-
}
|
|
1064
|
-
try { process.kill(child.pid, 0); process.kill(-child.pid, 'SIGKILL'); } catch { /* reaped */ }
|
|
1065
|
-
}
|
|
1066
|
-
await rm(controllerSocketPath, { force: true }).catch(() => {});
|
|
1067
|
-
};
|
|
1068
|
-
const registerWithManagedSupervisor = async (managedSupervisor) => {
|
|
1069
|
-
await controllerTransport.ensureConnected();
|
|
1070
|
-
await managedSupervisor.registerController({ descriptor: controllerDescriptor, registration, transport: controllerTransport });
|
|
1071
|
-
controllerTransport.setHeartbeatHandler(async (heartbeat) => {
|
|
1072
|
-
if (digestArtifact(heartbeat.supervisor_session_nonce) !== sessionNonceDigest
|
|
1073
|
-
|| heartbeat.controller_id !== controllerDescriptor.controller_id) {
|
|
1074
|
-
await managedSupervisor.disconnect(controllerDescriptor.controller_id);
|
|
1075
|
-
return;
|
|
1076
|
-
}
|
|
1077
|
-
await managedSupervisor.heartbeat({
|
|
1078
|
-
controllerId: heartbeat.controller_id,
|
|
1079
|
-
registrationDigest: heartbeat.registration_digest,
|
|
1080
|
-
sequence: heartbeat.sequence,
|
|
1081
|
-
leaseSetDigest: heartbeat.lease_set_digest,
|
|
1082
|
-
sessionNonceDigest,
|
|
1083
|
-
});
|
|
1084
|
-
});
|
|
1085
|
-
controllerTransport.setDisconnectHandler(() => managedSupervisor.disconnect(controllerDescriptor.controller_id));
|
|
1086
|
-
};
|
|
1087
|
-
const createManagedSupervisor = async ({ clock, resolveObservationBinding, resolveRunningBindings, processObserver = null, allowTestObserver = false, journal, gateWriter }) => {
|
|
1088
|
-
if (processObserver !== null && allowTestObserver !== true) fail('SUPERVISOR_CONFIGURATION_INVALID', 'production observer差替え禁止');
|
|
1089
|
-
const directObserver = processObserver ?? createDirectOsProcessObserverV2({ resolveObservationBinding });
|
|
1090
|
-
const managedSupervisor = new RuntimeManagedSupervisor({ runId, sessionNonceDigest, clock, processObserver: directObserver, runningBindingResolver: resolveRunningBindings, journal, gateWriter, shutdownFinalizer: disposeController });
|
|
1091
|
-
await managedSupervisor.hydrateCommittedGate();
|
|
1092
|
-
await registerWithManagedSupervisor(managedSupervisor);
|
|
1093
|
-
return managedSupervisor;
|
|
1094
|
-
};
|
|
1095
|
-
return {
|
|
1096
|
-
supervisorDescriptor,
|
|
1097
|
-
activationControlEvent,
|
|
1098
|
-
controllerDescriptor,
|
|
1099
|
-
registration,
|
|
1100
|
-
launchDescriptor: structuredClone(launch),
|
|
1101
|
-
sessionNonce: supervisorSessionNonce,
|
|
1102
|
-
childPid: child?.pid ?? controllerDescriptor.pid,
|
|
1103
|
-
createManagedSupervisor,
|
|
1104
|
-
registerWithManagedSupervisor,
|
|
1105
|
-
disposeController,
|
|
1106
|
-
};
|
|
1107
|
-
} catch (error) {
|
|
1108
|
-
if (child?.pid) { try { process.kill(child.pid, 'SIGTERM'); } catch { /* already exited */ } }
|
|
1109
|
-
await rm(controllerSocketPath, { force: true }).catch(() => {});
|
|
1110
|
-
throw error;
|
|
1111
|
-
}
|
|
1112
|
-
}
|
|
1113
|
-
|
|
1114
|
-
/** CLI→durable supervisorのUnix socket client。nonceはrequest内だけ、stdoutへ返さない。 */
|
|
1115
|
-
export async function sendRuntimeControlRequest({ socketPath, request, timeoutMs = 5_000 }) {
|
|
1116
|
-
if (typeof socketPath !== 'string' || socketPath.length === 0 || !validateRuntimeControlRequest(request)
|
|
1117
|
-
|| !Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 60_000) fail('RUN_NOT_MANAGED', 'control request不正');
|
|
1118
|
-
const absoluteSocket = path.resolve(socketPath);
|
|
1119
|
-
const runDir = path.dirname(path.dirname(absoluteSocket));
|
|
1120
|
-
const socketRef = path.relative(runDir, absoluteSocket);
|
|
1121
|
-
const runInfo = await lstat(runDir).catch(() => null);
|
|
1122
|
-
const supervisorInfo = await lstat(path.join(runDir, 'supervisor')).catch(() => null);
|
|
1123
|
-
const socketInfo = await lstat(absoluteSocket).catch(() => null);
|
|
1124
|
-
if (socketRef !== 'supervisor/control.sock' || runInfo === null || !runInfo.isDirectory() || runInfo.isSymbolicLink()
|
|
1125
|
-
|| supervisorInfo === null || !supervisorInfo.isDirectory() || supervisorInfo.isSymbolicLink()
|
|
1126
|
-
|| socketInfo === null || !socketInfo.isSocket() || socketInfo.isSymbolicLink()
|
|
1127
|
-
|| await realpath(runDir) !== runDir) fail('RUN_NOT_MANAGED', 'control socket cwd anchor不正');
|
|
1128
|
-
const activePaths = await resolveActiveRuntimePaths({ runDir });
|
|
1129
|
-
const descriptorArtifact = await readCanonicalRegular(activePaths.descriptorPath).catch(() => null);
|
|
1130
|
-
const descriptor = descriptorArtifact?.value;
|
|
1131
|
-
const descriptorFields = ['schema', 'run_id', 'pid', 'process_start_identity', 'socket_ref', 'session_nonce_digest', 'protocol_version', 'activated_at', 'descriptor_digest'];
|
|
1132
|
-
if (descriptor === undefined || descriptor.schema !== 'lattice.runtime_supervisor_descriptor.v1'
|
|
1133
|
-
|| Object.keys(descriptor).sort().join('\0') !== descriptorFields.sort().join('\0')
|
|
1134
|
-
|| descriptor.run_id !== request.run_id || descriptor.socket_ref !== socketRef
|
|
1135
|
-
|| descriptor.session_nonce_digest !== digestArtifact(request.session_nonce)
|
|
1136
|
-
|| (activePaths.pointer !== null
|
|
1137
|
-
&& (descriptor.descriptor_digest !== activePaths.pointer.descriptor_digest
|
|
1138
|
-
|| descriptor.session_nonce_digest !== activePaths.pointer.session_nonce_digest))
|
|
1139
|
-
|| descriptor.descriptor_digest !== selfDigest(descriptor, 'descriptor_digest')) fail('RUN_NOT_MANAGED', 'supervisor descriptor/session binding不正');
|
|
1140
|
-
let observedSupervisor;
|
|
1141
|
-
try { observedSupervisor = await observeManagedProcessStartIdentity(descriptor.pid); }
|
|
1142
|
-
catch { fail('RUN_NOT_MANAGED', 'supervisor process不在'); }
|
|
1143
|
-
if (canonicalizeArtifact(observedSupervisor) !== canonicalizeArtifact(descriptor.process_start_identity)) fail('RUN_NOT_MANAGED', 'supervisor PID/start identity不一致');
|
|
1144
|
-
let ownedSocketPaths;
|
|
1145
|
-
try { ownedSocketPaths = await socketPathsOwnedByPid(descriptor.pid); }
|
|
1146
|
-
catch { fail('RUN_NOT_MANAGED', 'supervisor socket owner観測失敗'); }
|
|
1147
|
-
const ownsExactSocket = ownedSocketPaths
|
|
1148
|
-
.some((name) => name === absoluteSocket || name === socketRef);
|
|
1149
|
-
if (!ownsExactSocket) fail('RUN_NOT_MANAGED', 'supervisor PIDがcontrol socket inodeを所有しない');
|
|
1150
|
-
return new Promise((resolve, reject) => {
|
|
1151
|
-
let originalCwd;
|
|
1152
|
-
let socket;
|
|
1153
|
-
try {
|
|
1154
|
-
originalCwd = process.cwd();
|
|
1155
|
-
process.chdir(runDir);
|
|
1156
|
-
if (process.cwd() !== runDir) throw new Error('run cwd identity不一致');
|
|
1157
|
-
socket = net.createConnection({ path: socketRef });
|
|
1158
|
-
} catch (error) {
|
|
1159
|
-
if (originalCwd) process.chdir(originalCwd);
|
|
1160
|
-
return reject(new ManagedRuntimeError('RUN_NOT_MANAGED', error.message));
|
|
1161
|
-
}
|
|
1162
|
-
let buffer = '';
|
|
1163
|
-
let settled = false;
|
|
1164
|
-
let sent = false;
|
|
1165
|
-
let cwdRestored = false;
|
|
1166
|
-
const restoreCwd = () => { if (!cwdRestored) { cwdRestored = true; process.chdir(originalCwd); } };
|
|
1167
|
-
const finish = (error, value) => {
|
|
1168
|
-
if (settled) return;
|
|
1169
|
-
settled = true;
|
|
1170
|
-
restoreCwd();
|
|
1171
|
-
socket.destroy();
|
|
1172
|
-
if (error) reject(error); else resolve(value);
|
|
1173
|
-
};
|
|
1174
|
-
socket.setEncoding('utf8');
|
|
1175
|
-
socket.setTimeout(timeoutMs, () => finish(new ManagedRuntimeError('RUN_OUTCOME_UNKNOWN',
|
|
1176
|
-
'control timeout。同一request_idで再照会が必要')));
|
|
1177
|
-
socket.on('connect', () => { restoreCwd(); sent = true; socket.write(`${canonicalizeArtifact(request)}\n`); });
|
|
1178
|
-
socket.on('data', (chunk) => {
|
|
1179
|
-
buffer += chunk;
|
|
1180
|
-
if (Buffer.byteLength(buffer, 'utf8') > 1_048_576) return finish(new ManagedRuntimeError('RUN_NOT_MANAGED', 'control response過大'));
|
|
1181
|
-
const newline = buffer.indexOf('\n');
|
|
1182
|
-
if (newline < 0) return;
|
|
1183
|
-
if (buffer.slice(newline + 1).length !== 0) return finish(new ManagedRuntimeError('RUN_NOT_MANAGED', 'control response複数document'));
|
|
1184
|
-
try {
|
|
1185
|
-
const value = JSON.parse(buffer.slice(0, newline));
|
|
1186
|
-
if (!validateRuntimeControlResponse(value, request.operation) || value.request_id !== request.request_id || value.run_id !== request.run_id) return finish(new ManagedRuntimeError('RUN_NOT_MANAGED', 'control response binding不正'));
|
|
1187
|
-
finish(null, value);
|
|
1188
|
-
} catch (error) { finish(error instanceof ManagedRuntimeError ? error : new ManagedRuntimeError('RUN_NOT_MANAGED', 'control response JSON不正')); }
|
|
1189
|
-
});
|
|
1190
|
-
socket.on('error', (error) => finish(new ManagedRuntimeError(sent ? 'RUN_OUTCOME_UNKNOWN' : 'RUN_NOT_MANAGED', error.message)));
|
|
1191
|
-
socket.on('end', () => { if (!settled) finish(new ManagedRuntimeError(sent ? 'RUN_OUTCOME_UNKNOWN' : 'RUN_NOT_MANAGED', 'control socket早期切断')); });
|
|
1192
|
-
});
|
|
1193
|
-
}
|
|
1194
|
-
|
|
1195
|
-
/** 初回activate専用。descriptor未公開のためIPCで得たPID+nonceとsocket ownerを直接bindする。 */
|
|
1196
|
-
export async function sendRuntimeActivationRequest({ socketPath, request, expectedPid, expectedProcessStartIdentity, timeoutMs = 5_000 }) {
|
|
1197
|
-
if (!validateRuntimeControlRequest(request) || request.operation !== 'activate'
|
|
1198
|
-
|| !Number.isSafeInteger(expectedPid) || expectedPid < 1
|
|
1199
|
-
|| !validateProcessStartIdentity(expectedProcessStartIdentity)
|
|
1200
|
-
|| expectedProcessStartIdentity.pid !== expectedPid) fail('RUN_NOT_MANAGED', 'activation bootstrap入力不正');
|
|
1201
|
-
const absoluteSocket = path.resolve(socketPath);
|
|
1202
|
-
const runDir = path.dirname(path.dirname(absoluteSocket));
|
|
1203
|
-
const socketRef = path.relative(runDir, absoluteSocket);
|
|
1204
|
-
const socketInfo = await lstat(absoluteSocket).catch(() => null);
|
|
1205
|
-
if (socketRef !== 'supervisor/control.sock' || socketInfo === null || !socketInfo.isSocket() || socketInfo.isSymbolicLink()
|
|
1206
|
-
|| await realpath(runDir) !== runDir) fail('RUN_NOT_MANAGED', 'activation socket anchor不正');
|
|
1207
|
-
let observedBootstrapIdentity;
|
|
1208
|
-
try { observedBootstrapIdentity = await observeManagedProcessStartIdentity(expectedPid); }
|
|
1209
|
-
catch { fail('RUN_NOT_MANAGED', 'activation bootstrap process不在'); }
|
|
1210
|
-
if (canonicalizeArtifact(observedBootstrapIdentity) !== canonicalizeArtifact(expectedProcessStartIdentity)) fail('RUN_NOT_MANAGED', 'activation bootstrap PID/start identity不一致');
|
|
1211
|
-
let bootstrapSocketPaths;
|
|
1212
|
-
try { bootstrapSocketPaths = await socketPathsOwnedByPid(expectedPid); }
|
|
1213
|
-
catch { fail('RUN_NOT_MANAGED', 'activation socket owner観測失敗'); }
|
|
1214
|
-
if (!bootstrapSocketPaths
|
|
1215
|
-
.some((name) => name === absoluteSocket || name === socketRef)) fail('RUN_NOT_MANAGED', 'bootstrap PIDがcontrol socketを所有しない');
|
|
1216
|
-
const originalCwd = process.cwd();
|
|
1217
|
-
process.chdir(runDir);
|
|
1218
|
-
let socket;
|
|
1219
|
-
try { socket = net.createConnection({ path: socketRef }); } catch (error) { process.chdir(originalCwd); fail('RUN_NOT_MANAGED', error.message); }
|
|
1220
|
-
return new Promise((resolve, reject) => {
|
|
1221
|
-
let restored = false; const restore = () => { if (!restored) { restored = true; process.chdir(originalCwd); } };
|
|
1222
|
-
let buffer = ''; let settled = false; let sent = false;
|
|
1223
|
-
const finish = (error, value) => { if (settled) return; settled = true; restore(); socket.destroy(); if (error) reject(error); else resolve(value); };
|
|
1224
|
-
socket.setEncoding('utf8');
|
|
1225
|
-
socket.setTimeout(timeoutMs, () => finish(new ManagedRuntimeError('RUN_OUTCOME_UNKNOWN',
|
|
1226
|
-
'activation timeout。同一request_idで再照会が必要')));
|
|
1227
|
-
socket.on('connect', () => { restore(); sent = true; socket.write(`${canonicalizeArtifact(request)}\n`); });
|
|
1228
|
-
socket.on('data', (chunk) => {
|
|
1229
|
-
buffer += chunk; const newline = buffer.indexOf('\n'); if (newline < 0) return;
|
|
1230
|
-
try {
|
|
1231
|
-
const response = JSON.parse(buffer.slice(0, newline));
|
|
1232
|
-
if (buffer.slice(newline + 1).length !== 0 || !validateRuntimeControlResponse(response, request.operation)
|
|
1233
|
-
|| response.request_id !== request.request_id || response.run_id !== request.run_id) throw new Error('activation response binding不正');
|
|
1234
|
-
finish(null, response);
|
|
1235
|
-
} catch (error) { finish(new ManagedRuntimeError('RUN_NOT_MANAGED', error.message)); }
|
|
1236
|
-
});
|
|
1237
|
-
socket.on('error', (error) => finish(new ManagedRuntimeError(sent ? 'RUN_OUTCOME_UNKNOWN' : 'RUN_NOT_MANAGED', error.message)));
|
|
1238
|
-
socket.on('end', () => { if (!settled) finish(new ManagedRuntimeError(sent ? 'RUN_OUTCOME_UNKNOWN' : 'RUN_NOT_MANAGED', 'activation socket早期切断')); });
|
|
1239
|
-
});
|
|
1240
|
-
}
|
|
1241
|
-
|
|
1242
|
-
/** 外部workerの完了を含む初回activationは、同じbootstrap接続で最終結果まで待つ。 */
|
|
1243
|
-
export function sendRuntimeActivationRequestUntilSettled(args) {
|
|
1244
|
-
return sendRuntimeActivationRequest({ ...args, timeoutMs: 0 });
|
|
1245
|
-
}
|
|
1246
|
-
|
|
1247
|
-
/** stopped/crashed supervisorの同一run明示再起動前処理。live/foreign socketは削除しない。 */
|
|
1248
|
-
export async function prepareManagedSupervisorRestart({ runDir }) {
|
|
1249
|
-
if (!path.isAbsolute(runDir) || await realpath(runDir) !== runDir) fail('RUN_NOT_MANAGED', 'restart runDir不正');
|
|
1250
|
-
const activePaths = await resolveActiveRuntimePaths({ runDir });
|
|
1251
|
-
const descriptorArtifact = await readCanonicalRegular(activePaths.descriptorPath).catch(() => null);
|
|
1252
|
-
const descriptor = descriptorArtifact?.value;
|
|
1253
|
-
if (descriptor?.schema !== 'lattice.runtime_supervisor_descriptor.v1'
|
|
1254
|
-
|| descriptor.descriptor_digest !== selfDigest(descriptor, 'descriptor_digest')
|
|
1255
|
-
|| !validateProcessStartIdentity(descriptor.process_start_identity)
|
|
1256
|
-
|| descriptor.process_start_identity.pid !== descriptor.pid
|
|
1257
|
-
|| (activePaths.pointer !== null
|
|
1258
|
-
&& (descriptor.descriptor_digest !== activePaths.pointer.descriptor_digest
|
|
1259
|
-
|| descriptor.session_nonce_digest !== activePaths.pointer.session_nonce_digest))) fail('RUN_NOT_MANAGED', 'restart descriptor不正');
|
|
1260
|
-
try {
|
|
1261
|
-
const observed = await observeManagedProcessStartIdentity(descriptor.pid);
|
|
1262
|
-
if (canonicalizeArtifact(observed) === canonicalizeArtifact(descriptor.process_start_identity)) fail('RUN_BUSY', 'supervisorは既にlive');
|
|
1263
|
-
} catch (error) {
|
|
1264
|
-
if (error instanceof ManagedRuntimeError && error.code === 'RUN_BUSY') throw error;
|
|
1265
|
-
// process absentだけを許容。PID再利用は旧supervisorではない。
|
|
1266
|
-
}
|
|
1267
|
-
const socketPath = path.join(runDir, descriptor.socket_ref);
|
|
1268
|
-
const socketInfo = await lstat(socketPath).catch(() => null);
|
|
1269
|
-
if (socketInfo !== null) {
|
|
1270
|
-
if (!socketInfo.isSocket() || socketInfo.isSymbolicLink()) fail('RUN_NOT_MANAGED', 'stale socket path差替え');
|
|
1271
|
-
// 所有者が居るなら消さない。観測できなければ「誰も居ない」へ丸めず観測失敗として止める
|
|
1272
|
-
// (lsofはヒット無しでexit 1を返すので、この2つを区別する必要がある)。
|
|
1273
|
-
let staleOwners;
|
|
1274
|
-
try { staleOwners = await pidsOwningSocketPath(socketPath); }
|
|
1275
|
-
catch (error) {
|
|
1276
|
-
if (error instanceof ManagedRuntimeError) throw error;
|
|
1277
|
-
fail('RUN_NOT_MANAGED', 'stale socket owner観測失敗');
|
|
1278
|
-
}
|
|
1279
|
-
if (staleOwners.length > 0) fail('RUN_BUSY', 'control socketは別processが所有中');
|
|
1280
|
-
await rm(socketPath, { force: true });
|
|
1281
|
-
}
|
|
1282
|
-
return { previousDescriptorDigest: descriptor.descriptor_digest };
|
|
1283
|
-
}
|
|
1284
|
-
|
|
1285
|
-
/** restart commit pointerがあればcandidate namespaceを正本として解決する。固定pathは初回互換cache。 */
|
|
1286
|
-
export async function resolveActiveRuntimePaths({ runDir }) {
|
|
1287
|
-
const fixed = {
|
|
1288
|
-
descriptorPath: path.join(runDir, 'supervisor/descriptor.json'),
|
|
1289
|
-
sessionPath: path.join(runDir, 'supervisor/session'),
|
|
1290
|
-
controlEventsPath: path.join(runDir, 'control-events.json'),
|
|
1291
|
-
pointer: null,
|
|
1292
|
-
};
|
|
1293
|
-
const artifact = await readCanonicalRegular(path.join(runDir, 'supervisor/active-runtime.json')).catch((error) => {
|
|
1294
|
-
if (error?.code === 'ENOENT') return null;
|
|
1295
|
-
throw error;
|
|
1296
|
-
});
|
|
1297
|
-
if (artifact === null) return fixed;
|
|
1298
|
-
const pointer = artifact.value;
|
|
1299
|
-
const fields = ['schema', 'run_id', 'candidate_ref', 'activation_request_id',
|
|
1300
|
-
'activation_request_digest', 'activation_intent_digest', 'activation_response_digest', 'descriptor_digest',
|
|
1301
|
-
'session_nonce_digest', 'control_head_digest', 'control_head_sequence',
|
|
1302
|
-
'committed_at', 'pointer_digest'];
|
|
1303
|
-
if (pointer?.schema !== 'lattice.runtime_active_pointer.v1'
|
|
1304
|
-
|| Object.keys(pointer).sort().join('\0') !== fields.sort().join('\0')
|
|
1305
|
-
|| !identifier(pointer.run_id) || !digest(pointer.descriptor_digest)
|
|
1306
|
-
|| !identifier(pointer.activation_request_id) || !digest(pointer.activation_request_digest)
|
|
1307
|
-
|| !digest(pointer.activation_intent_digest)
|
|
1308
|
-
|| !digest(pointer.activation_response_digest)
|
|
1309
|
-
|| !Number.isSafeInteger(pointer.control_head_sequence) || pointer.control_head_sequence < 1
|
|
1310
|
-
|| !digest(pointer.session_nonce_digest) || !digest(pointer.control_head_digest)
|
|
1311
|
-
|| typeof pointer.committed_at !== 'string' || Number.isNaN(Date.parse(pointer.committed_at))
|
|
1312
|
-
|| pointer.pointer_digest !== selfDigest(pointer, 'pointer_digest')
|
|
1313
|
-
|| !/^supervisor\/restart-candidates\/[0-9a-f]{64}$/u.test(pointer.candidate_ref)) {
|
|
1314
|
-
fail('RUN_NOT_MANAGED', 'active runtime pointer不正');
|
|
1315
|
-
}
|
|
1316
|
-
const candidateDir = path.join(runDir, pointer.candidate_ref);
|
|
1317
|
-
if (!candidateDir.startsWith(`${path.join(runDir, 'supervisor/restart-candidates')}${path.sep}`)) {
|
|
1318
|
-
fail('RUN_NOT_MANAGED', 'active runtime pointerがcandidate外');
|
|
1319
|
-
}
|
|
1320
|
-
const descriptorPath = path.join(candidateDir, 'descriptor.json');
|
|
1321
|
-
const sessionPath = path.join(candidateDir, 'session');
|
|
1322
|
-
const controlEventsPath = path.join(candidateDir, 'control-events.json');
|
|
1323
|
-
const descriptor = (await readCanonicalRegular(descriptorPath)).value;
|
|
1324
|
-
const sessionNonce = (await readFile(sessionPath, 'utf8')).trim();
|
|
1325
|
-
const controlEvents = (await readCanonicalRegular(controlEventsPath)).value;
|
|
1326
|
-
const validControlChain = Array.isArray(controlEvents) && controlEvents.every((event, index) => (
|
|
1327
|
-
event?.schema === 'lattice.runtime_control_event.v1'
|
|
1328
|
-
&& event.run_id === pointer.run_id && event.sequence === index + 1
|
|
1329
|
-
&& event.previous_digest === (index === 0 ? null : controlEvents[index - 1].event_digest)
|
|
1330
|
-
&& event.event_digest === selfDigest(event, 'event_digest')
|
|
1331
|
-
));
|
|
1332
|
-
if (descriptor?.descriptor_digest !== pointer.descriptor_digest
|
|
1333
|
-
|| digestArtifact(sessionNonce) !== pointer.session_nonce_digest
|
|
1334
|
-
|| !validControlChain
|
|
1335
|
-
|| controlEvents[pointer.control_head_sequence - 1]?.event_digest !== pointer.control_head_digest) {
|
|
1336
|
-
fail('RUN_NOT_MANAGED', 'active runtime candidate digest binding不正');
|
|
1337
|
-
}
|
|
1338
|
-
return {
|
|
1339
|
-
descriptorPath, sessionPath, controlEventsPath,
|
|
1340
|
-
pointer: structuredClone(pointer),
|
|
1341
|
-
};
|
|
1342
|
-
}
|
|
1343
|
-
|
|
1344
|
-
/** durable daemon側の一要求一応答server。mutation本体はstore所有handlerだけが行う。 */
|
|
1345
|
-
export async function serveRuntimeControlSocket({ socketPath, handler }) {
|
|
1346
|
-
if (typeof socketPath !== 'string' || socketPath.length === 0 || typeof handler !== 'function') fail('SUPERVISOR_CONFIGURATION_INVALID', 'control server入力不正');
|
|
1347
|
-
try {
|
|
1348
|
-
const existing = await lstat(socketPath);
|
|
1349
|
-
if (!existing.isSocket()) fail('RUN_NOT_MANAGED', 'control.sockがsocketでない');
|
|
1350
|
-
let liveOwners;
|
|
1351
|
-
try { liveOwners = await pidsOwningSocketPath(socketPath); }
|
|
1352
|
-
catch (error) {
|
|
1353
|
-
if (error instanceof ManagedRuntimeError) throw error;
|
|
1354
|
-
fail('RUN_NOT_MANAGED', 'control.sock owner観測失敗');
|
|
1355
|
-
}
|
|
1356
|
-
if (liveOwners.length > 0) fail('RUN_BUSY', 'control.sockはlive processが所有する');
|
|
1357
|
-
await rm(socketPath, { force: true });
|
|
1358
|
-
} catch (error) {
|
|
1359
|
-
if (error?.code !== 'ENOENT') throw error;
|
|
1360
|
-
}
|
|
1361
|
-
const absoluteSocket = path.resolve(socketPath);
|
|
1362
|
-
const daemonCwd = await realpath(process.cwd());
|
|
1363
|
-
const socketRef = path.relative(daemonCwd, absoluteSocket);
|
|
1364
|
-
if (socketRef !== 'supervisor/control.sock' || path.resolve(daemonCwd, socketRef) !== absoluteSocket) fail('RUN_NOT_MANAGED', 'daemon control socket cwd binding不正');
|
|
1365
|
-
const server = net.createServer((socket) => {
|
|
1366
|
-
socket.setEncoding('utf8');
|
|
1367
|
-
let buffer = '';
|
|
1368
|
-
let handled = false;
|
|
1369
|
-
socket.on('data', async (chunk) => {
|
|
1370
|
-
if (handled) return;
|
|
1371
|
-
buffer += chunk;
|
|
1372
|
-
if (Buffer.byteLength(buffer, 'utf8') > 1_048_576) { handled = true; socket.destroy(); return; }
|
|
1373
|
-
const newline = buffer.indexOf('\n');
|
|
1374
|
-
if (newline < 0) return;
|
|
1375
|
-
handled = true;
|
|
1376
|
-
try {
|
|
1377
|
-
if (buffer.slice(newline + 1).length !== 0) throw new Error('multiple documents');
|
|
1378
|
-
const request = JSON.parse(buffer.slice(0, newline));
|
|
1379
|
-
if (!validateRuntimeControlRequest(request)) throw new Error('invalid request');
|
|
1380
|
-
const response = await handler(structuredClone(request));
|
|
1381
|
-
if (!validateRuntimeControlResponse(response, request.operation) || response.request_id !== request.request_id || response.run_id !== request.run_id) throw new Error('invalid response');
|
|
1382
|
-
socket.end(`${canonicalizeArtifact(response)}\n`);
|
|
1383
|
-
} catch { socket.destroy(); }
|
|
1384
|
-
});
|
|
1385
|
-
});
|
|
1386
|
-
await new Promise((resolve, reject) => {
|
|
1387
|
-
server.once('error', reject);
|
|
1388
|
-
server.listen(socketRef, () => { server.off('error', reject); resolve(); });
|
|
1389
|
-
});
|
|
1390
|
-
await chmod(socketPath, 0o600);
|
|
1391
|
-
return server;
|
|
1392
|
-
}
|
|
1393
|
-
|
|
1394
|
-
/** CLI寿命から独立する同module daemonを起動する。session/descriptor/store昇格はcallerが検証する。 */
|
|
1395
|
-
export async function launchDurableSupervisor({ runDir, timeoutMs = 5_000 }) {
|
|
1396
|
-
if (typeof runDir !== 'string' || runDir.length === 0 || !Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 60_000) fail('SUPERVISOR_CONFIGURATION_INVALID', 'daemon launch入力不正');
|
|
1397
|
-
const entryPath = fileURLToPath(import.meta.url);
|
|
1398
|
-
const child = spawn(process.execPath, [entryPath, '--serve-managed-run', runDir], {
|
|
1399
|
-
detached: true, stdio: ['ignore', 'ignore', 'ignore', 'ipc'], cwd: runDir,
|
|
1400
|
-
});
|
|
1401
|
-
return new Promise((resolve, reject) => {
|
|
1402
|
-
const timer = setTimeout(() => {
|
|
1403
|
-
try { process.kill(child.pid, 'SIGTERM'); } catch { /* already exited */ }
|
|
1404
|
-
reject(new ManagedRuntimeError('ADAPTER_CONTROLLER_UNAVAILABLE', 'supervisor daemon起動timeout'));
|
|
1405
|
-
}, timeoutMs);
|
|
1406
|
-
const failed = (detail) => { clearTimeout(timer); reject(new ManagedRuntimeError('ADAPTER_CONTROLLER_UNAVAILABLE', detail)); };
|
|
1407
|
-
child.once('error', (error) => failed(error.message));
|
|
1408
|
-
child.once('exit', (code) => failed(`supervisor daemon exited: ${code}`));
|
|
1409
|
-
child.once('message', (message) => {
|
|
1410
|
-
if (message?.schema === 'lattice.runtime_supervisor_bootstrap_failure.v1'
|
|
1411
|
-
&& typeof message.code === 'string' && typeof message.detail === 'string') {
|
|
1412
|
-
clearTimeout(timer);
|
|
1413
|
-
return reject(new ManagedRuntimeError(message.code, message.detail));
|
|
1414
|
-
}
|
|
1415
|
-
if (message?.schema !== 'lattice.runtime_supervisor_bootstrap_ready.v1'
|
|
1416
|
-
|| message.pid !== child.pid || typeof message.socketPath !== 'string'
|
|
1417
|
-
|| typeof message.sessionNonce !== 'string' || message.sessionNonce.length < 32
|
|
1418
|
-
|| !validateProcessStartIdentity(message.processStartIdentity)
|
|
1419
|
-
|| message.processStartIdentity.pid !== child.pid) return failed('supervisor bootstrap message不正');
|
|
1420
|
-
observeManagedProcessStartIdentity(child.pid).then((observed) => {
|
|
1421
|
-
if (canonicalizeArtifact(observed) !== canonicalizeArtifact(message.processStartIdentity)) return failed('supervisor bootstrap PID/start identity不一致');
|
|
1422
|
-
clearTimeout(timer);
|
|
1423
|
-
child.disconnect();
|
|
1424
|
-
child.unref();
|
|
1425
|
-
resolve({ pid: child.pid, processStartIdentity: observed, socketPath: message.socketPath, sessionNonce: message.sessionNonce });
|
|
1426
|
-
}).catch((error) => failed(error.message));
|
|
1427
|
-
});
|
|
1428
|
-
});
|
|
1429
|
-
}
|
|
1430
|
-
|
|
1431
|
-
const isDirectDaemon = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]
|
|
1432
|
-
&& process.argv[2] === '--serve-managed-run';
|
|
1433
|
-
if (isDirectDaemon) {
|
|
1434
|
-
const runDir = process.argv[3];
|
|
1435
|
-
const sessionNonce = randomBytes(32).toString('hex');
|
|
1436
|
-
Promise.all([
|
|
1437
|
-
import('./runtime-cli.mjs'),
|
|
1438
|
-
import('./todo-revision.mjs'),
|
|
1439
|
-
import('./todo-store.mjs'),
|
|
1440
|
-
]).then(async ([{ runManagedSupervisorDaemon }, { validatePhaseTodoRevision },
|
|
1441
|
-
{ applyPhaseTodoRevision, createTodoStoreWriter }]) => {
|
|
1442
|
-
if (typeof runManagedSupervisorDaemon !== 'function') fail('SUPERVISOR_CONFIGURATION_INVALID', 'runtime-cli daemon handlerなし');
|
|
1443
|
-
let daemonCleanup = async () => {};
|
|
1444
|
-
let shuttingDown = false;
|
|
1445
|
-
const stop = async (signal) => {
|
|
1446
|
-
if (shuttingDown) return;
|
|
1447
|
-
shuttingDown = true;
|
|
1448
|
-
try { await daemonCleanup(signal); process.exitCode = 0; }
|
|
1449
|
-
catch (error) { process.stderr.write(`${error.message}\n`); process.exitCode = 1; }
|
|
1450
|
-
};
|
|
1451
|
-
process.once('SIGTERM', () => { stop('SIGTERM'); });
|
|
1452
|
-
process.once('SIGINT', () => { stop('SIGINT'); });
|
|
1453
|
-
await runManagedSupervisorDaemon({
|
|
1454
|
-
runDir,
|
|
1455
|
-
sessionNonce,
|
|
1456
|
-
serveRuntimeControlSocket,
|
|
1457
|
-
activateController: (args) => activateManagedSupervisorController({ ...args, runDir, supervisorSessionNonce: sessionNonce }),
|
|
1458
|
-
onReady: async (socketPath) => process.send?.({ schema: 'lattice.runtime_supervisor_bootstrap_ready.v1', pid: process.pid, processStartIdentity: await observeManagedProcessStartIdentity(process.pid), socketPath, sessionNonce }),
|
|
1459
|
-
registerDaemonCleanup: (handler) => { if (typeof handler !== 'function') fail('SUPERVISOR_CONFIGURATION_INVALID', 'daemon cleanup handler不正'); daemonCleanup = handler; },
|
|
1460
|
-
crashInjector: process.env.NODE_ENV === 'test' && process.env.LATTICE_INTERNAL_TEST_CRASH_POINT
|
|
1461
|
-
? async (point) => {
|
|
1462
|
-
if (point === process.env.LATTICE_INTERNAL_TEST_CRASH_POINT) process.kill(process.pid, 'SIGKILL');
|
|
1463
|
-
}
|
|
1464
|
-
: null,
|
|
1465
|
-
validatePhaseRevision: validatePhaseTodoRevision,
|
|
1466
|
-
commitPhaseRevision: (revision) => applyPhaseTodoRevision({
|
|
1467
|
-
repoRoot: path.resolve(runDir, '..', '..', '..'),
|
|
1468
|
-
writer: createTodoStoreWriter({ caller: 'g5-authoring' }), revision,
|
|
1469
|
-
// actorは`{host, session, agent}`のexact recordである(`lattice.todo_event.v4`)。
|
|
1470
|
-
// 文字列を渡していた間、seam_splitの再計画はphase revisionをcommitできなかった
|
|
1471
|
-
// ——工程storeへ書く直前で必ず落ちるので、請求項8の再開まで一度も届いていない。
|
|
1472
|
-
actor: {
|
|
1473
|
-
host: 'lattice-runtime',
|
|
1474
|
-
session: sessionNonce.slice(0, 32),
|
|
1475
|
-
agent: 'lattice-runtime-supervisor',
|
|
1476
|
-
},
|
|
1477
|
-
recordedAt: new Date().toISOString(),
|
|
1478
|
-
}),
|
|
1479
|
-
});
|
|
1480
|
-
})
|
|
1481
|
-
.catch((error) => {
|
|
1482
|
-
process.send?.({
|
|
1483
|
-
schema: 'lattice.runtime_supervisor_bootstrap_failure.v1',
|
|
1484
|
-
code: error instanceof ManagedRuntimeError ? error.code : 'ADAPTER_CONTROLLER_UNAVAILABLE',
|
|
1485
|
-
detail: String(error?.message ?? error),
|
|
1486
|
-
});
|
|
1487
|
-
process.stderr.write(`${error.message}\n`);
|
|
1488
|
-
process.exitCode = 1;
|
|
1489
|
-
});
|
|
1490
|
-
}
|
|
1
|
+
import net from 'node:net';
|
|
2
|
+
import { createHash, randomBytes, randomUUID } from 'node:crypto';
|
|
3
|
+
import { execFile, spawn } from 'node:child_process';
|
|
4
|
+
import { chmod, lstat, mkdir, readFile, realpath, rm } from 'node:fs/promises';
|
|
5
|
+
import path from 'node:path';
|
|
6
|
+
import { fileURLToPath } from 'node:url';
|
|
7
|
+
import { promisify } from 'node:util';
|
|
8
|
+
import { canonicalizeArtifact, digestArtifact } from './artifact-contracts.mjs';
|
|
9
|
+
import { selfDigest } from './runtime-contracts.mjs';
|
|
10
|
+
import { captureWorktreeDiff } from './runtime-diff-observer.mjs';
|
|
11
|
+
import { pidsOwningSocketPath, socketPathsOwnedByPid } from './runtime-socket-owner.mjs';
|
|
12
|
+
import { createDirectOsProcessObserver as createDirectOsProcessObserverV2 } from './runtime-direct-os-observer.mjs';
|
|
13
|
+
import {
|
|
14
|
+
armStagedWriteLease,
|
|
15
|
+
createControllerBootstrap,
|
|
16
|
+
createControllerRequest,
|
|
17
|
+
createWriteGate,
|
|
18
|
+
validateArmedWriteLease,
|
|
19
|
+
validateAdapterLaunchDescriptor,
|
|
20
|
+
validateAdapterRegistry,
|
|
21
|
+
validateControllerDescriptor,
|
|
22
|
+
validateControllerError,
|
|
23
|
+
validateControllerHeartbeat,
|
|
24
|
+
validateControllerRegistration,
|
|
25
|
+
validateControllerResponse,
|
|
26
|
+
validateControllerHandshakeResponse,
|
|
27
|
+
validateEpochRebindAck,
|
|
28
|
+
validateProcessStartIdentity,
|
|
29
|
+
validateQuiescenceAck,
|
|
30
|
+
validateReleaseAck,
|
|
31
|
+
validateRuntimeControlRequest,
|
|
32
|
+
validateRuntimeControlResponse,
|
|
33
|
+
validateStagedWriteLease,
|
|
34
|
+
verifyCentralWriteGate,
|
|
35
|
+
} from './runtime-controller-protocol.mjs';
|
|
36
|
+
|
|
37
|
+
const SHA256 = /^[0-9a-f]{64}$/;
|
|
38
|
+
const ID = /^[0-9A-Za-z](?:[0-9A-Za-z._-]{0,127})$/;
|
|
39
|
+
|
|
40
|
+
export class ManagedRuntimeError extends Error {
|
|
41
|
+
constructor(code, detail) {
|
|
42
|
+
super(`${code}: ${detail}`);
|
|
43
|
+
this.name = 'ManagedRuntimeError';
|
|
44
|
+
this.code = code;
|
|
45
|
+
this.detail = detail;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function fail(code, detail) { throw new ManagedRuntimeError(code, detail); }
|
|
50
|
+
function digest(value) { return typeof value === 'string' && SHA256.test(value); }
|
|
51
|
+
function identifier(value) { return typeof value === 'string' && ID.test(value); }
|
|
52
|
+
function monotonicDefault() { return Number(process.hrtime.bigint() / 1_000_000n); }
|
|
53
|
+
function sha256Bytes(bytes) { return createHash('sha256').update(bytes).digest('hex'); }
|
|
54
|
+
const execFileAsync = promisify(execFile);
|
|
55
|
+
|
|
56
|
+
export async function observeManagedProcessStartIdentity(pid) {
|
|
57
|
+
let startedIdentity;
|
|
58
|
+
try {
|
|
59
|
+
const { stdout } = await execFileAsync('/bin/ps', ['-o', 'lstart=', '-p', String(pid)], { encoding: 'utf8' });
|
|
60
|
+
startedIdentity = stdout.trim();
|
|
61
|
+
} catch { fail('ADAPTER_CONTROLLER_UNAVAILABLE', `process start identity観測失敗: ${pid}`); }
|
|
62
|
+
if (!startedIdentity) fail('ADAPTER_CONTROLLER_UNAVAILABLE', `process不在: ${pid}`);
|
|
63
|
+
const identity = { schema: 'lattice.process_start_identity.v1', platform: process.platform, pid, started_identity: startedIdentity, identity_digest: '' };
|
|
64
|
+
identity.identity_digest = selfDigest(identity, 'identity_digest');
|
|
65
|
+
return identity;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export async function observeMacosBinaryIdentity(binaryPath) {
|
|
69
|
+
if (process.platform !== 'darwin') fail('ADAPTER_BINARY_IDENTITY_MISMATCH', 'macOS codesign identityはdarwin専用');
|
|
70
|
+
let output;
|
|
71
|
+
try {
|
|
72
|
+
const observed = await execFileAsync('/usr/bin/codesign',
|
|
73
|
+
['-d', '--verbose=4', '--requirements', '-', binaryPath], { encoding: 'utf8' });
|
|
74
|
+
output = `${observed.stdout ?? ''}\n${observed.stderr ?? ''}`;
|
|
75
|
+
} catch (error) {
|
|
76
|
+
output = `${error?.stdout ?? ''}\n${error?.stderr ?? ''}`;
|
|
77
|
+
if (!output.trim()) fail('ADAPTER_BINARY_IDENTITY_MISMATCH', 'codesign identity観測失敗');
|
|
78
|
+
}
|
|
79
|
+
const field = (name) => output.match(new RegExp(`(?:^|\\n)${name}=([^\\n]+)`, 'u'))?.[1]?.trim() ?? '';
|
|
80
|
+
const requirement = output.match(/(?:^|\n)#?\s*designated => ([^\n]+)/u)?.[1]?.trim() ?? '';
|
|
81
|
+
const rawCdhash = (output.match(/(?:^|\n)CandidateCDHashFull sha256=([0-9a-f]+)/u)?.[1]
|
|
82
|
+
?? field('CDHash')).toLowerCase();
|
|
83
|
+
if (!/^[0-9a-f]+$/u.test(rawCdhash) || !field('Identifier') || !requirement) {
|
|
84
|
+
fail('ADAPTER_BINARY_IDENTITY_MISMATCH', 'codesign出力の必須identity不足');
|
|
85
|
+
}
|
|
86
|
+
const identity = {
|
|
87
|
+
schema: 'lattice.macos_binary_identity.v1', kind: 'macos_codesign',
|
|
88
|
+
cdhash: rawCdhash, signing_identifier: field('Identifier'),
|
|
89
|
+
team_identifier: field('TeamIdentifier'),
|
|
90
|
+
designated_requirement_digest: sha256Bytes(Buffer.from(requirement)), identity_digest: '',
|
|
91
|
+
};
|
|
92
|
+
identity.identity_digest = selfDigest(identity, 'identity_digest');
|
|
93
|
+
return identity;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* 起動済みprocessが実際に実行しているimageのcanonical pathをOSから観測する。
|
|
98
|
+
* bytesの同一性はcallerがlaunch descriptorのSHA-256と照合する。
|
|
99
|
+
*/
|
|
100
|
+
export async function observeProcessExecutablePath(pid) {
|
|
101
|
+
if (!Number.isSafeInteger(pid) || pid < 1) {
|
|
102
|
+
fail('ADAPTER_BINARY_IDENTITY_MISMATCH', 'exec後image PID不正');
|
|
103
|
+
}
|
|
104
|
+
if (process.platform === 'linux') {
|
|
105
|
+
return realpath(`/proc/${pid}/exe`);
|
|
106
|
+
}
|
|
107
|
+
if (process.platform === 'darwin') {
|
|
108
|
+
const { stdout } = await execFileAsync(
|
|
109
|
+
'/bin/ps',
|
|
110
|
+
['-o', 'comm=', '-p', String(pid)],
|
|
111
|
+
{ encoding: 'utf8' },
|
|
112
|
+
);
|
|
113
|
+
return realpath(stdout.trim());
|
|
114
|
+
}
|
|
115
|
+
fail('ADAPTER_BINARY_IDENTITY_MISMATCH', `exec後image path観測未対応platform: ${process.platform}`);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** storeが解決したimmutable bindingからprocess/worktree/checkpointをDirect OSで再観測する。 */
|
|
119
|
+
/**
|
|
120
|
+
* 本repositoryの現在状態を1つのdigestへ畳む。
|
|
121
|
+
*
|
|
122
|
+
* HEAD・作業ツリー状態(untracked/ignoredを含む)・全refを見る。workerがworktreeの外へ
|
|
123
|
+
* 書いてもworktreeのdiffには映らないので、ここだけが本repositoryへの書き込みを捕まえる。
|
|
124
|
+
*/
|
|
125
|
+
export async function canonicalRepositoryFingerprint(repoRoot) {
|
|
126
|
+
const parts = [];
|
|
127
|
+
for (const args of [
|
|
128
|
+
['rev-parse', 'HEAD'],
|
|
129
|
+
['status', '--porcelain=v1', '--untracked-files=all', '--ignored=matching'],
|
|
130
|
+
['for-each-ref', '--format=%(refname) %(objectname)'],
|
|
131
|
+
]) {
|
|
132
|
+
const { stdout } = await execFileAsync('git', args, {
|
|
133
|
+
cwd: repoRoot, encoding: 'utf8', maxBuffer: 32 * 1024 * 1024,
|
|
134
|
+
});
|
|
135
|
+
parts.push(stdout);
|
|
136
|
+
}
|
|
137
|
+
return createHash('sha256').update(parts.join('\u0000'), 'utf8').digest('hex');
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
export function createDirectOsProcessObserver({ resolveObservationBinding }) {
|
|
141
|
+
if (typeof resolveObservationBinding !== 'function') fail('SUPERVISOR_CONFIGURATION_INVALID', 'observation binding resolverなし');
|
|
142
|
+
return async ({ kind, binding, ack }) => {
|
|
143
|
+
if (!['quiescence', 'rebind'].includes(kind)) fail('HOLD_ACKS_INCOMPLETE', '未知のDirect OS observation kind');
|
|
144
|
+
const resolved = await resolveObservationBinding({ binding: structuredClone(binding), ack: structuredClone(ack) });
|
|
145
|
+
if (resolved === null || typeof resolved !== 'object' || Array.isArray(resolved)
|
|
146
|
+
|| !Number.isSafeInteger(resolved.process_pid) || resolved.process_pid < 1
|
|
147
|
+
|| !Number.isSafeInteger(resolved.process_group_id) || resolved.process_group_id < 1
|
|
148
|
+
|| typeof resolved.worktree_path !== 'string' || !path.isAbsolute(resolved.worktree_path)
|
|
149
|
+
|| typeof resolved.base_sha !== 'string' || !/^[0-9a-f]{40}$/.test(resolved.base_sha)
|
|
150
|
+
|| !validateProcessStartIdentity(resolved.process_start_identity)
|
|
151
|
+
|| resolved.process_start_identity.pid !== resolved.process_pid) fail('HOLD_ACKS_INCOMPLETE', 'observation binding不正');
|
|
152
|
+
// 本repositoryを見る材料は対で渡す。片方だけでは照合できず、片方だけを受けると
|
|
153
|
+
// 「検査した」と読める記録が検査なしで作れてしまう。
|
|
154
|
+
const canonicalRoot = resolved.canonical_root ?? null;
|
|
155
|
+
const canonicalBaseline = resolved.canonical_fingerprint_digest ?? null;
|
|
156
|
+
if ((canonicalRoot === null) !== (canonicalBaseline === null)
|
|
157
|
+
|| (canonicalRoot !== null && !path.isAbsolute(canonicalRoot))
|
|
158
|
+
|| (canonicalBaseline !== null && !/^[0-9a-f]{64}$/u.test(canonicalBaseline))) {
|
|
159
|
+
fail('HOLD_ACKS_INCOMPLETE', 'canonical repository観測bindingが不正');
|
|
160
|
+
}
|
|
161
|
+
let processState = 'exited';
|
|
162
|
+
let observedIdentity = resolved.process_start_identity;
|
|
163
|
+
let processGroupId = resolved.process_group_id;
|
|
164
|
+
try {
|
|
165
|
+
const { stdout } = await execFileAsync('/bin/ps', ['-o', 'lstart=,pgid=,state=', '-p', String(resolved.process_pid)], { encoding: 'utf8' });
|
|
166
|
+
const line = stdout.trim();
|
|
167
|
+
if (line) {
|
|
168
|
+
const match = line.match(/^(.*\d{4})\s+(\d+)\s+(\S+)$/);
|
|
169
|
+
if (!match) fail('HOLD_ACKS_INCOMPLETE', 'process observation parse失敗');
|
|
170
|
+
observedIdentity = { schema: 'lattice.process_start_identity.v1', platform: process.platform, pid: resolved.process_pid, started_identity: match[1].trim(), identity_digest: '' };
|
|
171
|
+
observedIdentity.identity_digest = selfDigest(observedIdentity, 'identity_digest');
|
|
172
|
+
if (observedIdentity.identity_digest !== resolved.process_start_identity.identity_digest) fail('HOLD_ACKS_INCOMPLETE', 'PID再利用又はstart identity差替え');
|
|
173
|
+
processGroupId = Number(match[2]);
|
|
174
|
+
if (processGroupId !== resolved.process_group_id) fail('HOLD_ACKS_INCOMPLETE', 'process group差替え');
|
|
175
|
+
processState = match[3].startsWith('T') ? 'stopped' : 'running';
|
|
176
|
+
}
|
|
177
|
+
} catch (error) {
|
|
178
|
+
if (error instanceof ManagedRuntimeError) throw error;
|
|
179
|
+
processState = 'exited';
|
|
180
|
+
}
|
|
181
|
+
if (!['stopped', 'exited'].includes(processState)) fail('HOLD_ACKS_INCOMPLETE', 'executor processがquiescedでない');
|
|
182
|
+
try {
|
|
183
|
+
const { stdout: groupStdout } = await execFileAsync('/bin/ps', ['-o', 'pid=,state=', '-g', String(resolved.process_group_id)], { encoding: 'utf8' });
|
|
184
|
+
for (const line of groupStdout.trim().split('\n').filter(Boolean)) {
|
|
185
|
+
const match = line.trim().match(/^(\d+)\s+(\S+)$/);
|
|
186
|
+
if (!match || (!match[2].startsWith('T') && !match[2].startsWith('Z'))) fail('HOLD_ACKS_INCOMPLETE', 'process group childがquiescedでない');
|
|
187
|
+
}
|
|
188
|
+
} catch (error) {
|
|
189
|
+
if (error instanceof ManagedRuntimeError) throw error;
|
|
190
|
+
if (!(error?.code === 1 && String(error?.stdout ?? '').trim() === '')) {
|
|
191
|
+
fail('HOLD_ACKS_INCOMPLETE', 'process group直接観測失敗');
|
|
192
|
+
}
|
|
193
|
+
// exit 1かつ出力空だけをgroup消滅済みと解釈する。
|
|
194
|
+
}
|
|
195
|
+
const worktreeInfo = await lstat(resolved.worktree_path);
|
|
196
|
+
if (!worktreeInfo.isDirectory() || worktreeInfo.isSymbolicLink()) fail('HOLD_ACKS_INCOMPLETE', 'worktreeがreal directoryでない');
|
|
197
|
+
const worktreeRealpath = await realpath(resolved.worktree_path);
|
|
198
|
+
if (worktreeRealpath !== resolved.worktree_path) fail('HOLD_ACKS_INCOMPLETE', 'worktree path差替え');
|
|
199
|
+
const checkpoint = await captureWorktreeDiff({ worktreePath: worktreeRealpath, baseSha: resolved.base_sha });
|
|
200
|
+
const processObservation = { schema: 'lattice.direct_process_observation.v1', pid: resolved.process_pid, process_start_identity_digest: observedIdentity.identity_digest, process_group_id: processGroupId, state: processState };
|
|
201
|
+
const processObservationDigest = digestArtifact(processObservation);
|
|
202
|
+
// worktreeの外——本repository——への書き込みは、worktreeのdiffにはまったく映らない。
|
|
203
|
+
// 見ていないことを「変更が無かった」と読ませないため、検査したかどうかを記録へ残す
|
|
204
|
+
// (ADR 0140)。渡されていなければ`null`=未検査であり、無変更の主張ではない。
|
|
205
|
+
let canonicalDigest = null;
|
|
206
|
+
if (canonicalRoot !== null) {
|
|
207
|
+
canonicalDigest = await canonicalRepositoryFingerprint(canonicalRoot);
|
|
208
|
+
if (canonicalDigest !== canonicalBaseline) {
|
|
209
|
+
fail('HOLD_ACKS_INCOMPLETE', 'canonical repositoryがworker実行中に変化した');
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
const worktreeFingerprint = { schema: 'lattice.direct_worktree_fingerprint.v2', worktree_id: binding?.worktree_id ?? ack.worktree_id, worktree_realpath: worktreeRealpath, checkpoint_digest: checkpoint.checkpoint_digest, canonical_fingerprint_digest: canonicalDigest };
|
|
213
|
+
const worktreeFingerprintDigest = digestArtifact(worktreeFingerprint);
|
|
214
|
+
return {
|
|
215
|
+
quiesced: true,
|
|
216
|
+
process_observation_digest: processObservationDigest,
|
|
217
|
+
worktree_fingerprint_digest: worktreeFingerprintDigest,
|
|
218
|
+
final_checkpoint_digest: checkpoint.checkpoint_digest,
|
|
219
|
+
observation: processObservation,
|
|
220
|
+
worktree_fingerprint: worktreeFingerprint,
|
|
221
|
+
checkpoint,
|
|
222
|
+
write_enabled: false,
|
|
223
|
+
};
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
async function readCanonicalRegular(filePath) {
|
|
228
|
+
const info = await lstat(filePath);
|
|
229
|
+
if (!info.isFile() || info.isSymbolicLink()) fail('ADAPTER_LAUNCH_INVALID', `regular fileでない: ${filePath}`);
|
|
230
|
+
const bytes = await readFile(filePath);
|
|
231
|
+
let value;
|
|
232
|
+
try { value = JSON.parse(bytes); } catch { fail('ADAPTER_LAUNCH_INVALID', `JSON不正: ${filePath}`); }
|
|
233
|
+
if (bytes.toString('utf8') !== `${canonicalizeArtifact(value)}\n`) fail('ADAPTER_LAUNCH_INVALID', `canonical bytesでない: ${filePath}`);
|
|
234
|
+
return { value, bytes };
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function validateRunningBinding(value) {
|
|
238
|
+
const fields = ['todo_id', 'executor_handle', 'worktree_id', 'plan_epoch', 'packet_digest', 'write_lease_id', 'controller_registration_digest'];
|
|
239
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
|
240
|
+
&& Object.keys(value).sort().join('\0') === fields.sort().join('\0')
|
|
241
|
+
&& identifier(value.todo_id) && identifier(value.executor_handle) && identifier(value.worktree_id)
|
|
242
|
+
&& Number.isSafeInteger(value.plan_epoch) && value.plan_epoch > 0
|
|
243
|
+
&& digest(value.packet_digest) && identifier(value.write_lease_id)
|
|
244
|
+
&& digest(value.controller_registration_digest);
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/**
|
|
248
|
+
* ADR 0064のcontrol plane本体。永続化はcallerが渡すjournal/gateWriterへ明示委譲し、
|
|
249
|
+
* controller transportとOS再観測なしには停止・再認可を成功扱いしない。
|
|
250
|
+
*/
|
|
251
|
+
export class RuntimeManagedSupervisor {
|
|
252
|
+
#runId;
|
|
253
|
+
#sessionNonceDigest;
|
|
254
|
+
#clock;
|
|
255
|
+
#processObserver;
|
|
256
|
+
#runningBindingResolver;
|
|
257
|
+
#journal;
|
|
258
|
+
#gateWriter;
|
|
259
|
+
#shutdownFinalizer;
|
|
260
|
+
#controllers = new Map();
|
|
261
|
+
#registrationToController = new Map();
|
|
262
|
+
#leases = new Map();
|
|
263
|
+
#gate = null;
|
|
264
|
+
#previousGate = null;
|
|
265
|
+
#releaseAcks = [];
|
|
266
|
+
#gateRecords = new Map();
|
|
267
|
+
#frozen = false;
|
|
268
|
+
#gateGeneration = 0;
|
|
269
|
+
|
|
270
|
+
constructor({ runId, sessionNonceDigest, clock = monotonicDefault, processObserver, runningBindingResolver, journal, gateWriter, shutdownFinalizer = async () => {} }) {
|
|
271
|
+
if (!identifier(runId) || !digest(sessionNonceDigest) || typeof clock !== 'function'
|
|
272
|
+
|| typeof processObserver !== 'function' || typeof runningBindingResolver !== 'function'
|
|
273
|
+
|| typeof journal?.append !== 'function'
|
|
274
|
+
|| typeof gateWriter?.commit !== 'function' || typeof shutdownFinalizer !== 'function') fail('SUPERVISOR_CONFIGURATION_INVALID', 'durable dependencyが不足');
|
|
275
|
+
this.#runId = runId;
|
|
276
|
+
this.#sessionNonceDigest = sessionNonceDigest;
|
|
277
|
+
this.#clock = clock;
|
|
278
|
+
this.#processObserver = processObserver;
|
|
279
|
+
this.#runningBindingResolver = runningBindingResolver;
|
|
280
|
+
this.#journal = journal;
|
|
281
|
+
this.#gateWriter = gateWriter;
|
|
282
|
+
this.#shutdownFinalizer = shutdownFinalizer;
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
async hydrateCommittedGate() {
|
|
286
|
+
if (typeof this.#gateWriter.read !== 'function') return null;
|
|
287
|
+
const committed = await this.#gateWriter.read();
|
|
288
|
+
if (committed === null) return null;
|
|
289
|
+
if (committed.gate.run_id !== this.#runId
|
|
290
|
+
|| !Number.isSafeInteger(committed.gate.gate_generation)
|
|
291
|
+
|| committed.gate.gate_generation < 1) {
|
|
292
|
+
fail('EPOCH_ACTIVATION_INCOMPLETE', 'persisted gate generation binding不正');
|
|
293
|
+
}
|
|
294
|
+
this.#gate = structuredClone(committed.gate);
|
|
295
|
+
this.#gateGeneration = committed.gate.gate_generation;
|
|
296
|
+
return structuredClone(committed.gate);
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
get runId() { return this.#runId; }
|
|
300
|
+
get frozen() { return this.#frozen; }
|
|
301
|
+
get gate() { return this.#gate === null ? null : structuredClone(this.#gate); }
|
|
302
|
+
|
|
303
|
+
async registerController({ descriptor, registration, transport }) {
|
|
304
|
+
if (!validateControllerDescriptor(descriptor) || !validateControllerRegistration(registration)
|
|
305
|
+
|| registration.run_id !== this.#runId
|
|
306
|
+
|| registration.supervisor_session_nonce_digest !== this.#sessionNonceDigest
|
|
307
|
+
|| registration.controller_descriptor_digest !== descriptor.descriptor_digest
|
|
308
|
+
|| typeof transport?.request !== 'function') fail('ADAPTER_CONTROLLER_UNAVAILABLE', 'controller registration binding不正');
|
|
309
|
+
if (this.#controllers.has(descriptor.controller_id)
|
|
310
|
+
|| this.#registrationToController.has(registration.registration_digest)) fail('ADAPTER_CONTROLLER_UNAVAILABLE', 'controller二重登録');
|
|
311
|
+
const now = this.#clock();
|
|
312
|
+
const record = { descriptor: structuredClone(descriptor), registration: structuredClone(registration),
|
|
313
|
+
transport, lastHeartbeat: now, lastHeartbeatSequence: 0,
|
|
314
|
+
lastRecordedLeaseSetDigest: null, connected: true, revoked: false };
|
|
315
|
+
this.#controllers.set(descriptor.controller_id, record);
|
|
316
|
+
this.#registrationToController.set(registration.registration_digest, descriptor.controller_id);
|
|
317
|
+
await this.#append('controller_registered', { controller_id: descriptor.controller_id, registration_digest: registration.registration_digest });
|
|
318
|
+
return structuredClone(registration);
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
async heartbeat({ controllerId, registrationDigest, sequence, leaseSetDigest, sessionNonceDigest }) {
|
|
322
|
+
const record = this.#requireController(controllerId);
|
|
323
|
+
if (!record.connected || sessionNonceDigest !== this.#sessionNonceDigest
|
|
324
|
+
|| registrationDigest !== record.registration.registration_digest
|
|
325
|
+
|| !Number.isSafeInteger(sequence) || sequence <= record.lastHeartbeatSequence || !digest(leaseSetDigest)) {
|
|
326
|
+
await this.#failClosed(record, 'CONTROLLER_HEARTBEAT_EXPIRED', 'heartbeat binding不正');
|
|
327
|
+
}
|
|
328
|
+
// prepare/release応答とsupervisor投影の間には短い非同期窓がある。heartbeatは
|
|
329
|
+
// livenessだけを担い、lease集合はwrite認可時のcentral gate full-chainで照合する。
|
|
330
|
+
record.lastHeartbeat = this.#clock();
|
|
331
|
+
record.lastHeartbeatSequence = sequence;
|
|
332
|
+
// heartbeatはlivenessの更新であり、同じlease集合のtickを全件journalへ積む必要はない。
|
|
333
|
+
// 初回とlease集合の変化だけを耐久記録し、壁時計に比例するjournal成長を止める。
|
|
334
|
+
if (record.lastRecordedLeaseSetDigest !== leaseSetDigest) {
|
|
335
|
+
await this.#append('controller_heartbeat', { controller_id: controllerId,
|
|
336
|
+
registration_digest: registrationDigest, sequence, lease_set_digest: leaseSetDigest });
|
|
337
|
+
record.lastRecordedLeaseSetDigest = leaseSetDigest;
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
async disconnect(controllerId) {
|
|
342
|
+
const record = this.#requireController(controllerId);
|
|
343
|
+
record.connected = false;
|
|
344
|
+
await this.#failClosed(record, 'CONTROLLER_HEARTBEAT_EXPIRED', 'controller socket disconnected');
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
async assertControllerHealth() {
|
|
348
|
+
for (const record of this.#controllers.values()) {
|
|
349
|
+
const age = this.#clock() - record.lastHeartbeat;
|
|
350
|
+
if (!record.connected || age < 0 || age > record.descriptor.heartbeat.ttl_ms) {
|
|
351
|
+
await this.#failClosed(record, 'CONTROLLER_HEARTBEAT_EXPIRED', 'heartbeat TTL超過');
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
async route(operation, controllerId, fields) {
|
|
357
|
+
await this.assertControllerHealth();
|
|
358
|
+
if (this.#frozen && ['dispatch', 'observe'].includes(operation)) fail('RUN_FROZEN', 'freeze中のroute');
|
|
359
|
+
const record = this.#requireController(controllerId);
|
|
360
|
+
const request = createControllerRequest(operation, {
|
|
361
|
+
request_id: fields.request_id ?? randomUUID(),
|
|
362
|
+
registration_digest: record.registration.registration_digest,
|
|
363
|
+
...Object.fromEntries(Object.entries(fields).filter(([key]) => !['request_id', 'registration_digest'].includes(key))),
|
|
364
|
+
});
|
|
365
|
+
const response = await record.transport.request(operation, structuredClone(request));
|
|
366
|
+
if (!validateControllerResponse(operation, response, request.request_id)) {
|
|
367
|
+
await this.#failClosed(record, 'ADAPTER_CONTROLLER_UNAVAILABLE', `${operation} response不正`);
|
|
368
|
+
}
|
|
369
|
+
// running pollは状態遷移ではない。全tickを耐久化すると長寿命workerほどcontrol journalを
|
|
370
|
+
// 膨らませ、artifact document上限でsupervisor自身を落とす。dispatchと非running観測だけを残す。
|
|
371
|
+
if (operation === 'dispatch' || response.observation?.state !== 'running') {
|
|
372
|
+
await this.#append(operation === 'dispatch' ? 'dispatch_routed' : 'observation_routed', {
|
|
373
|
+
controller_id: controllerId, request_digest: request.request_digest,
|
|
374
|
+
response_digest: response.response_digest,
|
|
375
|
+
});
|
|
376
|
+
}
|
|
377
|
+
return structuredClone(response);
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
/** 全running bindingを一件も省略せずbarrierし、controller ackと独立OS観測を照合する。 */
|
|
381
|
+
async barrierAll({ barrierId, reason, frozenEventDigest }) {
|
|
382
|
+
return this.#barrier({ barrierId, reason, frozenEventDigest, targetTodoIds: null });
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/** findingの影響群だけを静止し、無関係なrunning bindingはそのまま通す。 */
|
|
386
|
+
async barrierSelected({ barrierId, reason, frozenEventDigest, todoIds }) {
|
|
387
|
+
if (!Array.isArray(todoIds) || todoIds.length === 0
|
|
388
|
+
|| todoIds.some((todoId) => !identifier(todoId))
|
|
389
|
+
|| new Set(todoIds).size !== todoIds.length) {
|
|
390
|
+
fail('HOLD_ACKS_INCOMPLETE', '対象barrierのTODO集合が不正');
|
|
391
|
+
}
|
|
392
|
+
return this.#barrier({ barrierId, reason, frozenEventDigest,
|
|
393
|
+
targetTodoIds: new Set(todoIds) });
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
async #barrier({ barrierId, reason, frozenEventDigest, targetTodoIds }) {
|
|
397
|
+
const durableBindings = await this.#runningBindingResolver({ runId: this.#runId, frozenEventDigest });
|
|
398
|
+
if (!identifier(barrierId) || typeof reason !== 'string' || reason.length === 0
|
|
399
|
+
|| !digest(frozenEventDigest) || !Array.isArray(durableBindings)
|
|
400
|
+
|| !durableBindings.every(validateRunningBinding)) fail('HOLD_ACKS_INCOMPLETE', 'durable running binding不正');
|
|
401
|
+
await this.assertControllerHealth();
|
|
402
|
+
this.#frozen = true;
|
|
403
|
+
const allRunningBindings = await this.#resolveRunningUnion({ durableBindings, frozenEventDigest });
|
|
404
|
+
const runningBindings = targetTodoIds === null ? allRunningBindings
|
|
405
|
+
: allRunningBindings.filter((binding) => targetTodoIds.has(binding.todo_id));
|
|
406
|
+
const unique = new Set(runningBindings.map((binding) => binding.todo_id));
|
|
407
|
+
if (unique.size !== runningBindings.length) fail('HOLD_ACKS_INCOMPLETE', 'running TODO重複');
|
|
408
|
+
const barrierControlDigest = await this.#append('barrier_requested', { barrier_id: barrierId,
|
|
409
|
+
reason, running_count: runningBindings.length,
|
|
410
|
+
running_todo_ids: runningBindings.map((binding) => binding.todo_id).sort(),
|
|
411
|
+
frozen_event_digest: frozenEventDigest });
|
|
412
|
+
if (!digest(barrierControlDigest)) fail('HOLD_ACKS_INCOMPLETE', 'barrier control event未耐久化');
|
|
413
|
+
const acknowledgements = [];
|
|
414
|
+
for (const record of this.#controllers.values()) {
|
|
415
|
+
const owned = runningBindings.filter((binding) => binding.controller_registration_digest === record.registration.registration_digest);
|
|
416
|
+
if (owned.length === 0) continue;
|
|
417
|
+
const response = await this.routeWhileFrozen('barrier', record, {
|
|
418
|
+
request_id: randomUUID(), barrier_id: barrierId, reason,
|
|
419
|
+
running_bindings: owned, frozen_event_digest: frozenEventDigest,
|
|
420
|
+
barrier_control_digest: barrierControlDigest,
|
|
421
|
+
});
|
|
422
|
+
if (response.barrier_id !== barrierId || !Array.isArray(response.quiescence_acks)
|
|
423
|
+
|| response.quiescence_acks.length !== owned.length) fail('HOLD_ACKS_INCOMPLETE', 'barrier ack件数不一致');
|
|
424
|
+
for (const binding of owned) {
|
|
425
|
+
const ack = response.quiescence_acks.find((candidate) => candidate.todo_id === binding.todo_id);
|
|
426
|
+
if (!validateQuiescenceAck(ack) || ack.run_id !== this.#runId
|
|
427
|
+
|| ack.executor_handle !== binding.executor_handle || ack.worktree_id !== binding.worktree_id
|
|
428
|
+
|| ack.plan_epoch !== binding.plan_epoch || ack.packet_digest !== binding.packet_digest
|
|
429
|
+
|| ack.write_lease_id !== binding.write_lease_id
|
|
430
|
+
|| ack.barrier_control_digest !== barrierControlDigest
|
|
431
|
+
|| ack.supervisor_session_nonce_digest !== this.#sessionNonceDigest) fail('HOLD_ACKS_INCOMPLETE', `ack binding不一致: ${binding.todo_id}`);
|
|
432
|
+
const observed = await this.#processObserver({ kind: 'quiescence', binding: structuredClone(binding), ack: structuredClone(ack), descriptor: structuredClone(record.descriptor) });
|
|
433
|
+
if (observed?.process_observation_digest !== ack.process_observation_digest
|
|
434
|
+
|| observed?.worktree_fingerprint_digest !== ack.worktree_fingerprint_digest
|
|
435
|
+
|| observed?.final_checkpoint_digest !== ack.final_checkpoint_digest
|
|
436
|
+
|| observed?.quiesced !== true) fail('HOLD_ACKS_INCOMPLETE', `直接再観測不一致: ${binding.todo_id}`);
|
|
437
|
+
acknowledgements.push(ack);
|
|
438
|
+
const directObservation = structuredClone(observed);
|
|
439
|
+
await this.#append('executor_quiesced', { barrier_id: barrierId,
|
|
440
|
+
barrier_control_digest: barrierControlDigest, todo_id: binding.todo_id,
|
|
441
|
+
ack: structuredClone(ack), direct_observation: directObservation,
|
|
442
|
+
evidence_digest: digestArtifact({ ack, direct_observation: directObservation }) });
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
if (acknowledgements.length !== runningBindings.length) fail('HOLD_ACKS_INCOMPLETE', '未登録controller所有bindingあり');
|
|
446
|
+
const observedAfterBarrier = await this.#collectControllerRunning({ frozenEventDigest });
|
|
447
|
+
const residualBindings = targetTodoIds === null ? observedAfterBarrier
|
|
448
|
+
: observedAfterBarrier.filter((binding) => targetTodoIds.has(binding.todo_id));
|
|
449
|
+
if (residualBindings.length !== 0) {
|
|
450
|
+
fail('HOLD_ACKS_INCOMPLETE', `barrier後もrunning executor残存: ${residualBindings.map((binding) => binding.todo_id).join(',')}`);
|
|
451
|
+
}
|
|
452
|
+
return acknowledgements.map((ack) => structuredClone(ack));
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
/** durable storeと全controllerの直接inventoryをexact binding単位で和集合にする。 */
|
|
456
|
+
async #resolveRunningUnion({ durableBindings, frozenEventDigest }) {
|
|
457
|
+
const observedBindings = await this.#collectControllerRunning({ frozenEventDigest });
|
|
458
|
+
const union = new Map();
|
|
459
|
+
const executorOwners = new Map();
|
|
460
|
+
for (const binding of [...durableBindings, ...observedBindings]) {
|
|
461
|
+
const controllerId = this.#registrationToController.get(binding.controller_registration_digest);
|
|
462
|
+
if (controllerId === undefined) fail('HOLD_ACKS_INCOMPLETE', `未登録controller binding: ${binding.todo_id}`);
|
|
463
|
+
const canonical = canonicalizeArtifact(binding);
|
|
464
|
+
const prior = union.get(binding.todo_id);
|
|
465
|
+
if (prior !== undefined && prior.canonical !== canonical) {
|
|
466
|
+
fail('HOLD_ACKS_INCOMPLETE', `durable/controller running binding不一致: ${binding.todo_id}`);
|
|
467
|
+
}
|
|
468
|
+
const executorOwner = executorOwners.get(binding.executor_handle);
|
|
469
|
+
if (executorOwner !== undefined && executorOwner !== binding.todo_id) {
|
|
470
|
+
fail('HOLD_ACKS_INCOMPLETE', `executor handle重複: ${binding.executor_handle}`);
|
|
471
|
+
}
|
|
472
|
+
union.set(binding.todo_id, { canonical, binding: structuredClone(binding) });
|
|
473
|
+
executorOwners.set(binding.executor_handle, binding.todo_id);
|
|
474
|
+
}
|
|
475
|
+
return [...union.values()].map(({ binding }) => binding)
|
|
476
|
+
.sort((left, right) => left.todo_id < right.todo_id ? -1 : left.todo_id > right.todo_id ? 1 : 0);
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
async #collectControllerRunning({ frozenEventDigest }) {
|
|
480
|
+
const observed = [];
|
|
481
|
+
for (const record of this.#controllers.values()) {
|
|
482
|
+
const response = await this.routeWhileFrozen('inventory', record, {
|
|
483
|
+
request_id: randomUUID(), frozen_event_digest: frozenEventDigest,
|
|
484
|
+
});
|
|
485
|
+
if (response.inventory_digest !== digestArtifact(response.running_bindings)
|
|
486
|
+
|| response.running_bindings.some((binding) => (
|
|
487
|
+
binding.controller_registration_digest !== record.registration.registration_digest))) {
|
|
488
|
+
fail('HOLD_ACKS_INCOMPLETE', `controller inventory binding不一致: ${record.descriptor.controller_id}`);
|
|
489
|
+
}
|
|
490
|
+
observed.push(...response.running_bindings.map((binding) => structuredClone(binding)));
|
|
491
|
+
}
|
|
492
|
+
return observed;
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
/** finding/store永続化後に呼ぶtyped hold閉路。対象群のquiescence完了後だけheldを返す。 */
|
|
496
|
+
async holdConflict({ findingDigest, frozenEventDigest, barrierId, reason, recordedAt,
|
|
497
|
+
todoIds = null }) {
|
|
498
|
+
if (!digest(findingDigest) || typeof recordedAt !== 'string' || Number.isNaN(Date.parse(recordedAt))) fail('FINDING_UNRESOLVED', 'hold finding binding不正');
|
|
499
|
+
const acknowledgements = todoIds === null
|
|
500
|
+
? await this.barrierAll({ barrierId, reason, frozenEventDigest })
|
|
501
|
+
: await this.barrierSelected({ barrierId, reason, frozenEventDigest,
|
|
502
|
+
todoIds });
|
|
503
|
+
const result = {
|
|
504
|
+
schema: 'lattice.runtime_hold_result.v1', run_id: this.#runId,
|
|
505
|
+
finding_digest: findingDigest, barrier_id: barrierId,
|
|
506
|
+
quiescence_ack_digests: acknowledgements.map((ack) => ack.ack_digest).sort(),
|
|
507
|
+
outcome: 'held', recorded_at: recordedAt, result_digest: '',
|
|
508
|
+
};
|
|
509
|
+
result.result_digest = selfDigest(result, 'result_digest');
|
|
510
|
+
return result;
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
async shutdownManaged({ mode, reason, barrierId, frozenEventDigest, recordedAt }) {
|
|
514
|
+
if (!['close', 'abandon'].includes(mode) || typeof reason !== 'string' || reason.length === 0
|
|
515
|
+
|| typeof recordedAt !== 'string' || Number.isNaN(Date.parse(recordedAt))) fail('MANAGED_SHUTDOWN_INCOMPLETE', 'shutdown入力不正');
|
|
516
|
+
const acknowledgements = await this.barrierAll({ barrierId, reason: `managed_${mode}:${reason}`, frozenEventDigest });
|
|
517
|
+
for (const record of this.#controllers.values()) {
|
|
518
|
+
const owned = [...this.#leases.values()].filter((entry) => entry.controllerId === record.descriptor.controller_id && !entry.revoked);
|
|
519
|
+
const leaseDigests = owned.map((entry) => entry.lease.lease_digest).sort();
|
|
520
|
+
const response = await this.routeWhileFrozen('revoke', record, { request_id: randomUUID(), reason, lease_digests: leaseDigests });
|
|
521
|
+
if (JSON.stringify(response.revoked_lease_digests) !== JSON.stringify(leaseDigests)
|
|
522
|
+
|| response.residual_processes.length !== 0) fail('MANAGED_SHUTDOWN_INCOMPLETE', 'controller revoke不完全');
|
|
523
|
+
for (const entry of owned) entry.revoked = true;
|
|
524
|
+
await this.#append('lease_revoked', { controller_id: record.descriptor.controller_id, reason });
|
|
525
|
+
}
|
|
526
|
+
await this.#shutdownFinalizer();
|
|
527
|
+
const result = { schema: 'lattice.runtime_shutdown_result.v1', run_id: this.#runId, mode, outcome: 'quiesced', quiescence_ack_digests: acknowledgements.map((ack) => ack.ack_digest).sort(), recorded_at: recordedAt, result_digest: '' };
|
|
528
|
+
result.result_digest = selfDigest(result, 'result_digest');
|
|
529
|
+
return result;
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
async rebindController({ controllerId, rebindPacket, stagedLease, expected }) {
|
|
533
|
+
const record = this.#requireController(controllerId);
|
|
534
|
+
if (!this.#frozen) fail('EPOCH_REBIND_INCOMPLETE', 'freeze外rebind');
|
|
535
|
+
const response = await this.routeWhileFrozen('rebind', record, {
|
|
536
|
+
request_id: randomUUID(), rebind_packet: rebindPacket, staged_lease: stagedLease,
|
|
537
|
+
});
|
|
538
|
+
if (response.staged_lease_digest !== stagedLease.lease_digest) fail('EPOCH_REBIND_INCOMPLETE', 'staged lease response不一致');
|
|
539
|
+
const controlEventDigest = await this.#acceptRebindAck({ controllerId,
|
|
540
|
+
ack: response.rebind_ack, stagedLease, expected });
|
|
541
|
+
return { ack: structuredClone(response.rebind_ack), control_event_digest: controlEventDigest };
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
/** affected successorを実行不能なstaged leaseのままprepareする。 */
|
|
545
|
+
async prepareController({ controllerId, executorPacket, stagedLease }) {
|
|
546
|
+
const record = this.#requireController(controllerId);
|
|
547
|
+
if (!this.#frozen || !validateStagedWriteLease(stagedLease)
|
|
548
|
+
|| stagedLease.run_id !== this.#runId || stagedLease.state !== 'staged'
|
|
549
|
+
|| stagedLease.todo_id !== executorPacket?.todo_id
|
|
550
|
+
|| stagedLease.packet_digest !== executorPacket?.packet_digest
|
|
551
|
+
|| stagedLease.controller_registration_digest !== record.registration.registration_digest) {
|
|
552
|
+
fail('EPOCH_REBIND_INCOMPLETE', 'prepare staged binding不正');
|
|
553
|
+
}
|
|
554
|
+
const response = await this.routeWhileFrozen('prepare', record, {
|
|
555
|
+
request_id: randomUUID(), executor_packet: executorPacket, staged_lease: stagedLease,
|
|
556
|
+
});
|
|
557
|
+
const ack = response.prepare_ack;
|
|
558
|
+
if (response.staged_lease_digest !== stagedLease.lease_digest
|
|
559
|
+
|| ack.run_id !== this.#runId || ack.todo_id !== executorPacket.todo_id
|
|
560
|
+
|| ack.plan_epoch !== executorPacket.plan_epoch
|
|
561
|
+
|| ack.packet_digest !== executorPacket.packet_digest
|
|
562
|
+
|| ack.staged_lease_digest !== stagedLease.lease_digest
|
|
563
|
+
|| ack.registration_digest !== record.registration.registration_digest
|
|
564
|
+
|| ack.controller_id !== controllerId
|
|
565
|
+
|| ack.supervisor_session_nonce_digest !== this.#sessionNonceDigest) {
|
|
566
|
+
fail('EPOCH_REBIND_INCOMPLETE', 'direct prepare ack不一致');
|
|
567
|
+
}
|
|
568
|
+
this.#leases.set(stagedLease.lease_digest, { lease: structuredClone(stagedLease),
|
|
569
|
+
controllerId, issuedAt: this.#clock(), revoked: false });
|
|
570
|
+
return structuredClone(ack);
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
async #acceptRebindAck({ controllerId, ack, stagedLease, expected }) {
|
|
574
|
+
const record = this.#requireController(controllerId);
|
|
575
|
+
if (!this.#frozen || !validateEpochRebindAck(ack) || !validateStagedWriteLease(stagedLease)
|
|
576
|
+
|| ack.run_id !== this.#runId || ack.supervisor_session_nonce_digest !== this.#sessionNonceDigest
|
|
577
|
+
|| stagedLease.run_id !== this.#runId || stagedLease.state !== 'staged'
|
|
578
|
+
|| stagedLease.controller_registration_digest !== record.registration.registration_digest
|
|
579
|
+
|| ack.new_write_lease_id !== stagedLease.lease_id
|
|
580
|
+
|| ack.todo_id !== expected.todo_id || ack.executor_handle !== expected.executor_handle
|
|
581
|
+
|| ack.worktree_id !== expected.worktree_id || ack.predecessor_packet_digest !== expected.predecessor_packet_digest
|
|
582
|
+
|| ack.rebind_packet_digest !== expected.rebind_packet_digest) fail('EPOCH_REBIND_INCOMPLETE', 'direct rebind ack不一致');
|
|
583
|
+
const observed = await this.#processObserver({ kind: 'rebind', ack: structuredClone(ack), stagedLease: structuredClone(stagedLease), descriptor: structuredClone(record.descriptor) });
|
|
584
|
+
if (observed?.write_enabled !== false) fail('EPOCH_REBIND_INCOMPLETE', 'rebind直接再観測不一致');
|
|
585
|
+
this.#leases.set(stagedLease.lease_digest, { lease: structuredClone(stagedLease), controllerId, issuedAt: this.#clock(), revoked: false });
|
|
586
|
+
return this.#append('epoch_rebind_acknowledged', { todo_id: ack.todo_id,
|
|
587
|
+
ack_digest: ack.ack_digest, staged_lease_digest: stagedLease.lease_digest });
|
|
588
|
+
}
|
|
589
|
+
|
|
590
|
+
/** 全release ackを回収した後だけ、中央gateを一回commitしてstaged v1→armed v2を有効化する。 */
|
|
591
|
+
async commitWriteGate({ planEpoch, committedEpochDigest, activationDigest,
|
|
592
|
+
releaseBarrierDigest = null, commitReleaseBarrier = null,
|
|
593
|
+
afterControllerRelease = null, committedAt }) {
|
|
594
|
+
await this.assertControllerHealth();
|
|
595
|
+
if (!this.#frozen || !Number.isSafeInteger(planEpoch) || planEpoch < 1
|
|
596
|
+
|| !(releaseBarrierDigest === null || digest(releaseBarrierDigest))
|
|
597
|
+
|| !digest(committedEpochDigest) || !digest(activationDigest)) fail('EPOCH_ACTIVATION_INCOMPLETE', 'gate入力不正');
|
|
598
|
+
const staged = [...this.#leases.values()].filter((entry) => !entry.revoked && entry.lease.plan_epoch === planEpoch && validateStagedWriteLease(entry.lease));
|
|
599
|
+
const nextGeneration = this.#gateGeneration + 1;
|
|
600
|
+
const readyAcks = [];
|
|
601
|
+
let releaseIndex = 0;
|
|
602
|
+
for (const record of this.#controllers.values()) {
|
|
603
|
+
const ownedStaged = staged.filter((entry) => entry.lease.controller_registration_digest === record.registration.registration_digest).map((entry) => entry.lease.lease_digest).sort();
|
|
604
|
+
const ready = await this.routeWhileFrozen('activate', record, {
|
|
605
|
+
request_id: randomUUID(), committed_epoch_digest: committedEpochDigest,
|
|
606
|
+
activation_digest: activationDigest, staged_lease_digests: ownedStaged,
|
|
607
|
+
});
|
|
608
|
+
if (ready.observed_pointer_digest !== committedEpochDigest
|
|
609
|
+
|| ready.ready_ack.registration_digest !== record.registration.registration_digest
|
|
610
|
+
|| ready.ready_ack.controller_id !== record.descriptor.controller_id
|
|
611
|
+
|| ready.ready_ack.run_id !== this.#runId || ready.ready_ack.plan_epoch !== planEpoch
|
|
612
|
+
|| ready.ready_ack.activation_digest !== activationDigest
|
|
613
|
+
|| ready.ready_ack.supervisor_session_nonce_digest !== this.#sessionNonceDigest
|
|
614
|
+
|| JSON.stringify(ready.ready_ack.staged_lease_digests) !== JSON.stringify(ownedStaged)) fail('EPOCH_ACTIVATION_INCOMPLETE', 'ready pointer/binding不一致');
|
|
615
|
+
readyAcks.push(ready.ready_ack);
|
|
616
|
+
}
|
|
617
|
+
if (typeof commitReleaseBarrier === 'function') {
|
|
618
|
+
const barrier = { schema: 'lattice.release_epoch_barrier.v1', run_id: this.#runId,
|
|
619
|
+
committed_epoch_pointer_digest: committedEpochDigest, plan_epoch: planEpoch,
|
|
620
|
+
activation_digest: activationDigest,
|
|
621
|
+
controller_ready_ack_digests: readyAcks.map((ack) => ack.ack_digest).sort(),
|
|
622
|
+
staged_lease_digests: staged.map((entry) => entry.lease.lease_digest).sort(),
|
|
623
|
+
gate_generation: nextGeneration, release_digest: '' };
|
|
624
|
+
barrier.release_digest = selfDigest(barrier, 'release_digest');
|
|
625
|
+
const receipt = await commitReleaseBarrier(structuredClone(barrier));
|
|
626
|
+
if (receipt?.release_digest !== barrier.release_digest) fail('EPOCH_ACTIVATION_INCOMPLETE', 'release barrier commit不一致');
|
|
627
|
+
releaseBarrierDigest = barrier.release_digest;
|
|
628
|
+
}
|
|
629
|
+
if (!digest(releaseBarrierDigest)) fail('EPOCH_ACTIVATION_INCOMPLETE', 'release barrier未commit');
|
|
630
|
+
const armed = staged.map((entry) => armStagedWriteLease(entry.lease, { releaseBarrierDigest, gateGeneration: nextGeneration }));
|
|
631
|
+
const releaseAcks = [];
|
|
632
|
+
for (const record of this.#controllers.values()) {
|
|
633
|
+
const ownedStaged = staged.filter((entry) => entry.lease.controller_registration_digest === record.registration.registration_digest).map((entry) => entry.lease.lease_digest).sort();
|
|
634
|
+
const released = await this.routeWhileFrozen('release', record, {
|
|
635
|
+
request_id: randomUUID(), release_barrier_digest: releaseBarrierDigest,
|
|
636
|
+
activation_digest: activationDigest, gate_generation: nextGeneration,
|
|
637
|
+
staged_lease_digests: ownedStaged,
|
|
638
|
+
});
|
|
639
|
+
const ownedArmed = armed.filter((lease) => lease.controller_registration_digest === record.registration.registration_digest).map((lease) => lease.lease_digest).sort();
|
|
640
|
+
if (released.observed_gate_generation !== nextGeneration
|
|
641
|
+
|| JSON.stringify(released.armed_lease_digests) !== JSON.stringify(ownedArmed)
|
|
642
|
+
|| !validateReleaseAck(released.release_ack)) fail('EPOCH_ACTIVATION_INCOMPLETE', 'direct release ack不一致');
|
|
643
|
+
releaseAcks.push(released.release_ack);
|
|
644
|
+
releaseIndex += 1;
|
|
645
|
+
if (typeof afterControllerRelease === 'function') await afterControllerRelease({
|
|
646
|
+
release_index: releaseIndex, controller_id: record.descriptor.controller_id,
|
|
647
|
+
release_ack_digest: released.release_ack.ack_digest,
|
|
648
|
+
});
|
|
649
|
+
}
|
|
650
|
+
const registrations = [...this.#controllers.values()].map((entry) => entry.registration);
|
|
651
|
+
const controllers = [...this.#controllers.values()].map((entry) => entry.descriptor);
|
|
652
|
+
const gate = createWriteGate({ runId: this.#runId, planEpoch, gateGeneration: nextGeneration, releaseBarrierDigest, releaseAcks, armedLeases: armed, previousGateDigest: this.#gate?.gate_digest ?? null, committedAt });
|
|
653
|
+
const verified = verifyCentralWriteGate({ gate, runId: this.#runId, planEpoch, releaseBarrierDigest, sessionNonceDigest: this.#sessionNonceDigest, registrations, controllers, releaseAcks, armedLeases: armed, previousGate: this.#gate });
|
|
654
|
+
if (!verified.valid) fail('EPOCH_ACTIVATION_INCOMPLETE', verified.reason);
|
|
655
|
+
const atomicActivation = {
|
|
656
|
+
gate: structuredClone(gate),
|
|
657
|
+
control_events: [
|
|
658
|
+
{ kind: 'write_gate_committed', payload: { gate_digest: gate.gate_digest, gate_generation: nextGeneration } },
|
|
659
|
+
{ kind: 'epoch_activated', payload: { plan_epoch: planEpoch, gate_digest: gate.gate_digest } },
|
|
660
|
+
{ kind: 'intake_resumed', payload: { plan_epoch: planEpoch, gate_digest: gate.gate_digest } },
|
|
661
|
+
],
|
|
662
|
+
};
|
|
663
|
+
const commitReceipt = await this.#gateWriter.commit(atomicActivation);
|
|
664
|
+
if (!digest(commitReceipt?.control_head_digest) || commitReceipt.gate_digest !== gate.gate_digest) fail('EPOCH_ACTIVATION_INCOMPLETE', 'gate/event atomic commit証拠不正');
|
|
665
|
+
// Durable central commit成功後にだけlocal projectionをarmedへ置換する。
|
|
666
|
+
for (const armedLease of armed) {
|
|
667
|
+
const old = staged.find((entry) => entry.lease.lease_id === armedLease.lease_id);
|
|
668
|
+
this.#leases.delete(old.lease.lease_digest);
|
|
669
|
+
this.#leases.set(armedLease.lease_digest, { ...old, lease: armedLease, issuedAt: this.#clock() });
|
|
670
|
+
}
|
|
671
|
+
this.#previousGate = this.#gate;
|
|
672
|
+
this.#gate = gate;
|
|
673
|
+
this.#releaseAcks = releaseAcks.map((ack) => structuredClone(ack));
|
|
674
|
+
this.#gateRecords.set(gate.gate_generation, {
|
|
675
|
+
gate: structuredClone(gate),
|
|
676
|
+
releaseAcks: releaseAcks.map((ack) => structuredClone(ack)),
|
|
677
|
+
});
|
|
678
|
+
this.#gateGeneration = nextGeneration;
|
|
679
|
+
this.#frozen = false;
|
|
680
|
+
return { gate: structuredClone(gate), armedLeases: armed.map((lease) => structuredClone(lease)) };
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
/** write/process開始の度にfull chainを再検証する。generationだけのshort pathは持たない。 */
|
|
684
|
+
async authorizeWrite({ leaseDigest }) {
|
|
685
|
+
await this.assertControllerHealth();
|
|
686
|
+
const entry = this.#leases.get(leaseDigest);
|
|
687
|
+
if (this.#frozen || !entry || entry.revoked || !validateArmedWriteLease(entry.lease)) fail('RUN_FROZEN', '有効なarmed leaseなし');
|
|
688
|
+
const gateRecord = this.#gateRecords.get(entry.lease.gate_generation);
|
|
689
|
+
const gate = gateRecord?.gate ?? (this.#gate?.gate_generation === entry.lease.gate_generation
|
|
690
|
+
? this.#gate : null);
|
|
691
|
+
const releaseAcks = gateRecord?.releaseAcks ?? (gate === this.#gate ? this.#releaseAcks : []);
|
|
692
|
+
if (gate === null) fail('RUN_FROZEN', 'leaseのorigin gateが無い');
|
|
693
|
+
const previousGate = this.#gateRecords.get(gate.gate_generation - 1)?.gate
|
|
694
|
+
?? (gate === this.#gate ? this.#previousGate : null);
|
|
695
|
+
const gateLeases = [...this.#leases.values()]
|
|
696
|
+
.filter((item) => !item.revoked && validateArmedWriteLease(item.lease)
|
|
697
|
+
&& item.lease.gate_generation === gate.gate_generation)
|
|
698
|
+
.map((item) => item.lease);
|
|
699
|
+
const verified = verifyCentralWriteGate({
|
|
700
|
+
gate, runId: this.#runId, planEpoch: entry.lease.plan_epoch,
|
|
701
|
+
releaseBarrierDigest: entry.lease.release_barrier_digest, sessionNonceDigest: this.#sessionNonceDigest,
|
|
702
|
+
registrations: [...this.#controllers.values()].map((record) => record.registration),
|
|
703
|
+
controllers: [...this.#controllers.values()].map((record) => record.descriptor),
|
|
704
|
+
releaseAcks, armedLeases: gateLeases,
|
|
705
|
+
previousGate,
|
|
706
|
+
});
|
|
707
|
+
// gate commit時にarmした後続frontierのleaseを壁時計で失効させると、正しく直列待ちした
|
|
708
|
+
// taskほどdispatch不能になる。freshnessはactive gate chainとrevokeで決める。
|
|
709
|
+
if (!verified.valid) {
|
|
710
|
+
entry.revoked = true;
|
|
711
|
+
this.#frozen = true;
|
|
712
|
+
fail('RUN_FROZEN', verified.reason);
|
|
713
|
+
}
|
|
714
|
+
return structuredClone(entry.lease);
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
/** restart後は旧nonce leaseを無効化し、観測集合の和を全件barrierする。 */
|
|
718
|
+
async recoveryBarrier({ barrierId, frozenEventDigest }) {
|
|
719
|
+
this.#frozen = true;
|
|
720
|
+
for (const entry of this.#leases.values()) entry.revoked = true;
|
|
721
|
+
await this.#append('supervisor_recovery_barrier', { barrier_id: barrierId });
|
|
722
|
+
return this.barrierAll({ barrierId, reason: 'supervisor_restart', frozenEventDigest });
|
|
723
|
+
}
|
|
724
|
+
|
|
725
|
+
async routeWhileFrozen(operation, record, fields) {
|
|
726
|
+
const request = createControllerRequest(operation, { request_id: fields.request_id ?? randomUUID(), registration_digest: record.registration.registration_digest, ...Object.fromEntries(Object.entries(fields).filter(([key]) => !['request_id', 'registration_digest'].includes(key))) });
|
|
727
|
+
const response = await record.transport.request(operation, structuredClone(request));
|
|
728
|
+
if (!validateControllerResponse(operation, response, request.request_id)) fail('ADAPTER_CONTROLLER_UNAVAILABLE', `${operation} response不正`);
|
|
729
|
+
return response;
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
#requireController(controllerId) {
|
|
733
|
+
const record = this.#controllers.get(controllerId);
|
|
734
|
+
if (!record) fail('ADAPTER_CONTROLLER_UNAVAILABLE', `unknown controller: ${controllerId}`);
|
|
735
|
+
return record;
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
async #failClosed(record, code, detail) {
|
|
739
|
+
this.#frozen = true;
|
|
740
|
+
record.revoked = true;
|
|
741
|
+
for (const entry of this.#leases.values()) {
|
|
742
|
+
if (entry.controllerId === record.descriptor.controller_id) entry.revoked = true;
|
|
743
|
+
}
|
|
744
|
+
const leaseDigests = [...this.#leases.values()].filter((entry) => entry.controllerId === record.descriptor.controller_id).map((entry) => entry.lease.lease_digest).sort();
|
|
745
|
+
if (record.connected) {
|
|
746
|
+
let response;
|
|
747
|
+
try { response = await this.routeWhileFrozen('revoke', record, { request_id: randomUUID(), reason: detail, lease_digests: leaseDigests }); }
|
|
748
|
+
catch { fail(code, `${detail}; controller revoke未確認`); }
|
|
749
|
+
if (JSON.stringify(response.revoked_lease_digests) !== JSON.stringify(leaseDigests)
|
|
750
|
+
|| response.residual_processes.length !== 0) fail(code, `${detail}; controller revoke不完全`);
|
|
751
|
+
await this.#append('lease_revoked', { controller_id: record.descriptor.controller_id, reason: detail, response_digest: response.response_digest });
|
|
752
|
+
}
|
|
753
|
+
fail(code, detail);
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
async #append(kind, payload) {
|
|
757
|
+
return this.#journal.append({ run_id: this.#runId, kind, session_nonce_digest: this.#sessionNonceDigest, payload });
|
|
758
|
+
}
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
async function exchangeControllerHandshake({ socketPath, runId, supervisorSessionNonce,
|
|
762
|
+
controllerSocketRef, timeoutMs }) {
|
|
763
|
+
const requestId = randomUUID();
|
|
764
|
+
const challenge = randomBytes(32).toString('hex');
|
|
765
|
+
const request = { schema: 'lattice.adapter_controller_handshake_request.v1', request_id: requestId,
|
|
766
|
+
run_id: runId, supervisor_session_nonce: supervisorSessionNonce,
|
|
767
|
+
controller_socket_ref: controllerSocketRef, challenge, request_digest: '' };
|
|
768
|
+
request.request_digest = selfDigest(request, 'request_digest');
|
|
769
|
+
return new Promise((resolve, reject) => {
|
|
770
|
+
const socket = net.createConnection({ path: socketPath });
|
|
771
|
+
let buffer = '';
|
|
772
|
+
const timer = setTimeout(() => { socket.destroy(); reject(new ManagedRuntimeError('ADAPTER_CONTROLLER_UNAVAILABLE', 'controller handshake timeout')); }, timeoutMs);
|
|
773
|
+
const done = (error, value) => { clearTimeout(timer); socket.destroy(); if (error) reject(error); else resolve(value); };
|
|
774
|
+
socket.setEncoding('utf8');
|
|
775
|
+
socket.once('connect', () => socket.write(`${canonicalizeArtifact(request)}\n`));
|
|
776
|
+
socket.on('data', (chunk) => {
|
|
777
|
+
buffer += chunk;
|
|
778
|
+
const newline = buffer.indexOf('\n');
|
|
779
|
+
if (newline < 0) return;
|
|
780
|
+
try {
|
|
781
|
+
if (buffer.slice(newline + 1).length !== 0) throw new Error('multiple documents');
|
|
782
|
+
const response = JSON.parse(buffer.slice(0, newline));
|
|
783
|
+
if (!validateControllerHandshakeResponse(response, { requestId, challenge, runId })) throw new Error('handshake binding不正');
|
|
784
|
+
done(null, response);
|
|
785
|
+
} catch (error) { done(new ManagedRuntimeError('ADAPTER_CONTROLLER_UNAVAILABLE', error.message)); }
|
|
786
|
+
});
|
|
787
|
+
socket.once('error', (error) => done(new ManagedRuntimeError('ADAPTER_CONTROLLER_UNAVAILABLE', error.message)));
|
|
788
|
+
});
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
function createControllerSocketTransport(
|
|
792
|
+
socketPath,
|
|
793
|
+
timeoutMs,
|
|
794
|
+
dispatchLivenessTimeoutMs = timeoutMs,
|
|
795
|
+
) {
|
|
796
|
+
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1
|
|
797
|
+
|| !Number.isSafeInteger(dispatchLivenessTimeoutMs) || dispatchLivenessTimeoutMs < 1) {
|
|
798
|
+
fail('SUPERVISOR_CONFIGURATION_INVALID', 'controller transport timeout不正');
|
|
799
|
+
}
|
|
800
|
+
const socket = net.createConnection({ path: socketPath });
|
|
801
|
+
const pending = new Map();
|
|
802
|
+
let buffer = '';
|
|
803
|
+
let connected = false;
|
|
804
|
+
let heartbeatHandler = null;
|
|
805
|
+
let disconnectHandler = null;
|
|
806
|
+
const ready = new Promise((resolve, reject) => {
|
|
807
|
+
socket.once('connect', () => { connected = true; resolve(); });
|
|
808
|
+
socket.once('error', reject);
|
|
809
|
+
});
|
|
810
|
+
socket.setEncoding('utf8');
|
|
811
|
+
const armRequestTimer = (requestId, entry) => {
|
|
812
|
+
clearTimeout(entry.timer);
|
|
813
|
+
const waitMs = entry.operation === 'dispatch' ? dispatchLivenessTimeoutMs : timeoutMs;
|
|
814
|
+
entry.timer = setTimeout(() => {
|
|
815
|
+
pending.delete(requestId);
|
|
816
|
+
entry.reject(new ManagedRuntimeError(
|
|
817
|
+
'ADAPTER_CONTROLLER_UNAVAILABLE',
|
|
818
|
+
`${entry.operation} timeout`,
|
|
819
|
+
));
|
|
820
|
+
}, waitMs);
|
|
821
|
+
};
|
|
822
|
+
const refreshDispatchLiveness = () => {
|
|
823
|
+
for (const [requestId, entry] of pending.entries()) {
|
|
824
|
+
if (entry.operation === 'dispatch') armRequestTimer(requestId, entry);
|
|
825
|
+
}
|
|
826
|
+
};
|
|
827
|
+
const failPending = (detail) => {
|
|
828
|
+
connected = false;
|
|
829
|
+
for (const entry of pending.values()) { clearTimeout(entry.timer); entry.reject(new ManagedRuntimeError('ADAPTER_CONTROLLER_UNAVAILABLE', detail)); }
|
|
830
|
+
pending.clear();
|
|
831
|
+
Promise.resolve(disconnectHandler?.()).catch(() => {});
|
|
832
|
+
};
|
|
833
|
+
socket.on('close', () => failPending('controller persistent socket disconnected'));
|
|
834
|
+
socket.on('error', (error) => failPending(error.message));
|
|
835
|
+
socket.on('data', (chunk) => {
|
|
836
|
+
buffer += chunk;
|
|
837
|
+
while (buffer.includes('\n')) {
|
|
838
|
+
const newline = buffer.indexOf('\n');
|
|
839
|
+
const line = buffer.slice(0, newline); buffer = buffer.slice(newline + 1);
|
|
840
|
+
let document;
|
|
841
|
+
try { document = JSON.parse(line); } catch { failPending('controller document JSON不正'); socket.destroy(); return; }
|
|
842
|
+
const errorEntry = pending.get(document?.request_id);
|
|
843
|
+
if (errorEntry && validateControllerError(document, document.request_id)) {
|
|
844
|
+
pending.delete(document.request_id);
|
|
845
|
+
clearTimeout(errorEntry.timer);
|
|
846
|
+
const detail = Object.keys(document.detail).length === 0
|
|
847
|
+
? ''
|
|
848
|
+
: `: ${canonicalizeArtifact(document.detail)}`;
|
|
849
|
+
errorEntry.reject(new ManagedRuntimeError(
|
|
850
|
+
'ADAPTER_CONTROLLER_UNAVAILABLE',
|
|
851
|
+
`${document.code}: ${document.message}${detail}`,
|
|
852
|
+
));
|
|
853
|
+
socket.destroy();
|
|
854
|
+
return;
|
|
855
|
+
}
|
|
856
|
+
if (validateControllerHeartbeat(document)) {
|
|
857
|
+
if (heartbeatHandler === null) {
|
|
858
|
+
socket.destroy();
|
|
859
|
+
continue;
|
|
860
|
+
}
|
|
861
|
+
Promise.resolve()
|
|
862
|
+
.then(() => heartbeatHandler(structuredClone(document)))
|
|
863
|
+
.then(() => refreshDispatchLiveness())
|
|
864
|
+
.catch(() => socket.destroy());
|
|
865
|
+
continue;
|
|
866
|
+
}
|
|
867
|
+
const entry = pending.get(document.request_id);
|
|
868
|
+
if (!entry || !validateControllerResponse(entry.operation, document, document.request_id)) { failPending('unsolicited/invalid controller response'); socket.destroy(); return; }
|
|
869
|
+
pending.delete(document.request_id); clearTimeout(entry.timer); entry.resolve(document);
|
|
870
|
+
}
|
|
871
|
+
});
|
|
872
|
+
return Object.freeze({
|
|
873
|
+
async ensureConnected() { await ready; if (!connected) fail('ADAPTER_CONTROLLER_UNAVAILABLE', 'persistent controller socket不達'); },
|
|
874
|
+
setHeartbeatHandler(handler) { heartbeatHandler = handler; },
|
|
875
|
+
setDisconnectHandler(handler) { disconnectHandler = handler; },
|
|
876
|
+
close() { disconnectHandler = null; socket.destroy(); },
|
|
877
|
+
async request(operation, request) {
|
|
878
|
+
await ready;
|
|
879
|
+
if (!connected) fail('ADAPTER_CONTROLLER_UNAVAILABLE', 'controller persistent socket不達');
|
|
880
|
+
return new Promise((resolve, reject) => {
|
|
881
|
+
const entry = { operation, resolve, reject, timer: null };
|
|
882
|
+
pending.set(request.request_id, entry);
|
|
883
|
+
armRequestTimer(request.request_id, entry);
|
|
884
|
+
socket.write(`${canonicalizeArtifact(request)}\n`);
|
|
885
|
+
});
|
|
886
|
+
},
|
|
887
|
+
});
|
|
888
|
+
}
|
|
889
|
+
|
|
890
|
+
export const runtimeManagedSupervisorInternal = Object.freeze({
|
|
891
|
+
createControllerSocketTransport,
|
|
892
|
+
});
|
|
893
|
+
|
|
894
|
+
/**
|
|
895
|
+
* durable registryをpreflightし、実controller hostとのnonce challenge後にだけactivation証拠を返す。
|
|
896
|
+
* callerは成功返却前にrun storeへ何もpublishしてはならない。
|
|
897
|
+
*/
|
|
898
|
+
async function activateManagedSupervisorController({ repoRoot, runDir, runId, adapterKind, supervisorSessionNonce, timeoutMs = 5_000, binaryIdentityObserver = observeMacosBinaryIdentity }) {
|
|
899
|
+
if (!path.isAbsolute(repoRoot) || !path.isAbsolute(runDir) || !identifier(runId) || !identifier(adapterKind)
|
|
900
|
+
|| typeof supervisorSessionNonce !== 'string' || supervisorSessionNonce.length < 32) fail('SUPERVISOR_CONFIGURATION_INVALID', 'activation入力不正');
|
|
901
|
+
const canonicalRepo = await realpath(repoRoot);
|
|
902
|
+
const registryPath = path.join(canonicalRepo, '.lattice/runtime/adapter-registry/registry.json');
|
|
903
|
+
let registryArtifact;
|
|
904
|
+
try { registryArtifact = await readCanonicalRegular(registryPath); } catch (error) {
|
|
905
|
+
if (error?.code === 'ENOENT') fail('ADAPTER_NOT_REGISTERED', adapterKind);
|
|
906
|
+
throw error;
|
|
907
|
+
}
|
|
908
|
+
if (!validateAdapterRegistry(registryArtifact.value)) fail('ADAPTER_LAUNCH_INVALID', 'registry不正');
|
|
909
|
+
const entry = registryArtifact.value.entries.find((candidate) => candidate.adapter_kind === adapterKind);
|
|
910
|
+
if (!entry) fail('ADAPTER_NOT_REGISTERED', adapterKind);
|
|
911
|
+
const descriptorPath = path.join(canonicalRepo, entry.launch_descriptor_ref);
|
|
912
|
+
if (!descriptorPath.startsWith(`${canonicalRepo}${path.sep}`)) fail('ADAPTER_LAUNCH_INVALID', 'descriptor repo外');
|
|
913
|
+
const launchArtifact = await readCanonicalRegular(descriptorPath);
|
|
914
|
+
const launch = launchArtifact.value;
|
|
915
|
+
if (!validateAdapterLaunchDescriptor(launch) || launch.adapter_kind !== adapterKind
|
|
916
|
+
|| launch.descriptor_digest !== entry.launch_descriptor_digest) fail('ADAPTER_LAUNCH_INVALID', 'launch descriptor binding不正');
|
|
917
|
+
const supervisorDir = path.join(runDir, 'supervisor');
|
|
918
|
+
const controllerDir = path.join(supervisorDir, 'controllers');
|
|
919
|
+
const controllerId = `${adapterKind}-${randomUUID()}`;
|
|
920
|
+
const controllerSocketRef = `supervisor/controllers/${controllerId}.sock`;
|
|
921
|
+
const controllerSocketPath = path.join(runDir, controllerSocketRef);
|
|
922
|
+
const supervisorSocketRef = 'supervisor/control.sock';
|
|
923
|
+
let child = null;
|
|
924
|
+
let childStderr = '';
|
|
925
|
+
try {
|
|
926
|
+
await mkdir(controllerDir, { recursive: true, mode: 0o700 });
|
|
927
|
+
let handshakeSocket = launch.endpoint;
|
|
928
|
+
let handshakeConnectPath = launch.endpoint;
|
|
929
|
+
let endpointOwnerPids = null;
|
|
930
|
+
if (launch.launch_kind === 'host_binary') {
|
|
931
|
+
const binaryInfo = await lstat(launch.binary_path);
|
|
932
|
+
if (!binaryInfo.isFile() || binaryInfo.isSymbolicLink() || (binaryInfo.mode & 0o111) === 0) fail('ADAPTER_LAUNCH_INVALID', 'binaryが実行可能regular fileでない');
|
|
933
|
+
const binaryReal = await realpath(launch.binary_path);
|
|
934
|
+
if (binaryReal !== launch.binary_path || sha256Bytes(await readFile(binaryReal)) !== launch.binary_digest) fail('ADAPTER_BINARY_IDENTITY_MISMATCH', 'binary pre-exec identity不一致');
|
|
935
|
+
if (launch.binary_identity !== null) {
|
|
936
|
+
if (typeof binaryIdentityObserver !== 'function'
|
|
937
|
+
|| canonicalizeArtifact(await binaryIdentityObserver(binaryReal)) !== canonicalizeArtifact(launch.binary_identity)) fail('ADAPTER_BINARY_IDENTITY_MISMATCH', 'macOS identity不一致');
|
|
938
|
+
}
|
|
939
|
+
const configPath = path.resolve(canonicalRepo, launch.config_ref);
|
|
940
|
+
if (!configPath.startsWith(`${canonicalRepo}${path.sep}`)) fail('ADAPTER_LAUNCH_INVALID', 'config repo外');
|
|
941
|
+
const config = await lstat(configPath);
|
|
942
|
+
if (!config.isFile() || config.isSymbolicLink() || sha256Bytes(await readFile(configPath)) !== launch.config_digest) fail('ADAPTER_LAUNCH_INVALID', 'config digest不一致');
|
|
943
|
+
const bootstrap = createControllerBootstrap({ requestId: randomUUID(), runId, controllerSocketRef, supervisorSocketRef, supervisorSessionNonce });
|
|
944
|
+
// controller hostのcwdはrun store。bootstrapの固定relative socket refを任意absolute pathへ拡張しない。
|
|
945
|
+
child = spawn(binaryReal, launch.argv, {
|
|
946
|
+
cwd: runDir,
|
|
947
|
+
detached: true,
|
|
948
|
+
stdio: ['ignore', 'ignore', 'pipe', 'pipe'],
|
|
949
|
+
});
|
|
950
|
+
child.stderr.setEncoding('utf8');
|
|
951
|
+
child.stderr.on('data', (chunk) => {
|
|
952
|
+
childStderr = `${childStderr}${chunk}`.slice(-8_192);
|
|
953
|
+
});
|
|
954
|
+
child.stdio[3].write(`${canonicalizeArtifact(bootstrap)}\n`);
|
|
955
|
+
child.stdio[3].end();
|
|
956
|
+
handshakeSocket = controllerSocketPath;
|
|
957
|
+
handshakeConnectPath = controllerSocketRef;
|
|
958
|
+
const deadline = Date.now() + timeoutMs;
|
|
959
|
+
while (Date.now() < deadline) {
|
|
960
|
+
try { if ((await lstat(handshakeSocket)).isSocket()) break; } catch (error) { if (error?.code !== 'ENOENT') throw error; }
|
|
961
|
+
if (child.exitCode !== null) {
|
|
962
|
+
fail(
|
|
963
|
+
'ADAPTER_CONTROLLER_UNAVAILABLE',
|
|
964
|
+
`controller exited: ${child.exitCode}${childStderr ? `: ${childStderr.trim()}` : ''}`,
|
|
965
|
+
);
|
|
966
|
+
}
|
|
967
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
968
|
+
}
|
|
969
|
+
let socketInfo;
|
|
970
|
+
try {
|
|
971
|
+
socketInfo = await lstat(handshakeSocket);
|
|
972
|
+
} catch {
|
|
973
|
+
fail(
|
|
974
|
+
'ADAPTER_CONTROLLER_UNAVAILABLE',
|
|
975
|
+
`controller socket未生成${childStderr ? `: ${childStderr.trim()}` : ''}`,
|
|
976
|
+
);
|
|
977
|
+
}
|
|
978
|
+
if (!socketInfo.isSocket()) fail('ADAPTER_CONTROLLER_UNAVAILABLE', 'controller endpointがsocketでない');
|
|
979
|
+
// exec後にも同じ実行image bytesを再検証する。PID生存も同時に要求する。
|
|
980
|
+
try { process.kill(child.pid, 0); } catch { fail('ADAPTER_CONTROLLER_UNAVAILABLE', 'controller process不達'); }
|
|
981
|
+
let executedImage;
|
|
982
|
+
try {
|
|
983
|
+
executedImage = await observeProcessExecutablePath(child.pid);
|
|
984
|
+
} catch { fail('ADAPTER_BINARY_IDENTITY_MISMATCH', 'exec後image path観測失敗'); }
|
|
985
|
+
if (executedImage !== binaryReal || sha256Bytes(await readFile(executedImage)) !== launch.binary_digest) {
|
|
986
|
+
fail('ADAPTER_BINARY_IDENTITY_MISMATCH', 'exec後image path/bytes不一致');
|
|
987
|
+
}
|
|
988
|
+
if (launch.binary_identity !== null
|
|
989
|
+
&& canonicalizeArtifact(await binaryIdentityObserver(executedImage)) !== canonicalizeArtifact(launch.binary_identity)) {
|
|
990
|
+
fail('ADAPTER_BINARY_IDENTITY_MISMATCH', 'exec後macOS identity不一致');
|
|
991
|
+
}
|
|
992
|
+
} else {
|
|
993
|
+
if (!path.isAbsolute(handshakeSocket) || await realpath(path.dirname(handshakeSocket)) !== path.dirname(handshakeSocket)) {
|
|
994
|
+
fail('ADAPTER_LAUNCH_INVALID', 'existing endpoint pathがcanonical absoluteでない');
|
|
995
|
+
}
|
|
996
|
+
const endpointInfo = await lstat(handshakeSocket);
|
|
997
|
+
if (!endpointInfo.isSocket() || endpointInfo.isSymbolicLink()) fail('ADAPTER_LAUNCH_INVALID', 'existing endpointがsocketでない');
|
|
998
|
+
let ownerPids;
|
|
999
|
+
try { ownerPids = await pidsOwningSocketPath(handshakeSocket); }
|
|
1000
|
+
catch { fail('ADAPTER_LAUNCH_INVALID', 'existing endpoint ownerを観測できない'); }
|
|
1001
|
+
endpointOwnerPids = new Set(ownerPids);
|
|
1002
|
+
if (endpointOwnerPids.size !== 1 || [...endpointOwnerPids].some((pid) => !Number.isSafeInteger(pid) || pid < 1)) {
|
|
1003
|
+
fail('ADAPTER_LAUNCH_INVALID', 'existing endpoint ownerが一意でない');
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
const handshake = await exchangeControllerHandshake({ socketPath: handshakeConnectPath, runId,
|
|
1007
|
+
supervisorSessionNonce, controllerSocketRef, timeoutMs });
|
|
1008
|
+
const controllerDescriptor = handshake.descriptor;
|
|
1009
|
+
if (controllerDescriptor.adapter_kind !== adapterKind || controllerDescriptor.socket_ref !== controllerSocketRef
|
|
1010
|
+
|| controllerDescriptor.capabilities.capabilities_digest !== launch.capabilities_digest
|
|
1011
|
+
|| (child !== null && controllerDescriptor.pid !== child.pid)) fail('ADAPTER_CONTROLLER_UNAVAILABLE', 'controller descriptor launch binding不一致');
|
|
1012
|
+
if (endpointOwnerPids !== null) {
|
|
1013
|
+
if (!endpointOwnerPids.has(controllerDescriptor.pid)) {
|
|
1014
|
+
fail('ADAPTER_CONTROLLER_UNAVAILABLE', 'existing endpoint inode ownerとdescriptor PID不一致');
|
|
1015
|
+
}
|
|
1016
|
+
const deadline = Date.now() + timeoutMs;
|
|
1017
|
+
while (Date.now() < deadline) {
|
|
1018
|
+
try { if ((await lstat(controllerSocketPath)).isSocket()) break; } catch (error) { if (error?.code !== 'ENOENT') throw error; }
|
|
1019
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
1020
|
+
}
|
|
1021
|
+
let controllerOwnerPids;
|
|
1022
|
+
try { controllerOwnerPids = await pidsOwningSocketPath(controllerSocketPath); }
|
|
1023
|
+
catch { fail('ADAPTER_CONTROLLER_UNAVAILABLE', 'existing controller socket ownerを観測できない'); }
|
|
1024
|
+
const persistentOwners = new Set(controllerOwnerPids);
|
|
1025
|
+
if (persistentOwners.size !== 1 || !persistentOwners.has(controllerDescriptor.pid)) {
|
|
1026
|
+
fail('ADAPTER_CONTROLLER_UNAVAILABLE', 'persistent controller socket ownerとdescriptor PID不一致');
|
|
1027
|
+
}
|
|
1028
|
+
handshakeConnectPath = controllerSocketRef;
|
|
1029
|
+
}
|
|
1030
|
+
const observedControllerIdentity = await observeManagedProcessStartIdentity(controllerDescriptor.pid);
|
|
1031
|
+
if (canonicalizeArtifact(observedControllerIdentity) !== canonicalizeArtifact(controllerDescriptor.process_start_identity)) fail('ADAPTER_CONTROLLER_UNAVAILABLE', `controller process start identity不一致: expected=${controllerDescriptor.process_start_identity.identity_digest} observed=${observedControllerIdentity.identity_digest}`);
|
|
1032
|
+
const sessionNonceDigest = digestArtifact(supervisorSessionNonce);
|
|
1033
|
+
const registration = {
|
|
1034
|
+
schema: 'lattice.runtime_adapter_registration.v1', registration_id: `registration-${controllerId}`,
|
|
1035
|
+
run_id: runId, supervisor_session_nonce_digest: sessionNonceDigest,
|
|
1036
|
+
controller_descriptor_digest: controllerDescriptor.descriptor_digest,
|
|
1037
|
+
registered_operations: [...controllerDescriptor.capabilities.operations],
|
|
1038
|
+
registered_at: new Date().toISOString(), registration_digest: '',
|
|
1039
|
+
};
|
|
1040
|
+
registration.registration_digest = selfDigest(registration, 'registration_digest');
|
|
1041
|
+
if (!validateControllerRegistration(registration)) fail('ADAPTER_CONTROLLER_UNAVAILABLE', 'registration生成不正');
|
|
1042
|
+
const processIdentity = await observeManagedProcessStartIdentity(process.pid);
|
|
1043
|
+
const supervisorDescriptor = { schema: 'lattice.runtime_supervisor_descriptor.v1', run_id: runId, pid: process.pid, process_start_identity: processIdentity, socket_ref: supervisorSocketRef, session_nonce_digest: sessionNonceDigest, protocol_version: 'v1', activated_at: new Date().toISOString(), descriptor_digest: '' };
|
|
1044
|
+
supervisorDescriptor.descriptor_digest = selfDigest(supervisorDescriptor, 'descriptor_digest');
|
|
1045
|
+
const activationControlEvent = { schema: 'lattice.runtime_control_event.v1', run_id: runId, sequence: 1, previous_digest: null, kind: 'supervisor_activated', session_nonce_digest: sessionNonceDigest, payload: { supervisor_descriptor_digest: supervisorDescriptor.descriptor_digest, controller_descriptor_digest: controllerDescriptor.descriptor_digest, registration_digest: registration.registration_digest }, recorded_at: supervisorDescriptor.activated_at, event_digest: '' };
|
|
1046
|
+
activationControlEvent.event_digest = selfDigest(activationControlEvent, 'event_digest');
|
|
1047
|
+
const controllerTransport = createControllerSocketTransport(
|
|
1048
|
+
handshakeConnectPath,
|
|
1049
|
+
timeoutMs,
|
|
1050
|
+
controllerDescriptor.heartbeat.ttl_ms,
|
|
1051
|
+
);
|
|
1052
|
+
let disposed = false;
|
|
1053
|
+
const disposeController = async () => {
|
|
1054
|
+
if (disposed) return;
|
|
1055
|
+
disposed = true;
|
|
1056
|
+
controllerTransport.close();
|
|
1057
|
+
if (child?.pid) {
|
|
1058
|
+
try { process.kill(-child.pid, 'SIGTERM'); } catch { /* already exited */ }
|
|
1059
|
+
const deadline = Date.now() + 2_000;
|
|
1060
|
+
while (Date.now() < deadline) {
|
|
1061
|
+
try { process.kill(child.pid, 0); } catch { break; }
|
|
1062
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
1063
|
+
}
|
|
1064
|
+
try { process.kill(child.pid, 0); process.kill(-child.pid, 'SIGKILL'); } catch { /* reaped */ }
|
|
1065
|
+
}
|
|
1066
|
+
await rm(controllerSocketPath, { force: true }).catch(() => {});
|
|
1067
|
+
};
|
|
1068
|
+
const registerWithManagedSupervisor = async (managedSupervisor) => {
|
|
1069
|
+
await controllerTransport.ensureConnected();
|
|
1070
|
+
await managedSupervisor.registerController({ descriptor: controllerDescriptor, registration, transport: controllerTransport });
|
|
1071
|
+
controllerTransport.setHeartbeatHandler(async (heartbeat) => {
|
|
1072
|
+
if (digestArtifact(heartbeat.supervisor_session_nonce) !== sessionNonceDigest
|
|
1073
|
+
|| heartbeat.controller_id !== controllerDescriptor.controller_id) {
|
|
1074
|
+
await managedSupervisor.disconnect(controllerDescriptor.controller_id);
|
|
1075
|
+
return;
|
|
1076
|
+
}
|
|
1077
|
+
await managedSupervisor.heartbeat({
|
|
1078
|
+
controllerId: heartbeat.controller_id,
|
|
1079
|
+
registrationDigest: heartbeat.registration_digest,
|
|
1080
|
+
sequence: heartbeat.sequence,
|
|
1081
|
+
leaseSetDigest: heartbeat.lease_set_digest,
|
|
1082
|
+
sessionNonceDigest,
|
|
1083
|
+
});
|
|
1084
|
+
});
|
|
1085
|
+
controllerTransport.setDisconnectHandler(() => managedSupervisor.disconnect(controllerDescriptor.controller_id));
|
|
1086
|
+
};
|
|
1087
|
+
const createManagedSupervisor = async ({ clock, resolveObservationBinding, resolveRunningBindings, processObserver = null, allowTestObserver = false, journal, gateWriter }) => {
|
|
1088
|
+
if (processObserver !== null && allowTestObserver !== true) fail('SUPERVISOR_CONFIGURATION_INVALID', 'production observer差替え禁止');
|
|
1089
|
+
const directObserver = processObserver ?? createDirectOsProcessObserverV2({ resolveObservationBinding });
|
|
1090
|
+
const managedSupervisor = new RuntimeManagedSupervisor({ runId, sessionNonceDigest, clock, processObserver: directObserver, runningBindingResolver: resolveRunningBindings, journal, gateWriter, shutdownFinalizer: disposeController });
|
|
1091
|
+
await managedSupervisor.hydrateCommittedGate();
|
|
1092
|
+
await registerWithManagedSupervisor(managedSupervisor);
|
|
1093
|
+
return managedSupervisor;
|
|
1094
|
+
};
|
|
1095
|
+
return {
|
|
1096
|
+
supervisorDescriptor,
|
|
1097
|
+
activationControlEvent,
|
|
1098
|
+
controllerDescriptor,
|
|
1099
|
+
registration,
|
|
1100
|
+
launchDescriptor: structuredClone(launch),
|
|
1101
|
+
sessionNonce: supervisorSessionNonce,
|
|
1102
|
+
childPid: child?.pid ?? controllerDescriptor.pid,
|
|
1103
|
+
createManagedSupervisor,
|
|
1104
|
+
registerWithManagedSupervisor,
|
|
1105
|
+
disposeController,
|
|
1106
|
+
};
|
|
1107
|
+
} catch (error) {
|
|
1108
|
+
if (child?.pid) { try { process.kill(child.pid, 'SIGTERM'); } catch { /* already exited */ } }
|
|
1109
|
+
await rm(controllerSocketPath, { force: true }).catch(() => {});
|
|
1110
|
+
throw error;
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
1113
|
+
|
|
1114
|
+
/** CLI→durable supervisorのUnix socket client。nonceはrequest内だけ、stdoutへ返さない。 */
|
|
1115
|
+
export async function sendRuntimeControlRequest({ socketPath, request, timeoutMs = 5_000 }) {
|
|
1116
|
+
if (typeof socketPath !== 'string' || socketPath.length === 0 || !validateRuntimeControlRequest(request)
|
|
1117
|
+
|| !Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 60_000) fail('RUN_NOT_MANAGED', 'control request不正');
|
|
1118
|
+
const absoluteSocket = path.resolve(socketPath);
|
|
1119
|
+
const runDir = path.dirname(path.dirname(absoluteSocket));
|
|
1120
|
+
const socketRef = path.relative(runDir, absoluteSocket);
|
|
1121
|
+
const runInfo = await lstat(runDir).catch(() => null);
|
|
1122
|
+
const supervisorInfo = await lstat(path.join(runDir, 'supervisor')).catch(() => null);
|
|
1123
|
+
const socketInfo = await lstat(absoluteSocket).catch(() => null);
|
|
1124
|
+
if (socketRef !== 'supervisor/control.sock' || runInfo === null || !runInfo.isDirectory() || runInfo.isSymbolicLink()
|
|
1125
|
+
|| supervisorInfo === null || !supervisorInfo.isDirectory() || supervisorInfo.isSymbolicLink()
|
|
1126
|
+
|| socketInfo === null || !socketInfo.isSocket() || socketInfo.isSymbolicLink()
|
|
1127
|
+
|| await realpath(runDir) !== runDir) fail('RUN_NOT_MANAGED', 'control socket cwd anchor不正');
|
|
1128
|
+
const activePaths = await resolveActiveRuntimePaths({ runDir });
|
|
1129
|
+
const descriptorArtifact = await readCanonicalRegular(activePaths.descriptorPath).catch(() => null);
|
|
1130
|
+
const descriptor = descriptorArtifact?.value;
|
|
1131
|
+
const descriptorFields = ['schema', 'run_id', 'pid', 'process_start_identity', 'socket_ref', 'session_nonce_digest', 'protocol_version', 'activated_at', 'descriptor_digest'];
|
|
1132
|
+
if (descriptor === undefined || descriptor.schema !== 'lattice.runtime_supervisor_descriptor.v1'
|
|
1133
|
+
|| Object.keys(descriptor).sort().join('\0') !== descriptorFields.sort().join('\0')
|
|
1134
|
+
|| descriptor.run_id !== request.run_id || descriptor.socket_ref !== socketRef
|
|
1135
|
+
|| descriptor.session_nonce_digest !== digestArtifact(request.session_nonce)
|
|
1136
|
+
|| (activePaths.pointer !== null
|
|
1137
|
+
&& (descriptor.descriptor_digest !== activePaths.pointer.descriptor_digest
|
|
1138
|
+
|| descriptor.session_nonce_digest !== activePaths.pointer.session_nonce_digest))
|
|
1139
|
+
|| descriptor.descriptor_digest !== selfDigest(descriptor, 'descriptor_digest')) fail('RUN_NOT_MANAGED', 'supervisor descriptor/session binding不正');
|
|
1140
|
+
let observedSupervisor;
|
|
1141
|
+
try { observedSupervisor = await observeManagedProcessStartIdentity(descriptor.pid); }
|
|
1142
|
+
catch { fail('RUN_NOT_MANAGED', 'supervisor process不在'); }
|
|
1143
|
+
if (canonicalizeArtifact(observedSupervisor) !== canonicalizeArtifact(descriptor.process_start_identity)) fail('RUN_NOT_MANAGED', 'supervisor PID/start identity不一致');
|
|
1144
|
+
let ownedSocketPaths;
|
|
1145
|
+
try { ownedSocketPaths = await socketPathsOwnedByPid(descriptor.pid); }
|
|
1146
|
+
catch { fail('RUN_NOT_MANAGED', 'supervisor socket owner観測失敗'); }
|
|
1147
|
+
const ownsExactSocket = ownedSocketPaths
|
|
1148
|
+
.some((name) => name === absoluteSocket || name === socketRef);
|
|
1149
|
+
if (!ownsExactSocket) fail('RUN_NOT_MANAGED', 'supervisor PIDがcontrol socket inodeを所有しない');
|
|
1150
|
+
return new Promise((resolve, reject) => {
|
|
1151
|
+
let originalCwd;
|
|
1152
|
+
let socket;
|
|
1153
|
+
try {
|
|
1154
|
+
originalCwd = process.cwd();
|
|
1155
|
+
process.chdir(runDir);
|
|
1156
|
+
if (process.cwd() !== runDir) throw new Error('run cwd identity不一致');
|
|
1157
|
+
socket = net.createConnection({ path: socketRef });
|
|
1158
|
+
} catch (error) {
|
|
1159
|
+
if (originalCwd) process.chdir(originalCwd);
|
|
1160
|
+
return reject(new ManagedRuntimeError('RUN_NOT_MANAGED', error.message));
|
|
1161
|
+
}
|
|
1162
|
+
let buffer = '';
|
|
1163
|
+
let settled = false;
|
|
1164
|
+
let sent = false;
|
|
1165
|
+
let cwdRestored = false;
|
|
1166
|
+
const restoreCwd = () => { if (!cwdRestored) { cwdRestored = true; process.chdir(originalCwd); } };
|
|
1167
|
+
const finish = (error, value) => {
|
|
1168
|
+
if (settled) return;
|
|
1169
|
+
settled = true;
|
|
1170
|
+
restoreCwd();
|
|
1171
|
+
socket.destroy();
|
|
1172
|
+
if (error) reject(error); else resolve(value);
|
|
1173
|
+
};
|
|
1174
|
+
socket.setEncoding('utf8');
|
|
1175
|
+
socket.setTimeout(timeoutMs, () => finish(new ManagedRuntimeError('RUN_OUTCOME_UNKNOWN',
|
|
1176
|
+
'control timeout。同一request_idで再照会が必要')));
|
|
1177
|
+
socket.on('connect', () => { restoreCwd(); sent = true; socket.write(`${canonicalizeArtifact(request)}\n`); });
|
|
1178
|
+
socket.on('data', (chunk) => {
|
|
1179
|
+
buffer += chunk;
|
|
1180
|
+
if (Buffer.byteLength(buffer, 'utf8') > 1_048_576) return finish(new ManagedRuntimeError('RUN_NOT_MANAGED', 'control response過大'));
|
|
1181
|
+
const newline = buffer.indexOf('\n');
|
|
1182
|
+
if (newline < 0) return;
|
|
1183
|
+
if (buffer.slice(newline + 1).length !== 0) return finish(new ManagedRuntimeError('RUN_NOT_MANAGED', 'control response複数document'));
|
|
1184
|
+
try {
|
|
1185
|
+
const value = JSON.parse(buffer.slice(0, newline));
|
|
1186
|
+
if (!validateRuntimeControlResponse(value, request.operation) || value.request_id !== request.request_id || value.run_id !== request.run_id) return finish(new ManagedRuntimeError('RUN_NOT_MANAGED', 'control response binding不正'));
|
|
1187
|
+
finish(null, value);
|
|
1188
|
+
} catch (error) { finish(error instanceof ManagedRuntimeError ? error : new ManagedRuntimeError('RUN_NOT_MANAGED', 'control response JSON不正')); }
|
|
1189
|
+
});
|
|
1190
|
+
socket.on('error', (error) => finish(new ManagedRuntimeError(sent ? 'RUN_OUTCOME_UNKNOWN' : 'RUN_NOT_MANAGED', error.message)));
|
|
1191
|
+
socket.on('end', () => { if (!settled) finish(new ManagedRuntimeError(sent ? 'RUN_OUTCOME_UNKNOWN' : 'RUN_NOT_MANAGED', 'control socket早期切断')); });
|
|
1192
|
+
});
|
|
1193
|
+
}
|
|
1194
|
+
|
|
1195
|
+
/** 初回activate専用。descriptor未公開のためIPCで得たPID+nonceとsocket ownerを直接bindする。 */
|
|
1196
|
+
export async function sendRuntimeActivationRequest({ socketPath, request, expectedPid, expectedProcessStartIdentity, timeoutMs = 5_000 }) {
|
|
1197
|
+
if (!validateRuntimeControlRequest(request) || request.operation !== 'activate'
|
|
1198
|
+
|| !Number.isSafeInteger(expectedPid) || expectedPid < 1
|
|
1199
|
+
|| !validateProcessStartIdentity(expectedProcessStartIdentity)
|
|
1200
|
+
|| expectedProcessStartIdentity.pid !== expectedPid) fail('RUN_NOT_MANAGED', 'activation bootstrap入力不正');
|
|
1201
|
+
const absoluteSocket = path.resolve(socketPath);
|
|
1202
|
+
const runDir = path.dirname(path.dirname(absoluteSocket));
|
|
1203
|
+
const socketRef = path.relative(runDir, absoluteSocket);
|
|
1204
|
+
const socketInfo = await lstat(absoluteSocket).catch(() => null);
|
|
1205
|
+
if (socketRef !== 'supervisor/control.sock' || socketInfo === null || !socketInfo.isSocket() || socketInfo.isSymbolicLink()
|
|
1206
|
+
|| await realpath(runDir) !== runDir) fail('RUN_NOT_MANAGED', 'activation socket anchor不正');
|
|
1207
|
+
let observedBootstrapIdentity;
|
|
1208
|
+
try { observedBootstrapIdentity = await observeManagedProcessStartIdentity(expectedPid); }
|
|
1209
|
+
catch { fail('RUN_NOT_MANAGED', 'activation bootstrap process不在'); }
|
|
1210
|
+
if (canonicalizeArtifact(observedBootstrapIdentity) !== canonicalizeArtifact(expectedProcessStartIdentity)) fail('RUN_NOT_MANAGED', 'activation bootstrap PID/start identity不一致');
|
|
1211
|
+
let bootstrapSocketPaths;
|
|
1212
|
+
try { bootstrapSocketPaths = await socketPathsOwnedByPid(expectedPid); }
|
|
1213
|
+
catch { fail('RUN_NOT_MANAGED', 'activation socket owner観測失敗'); }
|
|
1214
|
+
if (!bootstrapSocketPaths
|
|
1215
|
+
.some((name) => name === absoluteSocket || name === socketRef)) fail('RUN_NOT_MANAGED', 'bootstrap PIDがcontrol socketを所有しない');
|
|
1216
|
+
const originalCwd = process.cwd();
|
|
1217
|
+
process.chdir(runDir);
|
|
1218
|
+
let socket;
|
|
1219
|
+
try { socket = net.createConnection({ path: socketRef }); } catch (error) { process.chdir(originalCwd); fail('RUN_NOT_MANAGED', error.message); }
|
|
1220
|
+
return new Promise((resolve, reject) => {
|
|
1221
|
+
let restored = false; const restore = () => { if (!restored) { restored = true; process.chdir(originalCwd); } };
|
|
1222
|
+
let buffer = ''; let settled = false; let sent = false;
|
|
1223
|
+
const finish = (error, value) => { if (settled) return; settled = true; restore(); socket.destroy(); if (error) reject(error); else resolve(value); };
|
|
1224
|
+
socket.setEncoding('utf8');
|
|
1225
|
+
socket.setTimeout(timeoutMs, () => finish(new ManagedRuntimeError('RUN_OUTCOME_UNKNOWN',
|
|
1226
|
+
'activation timeout。同一request_idで再照会が必要')));
|
|
1227
|
+
socket.on('connect', () => { restore(); sent = true; socket.write(`${canonicalizeArtifact(request)}\n`); });
|
|
1228
|
+
socket.on('data', (chunk) => {
|
|
1229
|
+
buffer += chunk; const newline = buffer.indexOf('\n'); if (newline < 0) return;
|
|
1230
|
+
try {
|
|
1231
|
+
const response = JSON.parse(buffer.slice(0, newline));
|
|
1232
|
+
if (buffer.slice(newline + 1).length !== 0 || !validateRuntimeControlResponse(response, request.operation)
|
|
1233
|
+
|| response.request_id !== request.request_id || response.run_id !== request.run_id) throw new Error('activation response binding不正');
|
|
1234
|
+
finish(null, response);
|
|
1235
|
+
} catch (error) { finish(new ManagedRuntimeError('RUN_NOT_MANAGED', error.message)); }
|
|
1236
|
+
});
|
|
1237
|
+
socket.on('error', (error) => finish(new ManagedRuntimeError(sent ? 'RUN_OUTCOME_UNKNOWN' : 'RUN_NOT_MANAGED', error.message)));
|
|
1238
|
+
socket.on('end', () => { if (!settled) finish(new ManagedRuntimeError(sent ? 'RUN_OUTCOME_UNKNOWN' : 'RUN_NOT_MANAGED', 'activation socket早期切断')); });
|
|
1239
|
+
});
|
|
1240
|
+
}
|
|
1241
|
+
|
|
1242
|
+
/** 外部workerの完了を含む初回activationは、同じbootstrap接続で最終結果まで待つ。 */
|
|
1243
|
+
export function sendRuntimeActivationRequestUntilSettled(args) {
|
|
1244
|
+
return sendRuntimeActivationRequest({ ...args, timeoutMs: 0 });
|
|
1245
|
+
}
|
|
1246
|
+
|
|
1247
|
+
/** stopped/crashed supervisorの同一run明示再起動前処理。live/foreign socketは削除しない。 */
|
|
1248
|
+
export async function prepareManagedSupervisorRestart({ runDir }) {
|
|
1249
|
+
if (!path.isAbsolute(runDir) || await realpath(runDir) !== runDir) fail('RUN_NOT_MANAGED', 'restart runDir不正');
|
|
1250
|
+
const activePaths = await resolveActiveRuntimePaths({ runDir });
|
|
1251
|
+
const descriptorArtifact = await readCanonicalRegular(activePaths.descriptorPath).catch(() => null);
|
|
1252
|
+
const descriptor = descriptorArtifact?.value;
|
|
1253
|
+
if (descriptor?.schema !== 'lattice.runtime_supervisor_descriptor.v1'
|
|
1254
|
+
|| descriptor.descriptor_digest !== selfDigest(descriptor, 'descriptor_digest')
|
|
1255
|
+
|| !validateProcessStartIdentity(descriptor.process_start_identity)
|
|
1256
|
+
|| descriptor.process_start_identity.pid !== descriptor.pid
|
|
1257
|
+
|| (activePaths.pointer !== null
|
|
1258
|
+
&& (descriptor.descriptor_digest !== activePaths.pointer.descriptor_digest
|
|
1259
|
+
|| descriptor.session_nonce_digest !== activePaths.pointer.session_nonce_digest))) fail('RUN_NOT_MANAGED', 'restart descriptor不正');
|
|
1260
|
+
try {
|
|
1261
|
+
const observed = await observeManagedProcessStartIdentity(descriptor.pid);
|
|
1262
|
+
if (canonicalizeArtifact(observed) === canonicalizeArtifact(descriptor.process_start_identity)) fail('RUN_BUSY', 'supervisorは既にlive');
|
|
1263
|
+
} catch (error) {
|
|
1264
|
+
if (error instanceof ManagedRuntimeError && error.code === 'RUN_BUSY') throw error;
|
|
1265
|
+
// process absentだけを許容。PID再利用は旧supervisorではない。
|
|
1266
|
+
}
|
|
1267
|
+
const socketPath = path.join(runDir, descriptor.socket_ref);
|
|
1268
|
+
const socketInfo = await lstat(socketPath).catch(() => null);
|
|
1269
|
+
if (socketInfo !== null) {
|
|
1270
|
+
if (!socketInfo.isSocket() || socketInfo.isSymbolicLink()) fail('RUN_NOT_MANAGED', 'stale socket path差替え');
|
|
1271
|
+
// 所有者が居るなら消さない。観測できなければ「誰も居ない」へ丸めず観測失敗として止める
|
|
1272
|
+
// (lsofはヒット無しでexit 1を返すので、この2つを区別する必要がある)。
|
|
1273
|
+
let staleOwners;
|
|
1274
|
+
try { staleOwners = await pidsOwningSocketPath(socketPath); }
|
|
1275
|
+
catch (error) {
|
|
1276
|
+
if (error instanceof ManagedRuntimeError) throw error;
|
|
1277
|
+
fail('RUN_NOT_MANAGED', 'stale socket owner観測失敗');
|
|
1278
|
+
}
|
|
1279
|
+
if (staleOwners.length > 0) fail('RUN_BUSY', 'control socketは別processが所有中');
|
|
1280
|
+
await rm(socketPath, { force: true });
|
|
1281
|
+
}
|
|
1282
|
+
return { previousDescriptorDigest: descriptor.descriptor_digest };
|
|
1283
|
+
}
|
|
1284
|
+
|
|
1285
|
+
/** restart commit pointerがあればcandidate namespaceを正本として解決する。固定pathは初回互換cache。 */
|
|
1286
|
+
export async function resolveActiveRuntimePaths({ runDir }) {
|
|
1287
|
+
const fixed = {
|
|
1288
|
+
descriptorPath: path.join(runDir, 'supervisor/descriptor.json'),
|
|
1289
|
+
sessionPath: path.join(runDir, 'supervisor/session'),
|
|
1290
|
+
controlEventsPath: path.join(runDir, 'control-events.json'),
|
|
1291
|
+
pointer: null,
|
|
1292
|
+
};
|
|
1293
|
+
const artifact = await readCanonicalRegular(path.join(runDir, 'supervisor/active-runtime.json')).catch((error) => {
|
|
1294
|
+
if (error?.code === 'ENOENT') return null;
|
|
1295
|
+
throw error;
|
|
1296
|
+
});
|
|
1297
|
+
if (artifact === null) return fixed;
|
|
1298
|
+
const pointer = artifact.value;
|
|
1299
|
+
const fields = ['schema', 'run_id', 'candidate_ref', 'activation_request_id',
|
|
1300
|
+
'activation_request_digest', 'activation_intent_digest', 'activation_response_digest', 'descriptor_digest',
|
|
1301
|
+
'session_nonce_digest', 'control_head_digest', 'control_head_sequence',
|
|
1302
|
+
'committed_at', 'pointer_digest'];
|
|
1303
|
+
if (pointer?.schema !== 'lattice.runtime_active_pointer.v1'
|
|
1304
|
+
|| Object.keys(pointer).sort().join('\0') !== fields.sort().join('\0')
|
|
1305
|
+
|| !identifier(pointer.run_id) || !digest(pointer.descriptor_digest)
|
|
1306
|
+
|| !identifier(pointer.activation_request_id) || !digest(pointer.activation_request_digest)
|
|
1307
|
+
|| !digest(pointer.activation_intent_digest)
|
|
1308
|
+
|| !digest(pointer.activation_response_digest)
|
|
1309
|
+
|| !Number.isSafeInteger(pointer.control_head_sequence) || pointer.control_head_sequence < 1
|
|
1310
|
+
|| !digest(pointer.session_nonce_digest) || !digest(pointer.control_head_digest)
|
|
1311
|
+
|| typeof pointer.committed_at !== 'string' || Number.isNaN(Date.parse(pointer.committed_at))
|
|
1312
|
+
|| pointer.pointer_digest !== selfDigest(pointer, 'pointer_digest')
|
|
1313
|
+
|| !/^supervisor\/restart-candidates\/[0-9a-f]{64}$/u.test(pointer.candidate_ref)) {
|
|
1314
|
+
fail('RUN_NOT_MANAGED', 'active runtime pointer不正');
|
|
1315
|
+
}
|
|
1316
|
+
const candidateDir = path.join(runDir, pointer.candidate_ref);
|
|
1317
|
+
if (!candidateDir.startsWith(`${path.join(runDir, 'supervisor/restart-candidates')}${path.sep}`)) {
|
|
1318
|
+
fail('RUN_NOT_MANAGED', 'active runtime pointerがcandidate外');
|
|
1319
|
+
}
|
|
1320
|
+
const descriptorPath = path.join(candidateDir, 'descriptor.json');
|
|
1321
|
+
const sessionPath = path.join(candidateDir, 'session');
|
|
1322
|
+
const controlEventsPath = path.join(candidateDir, 'control-events.json');
|
|
1323
|
+
const descriptor = (await readCanonicalRegular(descriptorPath)).value;
|
|
1324
|
+
const sessionNonce = (await readFile(sessionPath, 'utf8')).trim();
|
|
1325
|
+
const controlEvents = (await readCanonicalRegular(controlEventsPath)).value;
|
|
1326
|
+
const validControlChain = Array.isArray(controlEvents) && controlEvents.every((event, index) => (
|
|
1327
|
+
event?.schema === 'lattice.runtime_control_event.v1'
|
|
1328
|
+
&& event.run_id === pointer.run_id && event.sequence === index + 1
|
|
1329
|
+
&& event.previous_digest === (index === 0 ? null : controlEvents[index - 1].event_digest)
|
|
1330
|
+
&& event.event_digest === selfDigest(event, 'event_digest')
|
|
1331
|
+
));
|
|
1332
|
+
if (descriptor?.descriptor_digest !== pointer.descriptor_digest
|
|
1333
|
+
|| digestArtifact(sessionNonce) !== pointer.session_nonce_digest
|
|
1334
|
+
|| !validControlChain
|
|
1335
|
+
|| controlEvents[pointer.control_head_sequence - 1]?.event_digest !== pointer.control_head_digest) {
|
|
1336
|
+
fail('RUN_NOT_MANAGED', 'active runtime candidate digest binding不正');
|
|
1337
|
+
}
|
|
1338
|
+
return {
|
|
1339
|
+
descriptorPath, sessionPath, controlEventsPath,
|
|
1340
|
+
pointer: structuredClone(pointer),
|
|
1341
|
+
};
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1344
|
+
/** durable daemon側の一要求一応答server。mutation本体はstore所有handlerだけが行う。 */
|
|
1345
|
+
export async function serveRuntimeControlSocket({ socketPath, handler }) {
|
|
1346
|
+
if (typeof socketPath !== 'string' || socketPath.length === 0 || typeof handler !== 'function') fail('SUPERVISOR_CONFIGURATION_INVALID', 'control server入力不正');
|
|
1347
|
+
try {
|
|
1348
|
+
const existing = await lstat(socketPath);
|
|
1349
|
+
if (!existing.isSocket()) fail('RUN_NOT_MANAGED', 'control.sockがsocketでない');
|
|
1350
|
+
let liveOwners;
|
|
1351
|
+
try { liveOwners = await pidsOwningSocketPath(socketPath); }
|
|
1352
|
+
catch (error) {
|
|
1353
|
+
if (error instanceof ManagedRuntimeError) throw error;
|
|
1354
|
+
fail('RUN_NOT_MANAGED', 'control.sock owner観測失敗');
|
|
1355
|
+
}
|
|
1356
|
+
if (liveOwners.length > 0) fail('RUN_BUSY', 'control.sockはlive processが所有する');
|
|
1357
|
+
await rm(socketPath, { force: true });
|
|
1358
|
+
} catch (error) {
|
|
1359
|
+
if (error?.code !== 'ENOENT') throw error;
|
|
1360
|
+
}
|
|
1361
|
+
const absoluteSocket = path.resolve(socketPath);
|
|
1362
|
+
const daemonCwd = await realpath(process.cwd());
|
|
1363
|
+
const socketRef = path.relative(daemonCwd, absoluteSocket);
|
|
1364
|
+
if (socketRef !== 'supervisor/control.sock' || path.resolve(daemonCwd, socketRef) !== absoluteSocket) fail('RUN_NOT_MANAGED', 'daemon control socket cwd binding不正');
|
|
1365
|
+
const server = net.createServer((socket) => {
|
|
1366
|
+
socket.setEncoding('utf8');
|
|
1367
|
+
let buffer = '';
|
|
1368
|
+
let handled = false;
|
|
1369
|
+
socket.on('data', async (chunk) => {
|
|
1370
|
+
if (handled) return;
|
|
1371
|
+
buffer += chunk;
|
|
1372
|
+
if (Buffer.byteLength(buffer, 'utf8') > 1_048_576) { handled = true; socket.destroy(); return; }
|
|
1373
|
+
const newline = buffer.indexOf('\n');
|
|
1374
|
+
if (newline < 0) return;
|
|
1375
|
+
handled = true;
|
|
1376
|
+
try {
|
|
1377
|
+
if (buffer.slice(newline + 1).length !== 0) throw new Error('multiple documents');
|
|
1378
|
+
const request = JSON.parse(buffer.slice(0, newline));
|
|
1379
|
+
if (!validateRuntimeControlRequest(request)) throw new Error('invalid request');
|
|
1380
|
+
const response = await handler(structuredClone(request));
|
|
1381
|
+
if (!validateRuntimeControlResponse(response, request.operation) || response.request_id !== request.request_id || response.run_id !== request.run_id) throw new Error('invalid response');
|
|
1382
|
+
socket.end(`${canonicalizeArtifact(response)}\n`);
|
|
1383
|
+
} catch { socket.destroy(); }
|
|
1384
|
+
});
|
|
1385
|
+
});
|
|
1386
|
+
await new Promise((resolve, reject) => {
|
|
1387
|
+
server.once('error', reject);
|
|
1388
|
+
server.listen(socketRef, () => { server.off('error', reject); resolve(); });
|
|
1389
|
+
});
|
|
1390
|
+
await chmod(socketPath, 0o600);
|
|
1391
|
+
return server;
|
|
1392
|
+
}
|
|
1393
|
+
|
|
1394
|
+
/** CLI寿命から独立する同module daemonを起動する。session/descriptor/store昇格はcallerが検証する。 */
|
|
1395
|
+
export async function launchDurableSupervisor({ runDir, timeoutMs = 5_000 }) {
|
|
1396
|
+
if (typeof runDir !== 'string' || runDir.length === 0 || !Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 60_000) fail('SUPERVISOR_CONFIGURATION_INVALID', 'daemon launch入力不正');
|
|
1397
|
+
const entryPath = fileURLToPath(import.meta.url);
|
|
1398
|
+
const child = spawn(process.execPath, [entryPath, '--serve-managed-run', runDir], {
|
|
1399
|
+
detached: true, stdio: ['ignore', 'ignore', 'ignore', 'ipc'], cwd: runDir,
|
|
1400
|
+
});
|
|
1401
|
+
return new Promise((resolve, reject) => {
|
|
1402
|
+
const timer = setTimeout(() => {
|
|
1403
|
+
try { process.kill(child.pid, 'SIGTERM'); } catch { /* already exited */ }
|
|
1404
|
+
reject(new ManagedRuntimeError('ADAPTER_CONTROLLER_UNAVAILABLE', 'supervisor daemon起動timeout'));
|
|
1405
|
+
}, timeoutMs);
|
|
1406
|
+
const failed = (detail) => { clearTimeout(timer); reject(new ManagedRuntimeError('ADAPTER_CONTROLLER_UNAVAILABLE', detail)); };
|
|
1407
|
+
child.once('error', (error) => failed(error.message));
|
|
1408
|
+
child.once('exit', (code) => failed(`supervisor daemon exited: ${code}`));
|
|
1409
|
+
child.once('message', (message) => {
|
|
1410
|
+
if (message?.schema === 'lattice.runtime_supervisor_bootstrap_failure.v1'
|
|
1411
|
+
&& typeof message.code === 'string' && typeof message.detail === 'string') {
|
|
1412
|
+
clearTimeout(timer);
|
|
1413
|
+
return reject(new ManagedRuntimeError(message.code, message.detail));
|
|
1414
|
+
}
|
|
1415
|
+
if (message?.schema !== 'lattice.runtime_supervisor_bootstrap_ready.v1'
|
|
1416
|
+
|| message.pid !== child.pid || typeof message.socketPath !== 'string'
|
|
1417
|
+
|| typeof message.sessionNonce !== 'string' || message.sessionNonce.length < 32
|
|
1418
|
+
|| !validateProcessStartIdentity(message.processStartIdentity)
|
|
1419
|
+
|| message.processStartIdentity.pid !== child.pid) return failed('supervisor bootstrap message不正');
|
|
1420
|
+
observeManagedProcessStartIdentity(child.pid).then((observed) => {
|
|
1421
|
+
if (canonicalizeArtifact(observed) !== canonicalizeArtifact(message.processStartIdentity)) return failed('supervisor bootstrap PID/start identity不一致');
|
|
1422
|
+
clearTimeout(timer);
|
|
1423
|
+
child.disconnect();
|
|
1424
|
+
child.unref();
|
|
1425
|
+
resolve({ pid: child.pid, processStartIdentity: observed, socketPath: message.socketPath, sessionNonce: message.sessionNonce });
|
|
1426
|
+
}).catch((error) => failed(error.message));
|
|
1427
|
+
});
|
|
1428
|
+
});
|
|
1429
|
+
}
|
|
1430
|
+
|
|
1431
|
+
const isDirectDaemon = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]
|
|
1432
|
+
&& process.argv[2] === '--serve-managed-run';
|
|
1433
|
+
if (isDirectDaemon) {
|
|
1434
|
+
const runDir = process.argv[3];
|
|
1435
|
+
const sessionNonce = randomBytes(32).toString('hex');
|
|
1436
|
+
Promise.all([
|
|
1437
|
+
import('./runtime-cli.mjs'),
|
|
1438
|
+
import('./todo-revision.mjs'),
|
|
1439
|
+
import('./todo-store.mjs'),
|
|
1440
|
+
]).then(async ([{ runManagedSupervisorDaemon }, { validatePhaseTodoRevision },
|
|
1441
|
+
{ applyPhaseTodoRevision, createTodoStoreWriter }]) => {
|
|
1442
|
+
if (typeof runManagedSupervisorDaemon !== 'function') fail('SUPERVISOR_CONFIGURATION_INVALID', 'runtime-cli daemon handlerなし');
|
|
1443
|
+
let daemonCleanup = async () => {};
|
|
1444
|
+
let shuttingDown = false;
|
|
1445
|
+
const stop = async (signal) => {
|
|
1446
|
+
if (shuttingDown) return;
|
|
1447
|
+
shuttingDown = true;
|
|
1448
|
+
try { await daemonCleanup(signal); process.exitCode = 0; }
|
|
1449
|
+
catch (error) { process.stderr.write(`${error.message}\n`); process.exitCode = 1; }
|
|
1450
|
+
};
|
|
1451
|
+
process.once('SIGTERM', () => { stop('SIGTERM'); });
|
|
1452
|
+
process.once('SIGINT', () => { stop('SIGINT'); });
|
|
1453
|
+
await runManagedSupervisorDaemon({
|
|
1454
|
+
runDir,
|
|
1455
|
+
sessionNonce,
|
|
1456
|
+
serveRuntimeControlSocket,
|
|
1457
|
+
activateController: (args) => activateManagedSupervisorController({ ...args, runDir, supervisorSessionNonce: sessionNonce }),
|
|
1458
|
+
onReady: async (socketPath) => process.send?.({ schema: 'lattice.runtime_supervisor_bootstrap_ready.v1', pid: process.pid, processStartIdentity: await observeManagedProcessStartIdentity(process.pid), socketPath, sessionNonce }),
|
|
1459
|
+
registerDaemonCleanup: (handler) => { if (typeof handler !== 'function') fail('SUPERVISOR_CONFIGURATION_INVALID', 'daemon cleanup handler不正'); daemonCleanup = handler; },
|
|
1460
|
+
crashInjector: process.env.NODE_ENV === 'test' && process.env.LATTICE_INTERNAL_TEST_CRASH_POINT
|
|
1461
|
+
? async (point) => {
|
|
1462
|
+
if (point === process.env.LATTICE_INTERNAL_TEST_CRASH_POINT) process.kill(process.pid, 'SIGKILL');
|
|
1463
|
+
}
|
|
1464
|
+
: null,
|
|
1465
|
+
validatePhaseRevision: validatePhaseTodoRevision,
|
|
1466
|
+
commitPhaseRevision: (revision) => applyPhaseTodoRevision({
|
|
1467
|
+
repoRoot: path.resolve(runDir, '..', '..', '..'),
|
|
1468
|
+
writer: createTodoStoreWriter({ caller: 'g5-authoring' }), revision,
|
|
1469
|
+
// actorは`{host, session, agent}`のexact recordである(`lattice.todo_event.v4`)。
|
|
1470
|
+
// 文字列を渡していた間、seam_splitの再計画はphase revisionをcommitできなかった
|
|
1471
|
+
// ——工程storeへ書く直前で必ず落ちるので、請求項8の再開まで一度も届いていない。
|
|
1472
|
+
actor: {
|
|
1473
|
+
host: 'lattice-runtime',
|
|
1474
|
+
session: sessionNonce.slice(0, 32),
|
|
1475
|
+
agent: 'lattice-runtime-supervisor',
|
|
1476
|
+
},
|
|
1477
|
+
recordedAt: new Date().toISOString(),
|
|
1478
|
+
}),
|
|
1479
|
+
});
|
|
1480
|
+
})
|
|
1481
|
+
.catch((error) => {
|
|
1482
|
+
process.send?.({
|
|
1483
|
+
schema: 'lattice.runtime_supervisor_bootstrap_failure.v1',
|
|
1484
|
+
code: error instanceof ManagedRuntimeError ? error.code : 'ADAPTER_CONTROLLER_UNAVAILABLE',
|
|
1485
|
+
detail: String(error?.message ?? error),
|
|
1486
|
+
});
|
|
1487
|
+
process.stderr.write(`${error.message}\n`);
|
|
1488
|
+
process.exitCode = 1;
|
|
1489
|
+
});
|
|
1490
|
+
}
|