@quolu/lattice 0.52.2 → 0.52.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/artifact-contracts-v2.mjs +325 -325
- package/src/artifact-contracts.mjs +895 -895
- package/src/boundary-compiler.mjs +712 -712
- package/src/boundary-observation-compiler-v2.mjs +344 -344
- package/src/bounded-seam.mjs +230 -230
- package/src/bridge-address.mjs +107 -107
- package/src/bridge-cli.mjs +297 -297
- package/src/bridge-config.mjs +346 -346
- package/src/bridge-daemon.mjs +378 -378
- package/src/bridge-hub-protocol.mjs +198 -0
- package/src/bridge-launch-agent.mjs +323 -323
- package/src/bridge-registrar.mjs +102 -102
- package/src/bridge-server.mjs +376 -376
- package/src/cli-help.mjs +308 -308
- package/src/cli-stdio.mjs +40 -40
- package/src/control-compiler.mjs +532 -532
- package/src/dag-chain.mjs +261 -261
- package/src/factory-diagnostics.mjs +188 -188
- package/src/git-process.mjs +72 -0
- package/src/hash-chain.mjs +76 -76
- package/src/hooks-cli.mjs +1057 -1057
- package/src/isolation-runner.mjs +409 -409
- package/src/node-version-guard.mjs +44 -44
- package/src/project-cli.mjs +699 -697
- package/src/project-identity.mjs +130 -130
- package/src/rc1-black-box-oracle.mjs +906 -906
- package/src/rc1-comparison.mjs +154 -154
- package/src/rc1-evidence-bundle.mjs +456 -456
- package/src/rc1-v4-campaign.mjs +708 -708
- package/src/rc1-v4-transform.mjs +467 -467
- package/src/rc1-v5-artifact-set.mjs +855 -855
- package/src/rc1-v5-behavior-evidence.mjs +594 -594
- package/src/rc1-v5-campaign.mjs +797 -797
- package/src/rc1-v5-transform.mjs +421 -421
- package/src/rc1-v6-artifact-set.mjs +807 -807
- package/src/rc1-v6-behavior-evidence.mjs +273 -273
- package/src/rc1-v6-campaign.mjs +623 -623
- package/src/rc1-v6-causal-binding.mjs +473 -473
- package/src/rc1-v6-measurement.mjs +313 -313
- package/src/rc2-artifact-set.mjs +1584 -1584
- package/src/rc2-campaign.mjs +1506 -1506
- package/src/rc2-delivery-policy-front-end.mjs +1079 -1079
- package/src/rc2-delivery-policy-oracle.mjs +134 -134
- package/src/rc2-delivery-policy-transform.mjs +1127 -1127
- package/src/rc2-rc1-transfer-front-end.mjs +511 -511
- package/src/rc3-actual-dogfood.mjs +652 -652
- package/src/rc3-dogfood-scaffold.mjs +315 -315
- package/src/rc3-scripted-campaign.mjs +1383 -1383
- package/src/rc4-stage1-dogfood.mjs +673 -673
- package/src/runtime-adapter-registry.mjs +524 -524
- package/src/runtime-cli.mjs +4588 -4588
- package/src/runtime-contracts.mjs +824 -824
- package/src/runtime-control-store.mjs +604 -604
- package/src/runtime-controller-protocol.mjs +587 -587
- package/src/runtime-decision-verifier.mjs +701 -701
- package/src/runtime-diff-observer.mjs +361 -361
- package/src/runtime-direct-os-observer.mjs +301 -301
- package/src/runtime-driver-state.mjs +166 -166
- package/src/runtime-engine.mjs +779 -779
- package/src/runtime-errors.mjs +356 -356
- package/src/runtime-event-store.mjs +189 -189
- package/src/runtime-front-end.mjs +925 -925
- package/src/runtime-gate-store.mjs +481 -481
- package/src/runtime-hold-recompile.mjs +916 -916
- package/src/runtime-io-sentinel.mjs +391 -391
- package/src/runtime-lifecycle-lock.mjs +294 -294
- package/src/runtime-managed-supervisor.mjs +1490 -1490
- package/src/runtime-multi-epoch-store.mjs +838 -838
- package/src/runtime-projection.mjs +269 -269
- package/src/runtime-pull-intake.mjs +1192 -1192
- package/src/runtime-scripted-adapter-controller.mjs +1160 -1160
- package/src/runtime-scripted-executor.mjs +163 -163
- package/src/runtime-scripted-worktree.mjs +104 -104
- package/src/runtime-seam-resolve.mjs +428 -428
- package/src/runtime-seam-treatment.mjs +173 -173
- package/src/runtime-socket-owner.mjs +125 -125
- package/src/runtime-work-order-contracts.mjs +91 -91
- package/src/runtime-work-order-controller.mjs +1171 -1171
- package/src/runtime-worktree-executor.mjs +199 -199
- package/src/schedulability-compiler-v2.mjs +303 -303
- package/src/schedulability-verifier-v2.mjs +317 -317
- package/src/seam-apply.mjs +549 -549
- package/src/seam-commit-shared.mjs +22 -22
- package/src/seam-commit-transform.mjs +81 -81
- package/src/seam-commit.mjs +18 -18
- package/src/seam-cost.mjs +322 -322
- package/src/seam-derivation.mjs +188 -188
- package/src/seam-gate.mjs +146 -146
- package/src/seam-proposal-contracts.mjs +446 -446
- package/src/seam-proposal-queries.mjs +521 -521
- package/src/seam-proposal.mjs +2011 -2011
- package/src/seam-ref.mjs +33 -33
- package/src/seam-rewrite.mjs +286 -286
- package/src/seam-transform.mjs +554 -554
- package/src/seam-verification.mjs +260 -260
- package/src/sensor-adapter.mjs +432 -432
- package/src/sensor-cli.mjs +139 -139
- package/src/sensor-diff.mjs +661 -661
- package/src/sensor-node-runtime.mjs +53 -53
- package/src/sensor-runtime.mjs +52 -52
- package/src/timestamp-contract.mjs +8 -8
- package/src/todo-audit-pending.mjs +91 -91
- package/src/todo-chain.mjs +178 -178
- package/src/todo-cli.mjs +3143 -3141
- package/src/todo-contracts.mjs +728 -728
- package/src/todo-dashboard-registry.mjs +573 -573
- package/src/todo-dispatch-shape.mjs +190 -190
- package/src/todo-gantt-html-independence.mjs +239 -239
- package/src/todo-gantt-html-shared.mjs +226 -226
- package/src/todo-gantt-html-style.mjs +131 -131
- package/src/todo-gantt-html.mjs +248 -248
- package/src/todo-gantt-layout.mjs +974 -974
- package/src/todo-gantt-live.mjs +361 -361
- package/src/todo-gantt-nested.mjs +263 -263
- package/src/todo-gantt-presentation.mjs +217 -217
- package/src/todo-gantt-scope.mjs +123 -123
- package/src/todo-gantt-svg.mjs +353 -353
- package/src/todo-independence-contracts.mjs +595 -595
- package/src/todo-independence-guidance.mjs +322 -322
- package/src/todo-independence.mjs +640 -640
- package/src/todo-markdown-renderer.mjs +260 -260
- package/src/todo-migration.mjs +448 -448
- package/src/todo-narrative-anchor.mjs +130 -130
- package/src/todo-note-store.mjs +629 -629
- package/src/todo-parallel-candidates.mjs +114 -114
- package/src/todo-revision.mjs +995 -995
- package/src/todo-split.mjs +472 -472
- package/src/todo-status.mjs +690 -690
- package/src/todo-store-git-transaction.mjs +418 -418
- package/src/todo-store.mjs +4360 -4322
- package/src/treatment-compiler.mjs +728 -728
- package/src/treatment-runner.mjs +656 -656
- package/src/witness-scaffold.mjs +180 -180
|
@@ -1,1171 +1,1171 @@
|
|
|
1
|
-
import net from 'node:net';
|
|
2
|
-
import { createHash, randomBytes } from 'node:crypto';
|
|
3
|
-
import { execFile } from 'node:child_process';
|
|
4
|
-
import { constants as fsConstants, readFileSync } from 'node:fs';
|
|
5
|
-
import {
|
|
6
|
-
chmod,
|
|
7
|
-
lstat,
|
|
8
|
-
mkdir,
|
|
9
|
-
open,
|
|
10
|
-
readFile,
|
|
11
|
-
realpath,
|
|
12
|
-
rename,
|
|
13
|
-
rm,
|
|
14
|
-
} from 'node:fs/promises';
|
|
15
|
-
import path from 'node:path';
|
|
16
|
-
import { promisify } from 'node:util';
|
|
17
|
-
|
|
18
|
-
import { canonicalizeArtifact, digestArtifact } from './artifact-contracts.mjs';
|
|
19
|
-
import {
|
|
20
|
-
armStagedWriteLease,
|
|
21
|
-
CONTROLLER_OPERATIONS,
|
|
22
|
-
validateArmedWriteLease,
|
|
23
|
-
validateControllerRequest,
|
|
24
|
-
validateRuntimeAdapterCapabilities,
|
|
25
|
-
validateRuntimeHeartbeatPolicy,
|
|
26
|
-
} from './runtime-controller-protocol.mjs';
|
|
27
|
-
import {
|
|
28
|
-
selfDigest,
|
|
29
|
-
validateExecutorPacket,
|
|
30
|
-
validateExecutorReceipt,
|
|
31
|
-
} from './runtime-contracts.mjs';
|
|
32
|
-
import { captureWorktreeDiff } from './runtime-diff-observer.mjs';
|
|
33
|
-
import { validateSupervisorWriteGate } from './runtime-gate-store.mjs';
|
|
34
|
-
import { observeManagedProcessStartIdentity } from './runtime-managed-supervisor.mjs';
|
|
35
|
-
import {
|
|
36
|
-
scriptedWorktreeId as workOrderWorktreeId,
|
|
37
|
-
scriptedWorktreePath as workOrderWorktreePath,
|
|
38
|
-
} from './runtime-scripted-worktree.mjs';
|
|
39
|
-
import {
|
|
40
|
-
createRunWorkOrder,
|
|
41
|
-
validateRunWorkOrder,
|
|
42
|
-
validateRunWorkReport,
|
|
43
|
-
} from './runtime-work-order-contracts.mjs';
|
|
44
|
-
|
|
45
|
-
const MAX_DOCUMENT_BYTES = 8_388_608;
|
|
46
|
-
const SHA256 = /^[0-9a-f]{64}$/u;
|
|
47
|
-
const ID = /^[0-9A-Za-z](?:[0-9A-Za-z._-]{0,127})$/u;
|
|
48
|
-
const execFileAsync = promisify(execFile);
|
|
49
|
-
const REQUEST_SCHEMA_TO_OPERATION = Object.freeze(Object.fromEntries([
|
|
50
|
-
['lattice.adapter_dispatch_request.v1', 'dispatch'],
|
|
51
|
-
['lattice.adapter_observe_request.v1', 'observe'],
|
|
52
|
-
['lattice.adapter_running_inventory_request.v1', 'inventory'],
|
|
53
|
-
['lattice.adapter_barrier_request.v1', 'barrier'],
|
|
54
|
-
['lattice.adapter_rebind_request.v1', 'rebind'],
|
|
55
|
-
['lattice.adapter_prepare_request.v1', 'prepare'],
|
|
56
|
-
['lattice.adapter_activate_request.v1', 'activate'],
|
|
57
|
-
['lattice.adapter_release_request.v1', 'release'],
|
|
58
|
-
['lattice.adapter_revoke_request.v1', 'revoke'],
|
|
59
|
-
]));
|
|
60
|
-
|
|
61
|
-
export class WorkOrderAdapterControllerError extends Error {
|
|
62
|
-
constructor(code, message, detail = {}) {
|
|
63
|
-
super(message);
|
|
64
|
-
this.name = 'WorkOrderAdapterControllerError';
|
|
65
|
-
this.code = code;
|
|
66
|
-
this.detail = detail;
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
function fail(code, message, detail) {
|
|
71
|
-
throw new WorkOrderAdapterControllerError(code, message, detail);
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
function plain(value) {
|
|
75
|
-
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
|
76
|
-
&& Object.getPrototypeOf(value) === Object.prototype;
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
function exact(value, fields) {
|
|
80
|
-
if (!plain(value)) return false;
|
|
81
|
-
const actual = Object.keys(value).sort();
|
|
82
|
-
const expected = [...fields].sort();
|
|
83
|
-
return actual.length === expected.length
|
|
84
|
-
&& actual.every((field, index) => field === expected[index]);
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
function sign(value, field) {
|
|
88
|
-
value[field] = '';
|
|
89
|
-
value[field] = selfDigest(value, field);
|
|
90
|
-
return value;
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
function sha256Bytes(bytes) {
|
|
94
|
-
return createHash('sha256').update(bytes).digest('hex');
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
function controllerErrorArtifact(error, requestId = null) {
|
|
98
|
-
const artifact = {
|
|
99
|
-
schema: 'lattice.work_order_adapter_error.v1',
|
|
100
|
-
code: error?.code ?? 'WORK_ORDER_CONTROLLER_FAILED',
|
|
101
|
-
message: String(error?.message ?? error),
|
|
102
|
-
request_id: typeof requestId === 'string' ? requestId : null,
|
|
103
|
-
detail: plain(error?.detail) ? structuredClone(error.detail) : {},
|
|
104
|
-
error_digest: '',
|
|
105
|
-
};
|
|
106
|
-
artifact.error_digest = selfDigest(artifact, 'error_digest');
|
|
107
|
-
return artifact;
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
function validateBootstrap(value) {
|
|
111
|
-
return exact(value, [
|
|
112
|
-
'schema',
|
|
113
|
-
'request_id',
|
|
114
|
-
'run_id',
|
|
115
|
-
'controller_socket_ref',
|
|
116
|
-
'supervisor_socket_ref',
|
|
117
|
-
'supervisor_session_nonce',
|
|
118
|
-
'bootstrap_digest',
|
|
119
|
-
])
|
|
120
|
-
&& value.schema === 'lattice.adapter_controller_bootstrap.v1'
|
|
121
|
-
&& ID.test(value.request_id ?? '')
|
|
122
|
-
&& ID.test(value.run_id ?? '')
|
|
123
|
-
&& typeof value.controller_socket_ref === 'string'
|
|
124
|
-
&& /^supervisor\/controllers\/[0-9A-Za-z][0-9A-Za-z._-]{0,127}\.sock$/u
|
|
125
|
-
.test(value.controller_socket_ref)
|
|
126
|
-
&& value.supervisor_socket_ref === 'supervisor/control.sock'
|
|
127
|
-
&& typeof value.supervisor_session_nonce === 'string'
|
|
128
|
-
&& value.supervisor_session_nonce.length >= 32
|
|
129
|
-
&& SHA256.test(value.bootstrap_digest ?? '')
|
|
130
|
-
&& selfDigest(value, 'bootstrap_digest') === value.bootstrap_digest;
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
function validateHandshakeRequest(value, bootstrap) {
|
|
134
|
-
return exact(value, [
|
|
135
|
-
'schema',
|
|
136
|
-
'request_id',
|
|
137
|
-
'run_id',
|
|
138
|
-
'supervisor_session_nonce',
|
|
139
|
-
'controller_socket_ref',
|
|
140
|
-
'challenge',
|
|
141
|
-
'request_digest',
|
|
142
|
-
])
|
|
143
|
-
&& value.schema === 'lattice.adapter_controller_handshake_request.v1'
|
|
144
|
-
&& ID.test(value.request_id ?? '')
|
|
145
|
-
&& value.run_id === bootstrap.run_id
|
|
146
|
-
&& value.supervisor_session_nonce === bootstrap.supervisor_session_nonce
|
|
147
|
-
&& value.controller_socket_ref === bootstrap.controller_socket_ref
|
|
148
|
-
&& typeof value.challenge === 'string'
|
|
149
|
-
&& value.challenge.length >= 32
|
|
150
|
-
&& SHA256.test(value.request_digest ?? '')
|
|
151
|
-
&& selfDigest(value, 'request_digest') === value.request_digest;
|
|
152
|
-
}
|
|
153
|
-
|
|
154
|
-
async function durableReplaceBytes(filePath, bytes, mode = 0o600) {
|
|
155
|
-
const directory = path.dirname(filePath);
|
|
156
|
-
await mkdir(directory, { recursive: true, mode: 0o700 });
|
|
157
|
-
const temporaryPath = path.join(directory, `.${path.basename(filePath)}-${process.pid}.tmp`);
|
|
158
|
-
let handle;
|
|
159
|
-
try {
|
|
160
|
-
handle = await open(
|
|
161
|
-
temporaryPath,
|
|
162
|
-
fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_WRONLY,
|
|
163
|
-
mode,
|
|
164
|
-
);
|
|
165
|
-
await handle.writeFile(bytes);
|
|
166
|
-
await handle.sync();
|
|
167
|
-
await handle.close();
|
|
168
|
-
handle = null;
|
|
169
|
-
await rename(temporaryPath, filePath);
|
|
170
|
-
await chmod(filePath, mode);
|
|
171
|
-
const directoryHandle = await open(directory, fsConstants.O_RDONLY);
|
|
172
|
-
// Windowsはdirectory handleのfsyncを許さず常にEPERM/EINVALを返す(Node仕様)。
|
|
173
|
-
// win32のこの2値だけ許容し、他OS・他エラーは従来どおり失敗させる。
|
|
174
|
-
try {
|
|
175
|
-
await directoryHandle.sync();
|
|
176
|
-
} catch (error) {
|
|
177
|
-
if (process.platform !== 'win32' || !['EPERM', 'EINVAL'].includes(error?.code)) throw error;
|
|
178
|
-
} finally {
|
|
179
|
-
await directoryHandle.close();
|
|
180
|
-
}
|
|
181
|
-
} finally {
|
|
182
|
-
await handle?.close().catch(() => {});
|
|
183
|
-
await rm(temporaryPath, { force: true }).catch(() => {});
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
async function readAndValidateGate(runDir, writeLease) {
|
|
188
|
-
const gatePath = path.join(runDir, 'supervisor', 'write-gate.json');
|
|
189
|
-
let gate;
|
|
190
|
-
try {
|
|
191
|
-
gate = JSON.parse(await readFile(gatePath, 'utf8'));
|
|
192
|
-
} catch (error) {
|
|
193
|
-
fail('WORK_ORDER_WRITE_GATE_REJECTED', '中央write gateを読めない', {
|
|
194
|
-
message: String(error?.message ?? error),
|
|
195
|
-
});
|
|
196
|
-
}
|
|
197
|
-
if (!validateSupervisorWriteGate(gate)
|
|
198
|
-
|| gate.run_id !== writeLease.run_id
|
|
199
|
-
|| gate.plan_epoch !== writeLease.plan_epoch
|
|
200
|
-
|| gate.gate_generation !== writeLease.gate_generation
|
|
201
|
-
|| gate.release_barrier_digest !== writeLease.release_barrier_digest
|
|
202
|
-
|| !gate.armed_lease_digests.includes(writeLease.lease_digest)) {
|
|
203
|
-
fail('WORK_ORDER_WRITE_GATE_REJECTED', '中央write gateがarmed leaseを認可しない');
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
/** 登録時のdigestへ束縛されたwork-order spool設定を読む。 */
|
|
208
|
-
async function readWorkOrderBehavior(repoRoot, capabilitiesDigest) {
|
|
209
|
-
const descriptorPath = path.join(repoRoot, '.lattice', 'runtime', 'adapter-registry',
|
|
210
|
-
'descriptors', 'work-order.json');
|
|
211
|
-
let descriptor;
|
|
212
|
-
try {
|
|
213
|
-
descriptor = JSON.parse(await readFile(descriptorPath, 'utf8'));
|
|
214
|
-
} catch { fail('WORK_ORDER_BOOTSTRAP_INVALID', 'work-order adapterが未登録'); }
|
|
215
|
-
if (typeof descriptor?.config_ref !== 'string' || typeof descriptor.config_digest !== 'string') {
|
|
216
|
-
fail('WORK_ORDER_BOOTSTRAP_INVALID', 'work-order adapter descriptorが不正');
|
|
217
|
-
}
|
|
218
|
-
if (descriptor.capabilities_digest !== capabilitiesDigest) {
|
|
219
|
-
fail(
|
|
220
|
-
'WORK_ORDER_CAPABILITIES_MISMATCH',
|
|
221
|
-
'work-order adapterをregistration input v2で再登録する必要がある',
|
|
222
|
-
{
|
|
223
|
-
registered_capabilities_digest: descriptor.capabilities_digest ?? null,
|
|
224
|
-
controller_capabilities_digest: capabilitiesDigest,
|
|
225
|
-
},
|
|
226
|
-
);
|
|
227
|
-
}
|
|
228
|
-
const configPath = path.join(repoRoot, ...descriptor.config_ref.split('/'));
|
|
229
|
-
let bytes;
|
|
230
|
-
try {
|
|
231
|
-
bytes = await readFile(configPath);
|
|
232
|
-
} catch {
|
|
233
|
-
fail('WORK_ORDER_BOOTSTRAP_INVALID', '登録済みadapter configを読めない', {
|
|
234
|
-
config_ref: descriptor.config_ref,
|
|
235
|
-
});
|
|
236
|
-
}
|
|
237
|
-
if (sha256Bytes(bytes) !== descriptor.config_digest) {
|
|
238
|
-
fail('WORK_ORDER_BOOTSTRAP_INVALID', 'adapter configが登録時のdigestと一致しない', {
|
|
239
|
-
config_ref: descriptor.config_ref,
|
|
240
|
-
});
|
|
241
|
-
}
|
|
242
|
-
let config;
|
|
243
|
-
try {
|
|
244
|
-
config = JSON.parse(bytes.toString('utf8'));
|
|
245
|
-
} catch {
|
|
246
|
-
fail('WORK_ORDER_BOOTSTRAP_INVALID', 'adapter configのJSONが不正');
|
|
247
|
-
}
|
|
248
|
-
if (!exact(config, ['schema', 'spool_dir'])
|
|
249
|
-
|| config.schema !== 'lattice.work_order_adapter_config.v1'
|
|
250
|
-
|| typeof config.spool_dir !== 'string'
|
|
251
|
-
|| !path.isAbsolute(config.spool_dir)) {
|
|
252
|
-
fail('WORK_ORDER_BOOTSTRAP_INVALID', 'work-order adapter configがexact contractを満たさない');
|
|
253
|
-
}
|
|
254
|
-
let spoolDir;
|
|
255
|
-
try { spoolDir = await realpath(config.spool_dir); }
|
|
256
|
-
catch { fail('WORK_ORDER_BOOTSTRAP_INVALID', 'spool_dirを解決できない'); }
|
|
257
|
-
const info = await lstat(spoolDir);
|
|
258
|
-
if (!info.isDirectory() || info.isSymbolicLink()) {
|
|
259
|
-
fail('WORK_ORDER_BOOTSTRAP_INVALID', 'spool_dirがreal directoryでない');
|
|
260
|
-
}
|
|
261
|
-
for (const child of ['orders', 'reports']) {
|
|
262
|
-
const directory = path.join(spoolDir, child);
|
|
263
|
-
await mkdir(directory, { recursive: true, mode: 0o700 });
|
|
264
|
-
await chmod(directory, 0o700);
|
|
265
|
-
const childInfo = await lstat(directory);
|
|
266
|
-
if (!childInfo.isDirectory() || childInfo.isSymbolicLink()
|
|
267
|
-
|| await realpath(directory) !== directory) {
|
|
268
|
-
fail('WORK_ORDER_BOOTSTRAP_INVALID', `${child} directoryがcanonicalでない`);
|
|
269
|
-
}
|
|
270
|
-
}
|
|
271
|
-
return { spoolDir };
|
|
272
|
-
}
|
|
273
|
-
|
|
274
|
-
const sleep = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
|
|
275
|
-
|
|
276
|
-
async function readWorkReport(reportPath, packetDigest) {
|
|
277
|
-
let info;
|
|
278
|
-
try { info = await lstat(reportPath); }
|
|
279
|
-
catch (error) {
|
|
280
|
-
if (error?.code === 'ENOENT') return null;
|
|
281
|
-
throw error;
|
|
282
|
-
}
|
|
283
|
-
if (!info.isFile() || info.isSymbolicLink() || (info.mode & 0o077) !== 0
|
|
284
|
-
|| info.size <= 0 || info.size > MAX_DOCUMENT_BYTES
|
|
285
|
-
|| await realpath(reportPath) !== reportPath) {
|
|
286
|
-
fail('WORK_ORDER_REPORT_INVALID', 'work reportがprivate canonical regular fileでない');
|
|
287
|
-
}
|
|
288
|
-
let bytes;
|
|
289
|
-
try { bytes = await readFile(reportPath); }
|
|
290
|
-
catch (error) { throw error; }
|
|
291
|
-
let report;
|
|
292
|
-
try { report = JSON.parse(bytes.toString('utf8')); }
|
|
293
|
-
catch { fail('WORK_ORDER_REPORT_INVALID', 'work reportのJSONが不正'); }
|
|
294
|
-
if (!validateRunWorkReport(report)
|
|
295
|
-
|| report.packet_digest !== packetDigest
|
|
296
|
-
|| bytes.toString('utf8') !== `${canonicalizeArtifact(report)}\n`) {
|
|
297
|
-
fail('WORK_ORDER_REPORT_INVALID', 'work reportがpacketへexact bindしない');
|
|
298
|
-
}
|
|
299
|
-
return report;
|
|
300
|
-
}
|
|
301
|
-
|
|
302
|
-
async function waitForWorkReport(reportPath, packetDigest) {
|
|
303
|
-
for (;;) {
|
|
304
|
-
const report = await readWorkReport(reportPath, packetDigest);
|
|
305
|
-
if (report !== null) return report;
|
|
306
|
-
await sleep(50);
|
|
307
|
-
}
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
function processIdentity(record) {
|
|
311
|
-
const identity = {
|
|
312
|
-
schema: 'lattice.process_start_identity.v1',
|
|
313
|
-
platform: process.platform,
|
|
314
|
-
pid: record.pid,
|
|
315
|
-
started_identity: record.startedIdentity,
|
|
316
|
-
identity_digest: '',
|
|
317
|
-
};
|
|
318
|
-
identity.identity_digest = selfDigest(identity, 'identity_digest');
|
|
319
|
-
return identity;
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
function quiescedState(rawState) {
|
|
323
|
-
if (rawState.startsWith('T')) return 'stopped';
|
|
324
|
-
if (rawState.startsWith('Z')) return 'zombie';
|
|
325
|
-
return null;
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
async function observeWorkerProcessTree(pid, { requireDescendantsInRootGroup = true } = {}) {
|
|
329
|
-
let stdout;
|
|
330
|
-
try {
|
|
331
|
-
({ stdout } = await execFileAsync('/bin/ps', [
|
|
332
|
-
'-axo', 'pid=,ppid=,pgid=,state=,lstart=',
|
|
333
|
-
], {
|
|
334
|
-
encoding: 'utf8',
|
|
335
|
-
maxBuffer: 8 * 1024 * 1024,
|
|
336
|
-
}));
|
|
337
|
-
} catch { fail('WORK_ORDER_REPORT_INVALID', 'worker_pidをOS観測できない'); }
|
|
338
|
-
const records = new Map();
|
|
339
|
-
for (const rawLine of stdout.split('\n')) {
|
|
340
|
-
const line = rawLine.trim();
|
|
341
|
-
if (line.length === 0) continue;
|
|
342
|
-
const match = line.match(/^(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.+)$/u);
|
|
343
|
-
if (!match) fail('WORK_ORDER_REPORT_INVALID', 'ps観測結果を解析できない');
|
|
344
|
-
const record = {
|
|
345
|
-
pid: Number(match[1]),
|
|
346
|
-
parentPid: Number(match[2]),
|
|
347
|
-
processGroupId: Number(match[3]),
|
|
348
|
-
rawState: match[4],
|
|
349
|
-
startedIdentity: match[5].trim(),
|
|
350
|
-
};
|
|
351
|
-
if (![record.pid, record.processGroupId].every((value) => Number.isSafeInteger(value)
|
|
352
|
-
&& value > 0) || !Number.isSafeInteger(record.parentPid) || record.parentPid < 0
|
|
353
|
-
|| record.startedIdentity.length === 0 || records.has(record.pid)) {
|
|
354
|
-
fail('WORK_ORDER_REPORT_INVALID', 'ps process recordが不正');
|
|
355
|
-
}
|
|
356
|
-
records.set(record.pid, record);
|
|
357
|
-
}
|
|
358
|
-
const root = records.get(pid);
|
|
359
|
-
if (root === undefined) fail('WORK_ORDER_REPORT_INVALID', 'worker_pidが存在しない');
|
|
360
|
-
if (root.processGroupId !== root.pid) {
|
|
361
|
-
fail('WORK_ORDER_REPORT_INVALID', 'worker rootがprocess group leaderでない');
|
|
362
|
-
}
|
|
363
|
-
const descendantPids = new Set();
|
|
364
|
-
let changed = true;
|
|
365
|
-
while (changed) {
|
|
366
|
-
changed = false;
|
|
367
|
-
for (const record of records.values()) {
|
|
368
|
-
if (record.pid !== pid && !descendantPids.has(record.pid)
|
|
369
|
-
&& (record.parentPid === pid || descendantPids.has(record.parentPid))) {
|
|
370
|
-
descendantPids.add(record.pid);
|
|
371
|
-
changed = true;
|
|
372
|
-
}
|
|
373
|
-
}
|
|
374
|
-
}
|
|
375
|
-
const descendants = [...descendantPids].map((childPid) => records.get(childPid))
|
|
376
|
-
.sort((left, right) => left.pid - right.pid);
|
|
377
|
-
if (requireDescendantsInRootGroup
|
|
378
|
-
&& descendants.some((record) => record.processGroupId !== root.processGroupId)) {
|
|
379
|
-
fail('WORK_ORDER_REPORT_INVALID', 'worker childがrootと別process groupに居る');
|
|
380
|
-
}
|
|
381
|
-
const expectedGroup = new Set([pid, ...descendantPids]);
|
|
382
|
-
if ([...records.values()].some((record) => record.processGroupId === root.processGroupId
|
|
383
|
-
&& !expectedGroup.has(record.pid))) {
|
|
384
|
-
fail('WORK_ORDER_REPORT_INVALID', 'worker process groupを無関係processと共有している');
|
|
385
|
-
}
|
|
386
|
-
return {
|
|
387
|
-
pid: root.pid,
|
|
388
|
-
parentPid: root.parentPid,
|
|
389
|
-
processGroupId: root.processGroupId,
|
|
390
|
-
processStartIdentity: processIdentity(root),
|
|
391
|
-
rawState: root.rawState,
|
|
392
|
-
observationChildren: descendants.map((record) => ({
|
|
393
|
-
pid: record.pid,
|
|
394
|
-
parent_pid: record.parentPid,
|
|
395
|
-
process_start_identity_digest: processIdentity(record).identity_digest,
|
|
396
|
-
process_group_id: record.processGroupId,
|
|
397
|
-
state: quiescedState(record.rawState),
|
|
398
|
-
})),
|
|
399
|
-
};
|
|
400
|
-
}
|
|
401
|
-
|
|
402
|
-
export const runtimeWorkOrderControllerInternal = Object.freeze({
|
|
403
|
-
observeWorkerProcessTree,
|
|
404
|
-
});
|
|
405
|
-
|
|
406
|
-
/** orderを書き、bridgeのreportを状態遷移の合図として待つ。 */
|
|
407
|
-
export async function spawnWorkOrderWorker({ packet, worktreePath, spoolDir }) {
|
|
408
|
-
const order = createRunWorkOrder({ packet, worktreePath });
|
|
409
|
-
if (!validateRunWorkOrder(order)) fail('WORK_ORDER_ORDER_INVALID', 'work order生成に失敗した');
|
|
410
|
-
const orderPath = path.join(spoolDir, 'orders', `${packet.packet_digest}.json`);
|
|
411
|
-
const reportPath = path.join(spoolDir, 'reports', `${packet.packet_digest}.json`);
|
|
412
|
-
// spoolはrunを跨いで再利用される。古いdoneを新dispatchの完了として読まないよう、
|
|
413
|
-
// reportを消してからorderをpublishする(bridgeはorderのrename後にしか動かない)。
|
|
414
|
-
await rm(reportPath, { force: true });
|
|
415
|
-
await durableReplaceBytes(orderPath, Buffer.from(`${canonicalizeArtifact(order)}\n`));
|
|
416
|
-
// 辞退はbridge内で再配車し、受諾した席が決まるまでreportを出さない。
|
|
417
|
-
// controller側の任意timeoutでactivateを失敗させず、実席pidだけをimmutable bindする。
|
|
418
|
-
const firstReport = await waitForWorkReport(reportPath, packet.packet_digest);
|
|
419
|
-
// dispatch時は席が稼働中で、一過性tool childが別PGIDへ出る。ここで束縛するのは
|
|
420
|
-
// 停止対象rootのstart identityとPGIDだけ。全子孫の同一PGID・静止はbarrierで検証する。
|
|
421
|
-
const processTree = await observeWorkerProcessTree(firstReport.worker_pid, {
|
|
422
|
-
requireDescendantsInRootGroup: false,
|
|
423
|
-
});
|
|
424
|
-
const completed = (async () => {
|
|
425
|
-
let report = firstReport;
|
|
426
|
-
while (report.state !== 'done') {
|
|
427
|
-
await sleep(50);
|
|
428
|
-
report = await waitForWorkReport(reportPath, packet.packet_digest);
|
|
429
|
-
if (report.worker_pid !== firstReport.worker_pid) {
|
|
430
|
-
fail('WORK_ORDER_REPORT_INVALID', 'work reportのworker_pidが途中で変化した');
|
|
431
|
-
}
|
|
432
|
-
}
|
|
433
|
-
const checkpoint = await captureWorktreeDiff({
|
|
434
|
-
worktreePath,
|
|
435
|
-
baseSha: packet.base_sha,
|
|
436
|
-
});
|
|
437
|
-
return {
|
|
438
|
-
observedDiff: checkpoint.diff.entries.map(({ path: changedPath, change }) => ({
|
|
439
|
-
path: changedPath,
|
|
440
|
-
change,
|
|
441
|
-
})),
|
|
442
|
-
checkpointDigest: checkpoint.checkpoint_digest,
|
|
443
|
-
};
|
|
444
|
-
})();
|
|
445
|
-
return {
|
|
446
|
-
pid: firstReport.worker_pid,
|
|
447
|
-
process_group_id: processTree.processGroupId,
|
|
448
|
-
process_start_identity: processTree.processStartIdentity,
|
|
449
|
-
process_membership: 'dynamic_group',
|
|
450
|
-
completed,
|
|
451
|
-
};
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
async function quiesceWorkOrderTask(task) {
|
|
455
|
-
try {
|
|
456
|
-
process.kill(-task.worker.process_group_id, 'SIGSTOP');
|
|
457
|
-
} catch (error) {
|
|
458
|
-
fail('WORK_ORDER_BARRIER_REJECTED', 'worker process groupを止められない', {
|
|
459
|
-
pid: task.worker.pid, reason: String(error?.code ?? error?.message ?? error),
|
|
460
|
-
});
|
|
461
|
-
}
|
|
462
|
-
try {
|
|
463
|
-
let processTree = null;
|
|
464
|
-
for (let attempt = 0; attempt < 100; attempt += 1) {
|
|
465
|
-
processTree = await observeWorkerProcessTree(task.worker.pid);
|
|
466
|
-
if (quiescedState(processTree.rawState) !== null
|
|
467
|
-
&& processTree.observationChildren.every((child) => child.state !== null)) break;
|
|
468
|
-
processTree = null;
|
|
469
|
-
await sleep(10);
|
|
470
|
-
}
|
|
471
|
-
if (processTree === null
|
|
472
|
-
|| processTree.processGroupId !== task.worker.process_group_id
|
|
473
|
-
|| processTree.processStartIdentity.identity_digest
|
|
474
|
-
!== task.worker.process_start_identity.identity_digest) {
|
|
475
|
-
fail('WORK_ORDER_BARRIER_REJECTED', 'worker process集合を静止・再認証できない');
|
|
476
|
-
}
|
|
477
|
-
const checkpoint = await captureWorktreeDiff({
|
|
478
|
-
worktreePath: task.worktreePath, baseSha: task.packet.base_sha,
|
|
479
|
-
});
|
|
480
|
-
return {
|
|
481
|
-
checkpoint,
|
|
482
|
-
processObservationDigest: digestArtifact({
|
|
483
|
-
schema: 'lattice.direct_process_observation.v2',
|
|
484
|
-
root: {
|
|
485
|
-
pid: task.worker.pid,
|
|
486
|
-
parent_pid: processTree.parentPid,
|
|
487
|
-
process_start_identity_digest: task.worker.process_start_identity.identity_digest,
|
|
488
|
-
process_group_id: task.worker.process_group_id,
|
|
489
|
-
state: quiescedState(processTree.rawState),
|
|
490
|
-
},
|
|
491
|
-
children: processTree.observationChildren,
|
|
492
|
-
process_group_id: task.worker.process_group_id,
|
|
493
|
-
quiesced: true,
|
|
494
|
-
}),
|
|
495
|
-
worktreeFingerprintDigest: digestArtifact({
|
|
496
|
-
schema: 'lattice.direct_worktree_fingerprint.v1',
|
|
497
|
-
worktree_id: task.worktreeId,
|
|
498
|
-
worktree_realpath: task.worktreePath,
|
|
499
|
-
checkpoint_digest: checkpoint.checkpoint_digest,
|
|
500
|
-
}),
|
|
501
|
-
};
|
|
502
|
-
} catch (error) {
|
|
503
|
-
try { process.kill(-task.worker.process_group_id, 'SIGCONT'); }
|
|
504
|
-
catch { /* 元のbarrier失敗を保持する */ }
|
|
505
|
-
task.state = 'running';
|
|
506
|
-
throw error;
|
|
507
|
-
}
|
|
508
|
-
}
|
|
509
|
-
|
|
510
|
-
export function createWorkOrderAdapterCapabilities() {
|
|
511
|
-
const capabilities = sign({
|
|
512
|
-
schema: 'lattice.runtime_adapter_capabilities.v2',
|
|
513
|
-
operations: [...CONTROLLER_OPERATIONS],
|
|
514
|
-
process_observation: true,
|
|
515
|
-
worktree_fingerprint: true,
|
|
516
|
-
staged_write_lease: true,
|
|
517
|
-
durable_dispatch: true,
|
|
518
|
-
host_driven_epoch: true,
|
|
519
|
-
capabilities_digest: '',
|
|
520
|
-
}, 'capabilities_digest');
|
|
521
|
-
if (!validateRuntimeAdapterCapabilities(capabilities)) {
|
|
522
|
-
fail('WORK_ORDER_CONTROLLER_INVALID', 'capabilities生成に失敗した');
|
|
523
|
-
}
|
|
524
|
-
return capabilities;
|
|
525
|
-
}
|
|
526
|
-
|
|
527
|
-
export async function createWorkOrderAdapterController({
|
|
528
|
-
bootstrap,
|
|
529
|
-
runDir,
|
|
530
|
-
repoRoot,
|
|
531
|
-
controllerSessionNonce = randomBytes(32).toString('hex'),
|
|
532
|
-
}) {
|
|
533
|
-
if (!validateBootstrap(bootstrap)) {
|
|
534
|
-
fail('WORK_ORDER_BOOTSTRAP_INVALID', 'controller bootstrapが不正');
|
|
535
|
-
}
|
|
536
|
-
const canonicalRunDir = await realpath(runDir);
|
|
537
|
-
const canonicalRepoRoot = await realpath(repoRoot);
|
|
538
|
-
if (path.resolve(canonicalRunDir, '..', '..', '..') !== canonicalRepoRoot) {
|
|
539
|
-
fail('WORK_ORDER_BOOTSTRAP_INVALID', 'run storeとrepo rootのbindingが不正');
|
|
540
|
-
}
|
|
541
|
-
const controllerId = path.basename(bootstrap.controller_socket_ref, '.sock');
|
|
542
|
-
if (!controllerId.startsWith('work-order-')
|
|
543
|
-
|| typeof controllerSessionNonce !== 'string'
|
|
544
|
-
|| controllerSessionNonce.length < 32) {
|
|
545
|
-
fail('WORK_ORDER_BOOTSTRAP_INVALID', 'work-order controller identityが不正');
|
|
546
|
-
}
|
|
547
|
-
const capabilities = createWorkOrderAdapterCapabilities();
|
|
548
|
-
const heartbeat = sign({
|
|
549
|
-
schema: 'lattice.runtime_heartbeat_policy.v1',
|
|
550
|
-
interval_ms: 1_000,
|
|
551
|
-
ttl_ms: 10_000,
|
|
552
|
-
disconnect_revokes_immediately: true,
|
|
553
|
-
policy_digest: '',
|
|
554
|
-
}, 'policy_digest');
|
|
555
|
-
if (!validateRuntimeHeartbeatPolicy(heartbeat)) {
|
|
556
|
-
fail('WORK_ORDER_CONTROLLER_INVALID', 'heartbeat policy生成に失敗した');
|
|
557
|
-
}
|
|
558
|
-
const descriptor = sign({
|
|
559
|
-
schema: 'lattice.runtime_adapter_controller_descriptor.v1',
|
|
560
|
-
controller_id: controllerId,
|
|
561
|
-
adapter_kind: 'work-order',
|
|
562
|
-
pid: process.pid,
|
|
563
|
-
process_start_identity: await observeManagedProcessStartIdentity(process.pid),
|
|
564
|
-
socket_ref: bootstrap.controller_socket_ref,
|
|
565
|
-
controller_session_nonce_digest: digestArtifact(controllerSessionNonce),
|
|
566
|
-
capabilities,
|
|
567
|
-
heartbeat,
|
|
568
|
-
descriptor_digest: '',
|
|
569
|
-
}, 'descriptor_digest');
|
|
570
|
-
const sessionNonceDigest = digestArtifact(bootstrap.supervisor_session_nonce);
|
|
571
|
-
const behavior = await readWorkOrderBehavior(
|
|
572
|
-
canonicalRepoRoot,
|
|
573
|
-
capabilities.capabilities_digest,
|
|
574
|
-
);
|
|
575
|
-
const stagedLeases = new Map();
|
|
576
|
-
const armedLeases = new Map();
|
|
577
|
-
const preparedPackets = new Map();
|
|
578
|
-
const tasks = new Map();
|
|
579
|
-
const todoToHandle = new Map();
|
|
580
|
-
let currentEpoch = 1;
|
|
581
|
-
let registrationDigest = null;
|
|
582
|
-
|
|
583
|
-
// worker poolのprocess寿命は外部adapterが所有する。controller closeで席を殺さない。
|
|
584
|
-
const reapWorkers = () => {};
|
|
585
|
-
|
|
586
|
-
const workerProcessOf = (task) => ({
|
|
587
|
-
pid: task.worker.pid,
|
|
588
|
-
process_group_id: task.worker.process_group_id,
|
|
589
|
-
process_start_identity: structuredClone(task.worker.process_start_identity),
|
|
590
|
-
process_membership: task.worker.process_membership,
|
|
591
|
-
});
|
|
592
|
-
|
|
593
|
-
const persistReceipt = async (receipt) => {
|
|
594
|
-
const payloadDigest = digestArtifact(receipt);
|
|
595
|
-
const receiptPath = path.join(
|
|
596
|
-
canonicalRunDir,
|
|
597
|
-
'controllers',
|
|
598
|
-
controllerId,
|
|
599
|
-
'receipts',
|
|
600
|
-
`${payloadDigest}.json`,
|
|
601
|
-
);
|
|
602
|
-
await durableReplaceBytes(
|
|
603
|
-
receiptPath,
|
|
604
|
-
Buffer.from(`${canonicalizeArtifact(receipt)}\n`),
|
|
605
|
-
);
|
|
606
|
-
return payloadDigest;
|
|
607
|
-
};
|
|
608
|
-
|
|
609
|
-
const responseFor = async (operation, request) => {
|
|
610
|
-
if (!validateControllerRequest(operation, request)) {
|
|
611
|
-
fail('WORK_ORDER_REQUEST_INVALID', `${operation} requestがprotocol contractを満たさない`);
|
|
612
|
-
}
|
|
613
|
-
if (registrationDigest !== null && request.registration_digest !== registrationDigest) {
|
|
614
|
-
fail('WORK_ORDER_REGISTRATION_MISMATCH', 'controller registration digestが変化した');
|
|
615
|
-
}
|
|
616
|
-
registrationDigest = request.registration_digest;
|
|
617
|
-
if (operation === 'prepare') {
|
|
618
|
-
const packet = request.executor_packet;
|
|
619
|
-
const lease = request.staged_lease;
|
|
620
|
-
if (lease.run_id !== bootstrap.run_id || lease.todo_id !== packet.todo_id
|
|
621
|
-
|| lease.plan_epoch !== packet.plan_epoch || lease.packet_digest !== packet.packet_digest
|
|
622
|
-
|| lease.controller_registration_digest !== request.registration_digest
|
|
623
|
-
|| lease.supervisor_session_nonce_digest !== sessionNonceDigest) {
|
|
624
|
-
fail('WORK_ORDER_LEASE_REJECTED', 'prepare packetとstaged leaseのbindingが不正');
|
|
625
|
-
}
|
|
626
|
-
const prior = stagedLeases.get(lease.lease_digest);
|
|
627
|
-
if (prior !== undefined
|
|
628
|
-
&& canonicalizeArtifact(prior) !== canonicalizeArtifact(lease)) {
|
|
629
|
-
fail('WORK_ORDER_LEASE_REJECTED', '同じdigestのstaged lease bytesが変化した');
|
|
630
|
-
}
|
|
631
|
-
stagedLeases.set(lease.lease_digest, structuredClone(lease));
|
|
632
|
-
preparedPackets.set(packet.packet_digest, structuredClone(packet));
|
|
633
|
-
currentEpoch = packet.plan_epoch;
|
|
634
|
-
const ack = sign({
|
|
635
|
-
schema: 'lattice.adapter_prepare_ack.v1',
|
|
636
|
-
ack_id: `prepare-${packet.packet_digest.slice(0, 24)}`,
|
|
637
|
-
registration_digest: request.registration_digest,
|
|
638
|
-
controller_id: controllerId,
|
|
639
|
-
run_id: bootstrap.run_id,
|
|
640
|
-
todo_id: packet.todo_id,
|
|
641
|
-
plan_epoch: packet.plan_epoch,
|
|
642
|
-
packet_digest: packet.packet_digest,
|
|
643
|
-
staged_lease_digest: lease.lease_digest,
|
|
644
|
-
supervisor_session_nonce_digest: sessionNonceDigest,
|
|
645
|
-
ack_digest: '',
|
|
646
|
-
}, 'ack_digest');
|
|
647
|
-
return sign({
|
|
648
|
-
schema: 'lattice.adapter_prepare_response.v1',
|
|
649
|
-
request_id: request.request_id,
|
|
650
|
-
prepare_ack: ack,
|
|
651
|
-
staged_lease_digest: lease.lease_digest,
|
|
652
|
-
response_digest: '',
|
|
653
|
-
}, 'response_digest');
|
|
654
|
-
}
|
|
655
|
-
if (operation === 'activate') {
|
|
656
|
-
if (request.staged_lease_digests.some((digest) => !stagedLeases.has(digest))) {
|
|
657
|
-
fail('WORK_ORDER_LEASE_REJECTED', 'activateが未知のstaged leaseを含む');
|
|
658
|
-
}
|
|
659
|
-
const epochs = new Set(request.staged_lease_digests
|
|
660
|
-
.map((digest) => stagedLeases.get(digest).plan_epoch));
|
|
661
|
-
if (epochs.size > 1) fail('WORK_ORDER_LEASE_REJECTED', 'activate leaseのepochが混在する');
|
|
662
|
-
if (epochs.size === 1) currentEpoch = [...epochs][0];
|
|
663
|
-
const ack = sign({
|
|
664
|
-
schema: 'lattice.adapter_ready_ack.v1',
|
|
665
|
-
ack_id: `ready-${request.activation_digest.slice(0, 24)}`,
|
|
666
|
-
registration_digest: request.registration_digest,
|
|
667
|
-
controller_id: controllerId,
|
|
668
|
-
run_id: bootstrap.run_id,
|
|
669
|
-
plan_epoch: currentEpoch,
|
|
670
|
-
activation_digest: request.activation_digest,
|
|
671
|
-
staged_lease_digests: [...request.staged_lease_digests],
|
|
672
|
-
supervisor_session_nonce_digest: sessionNonceDigest,
|
|
673
|
-
ack_digest: '',
|
|
674
|
-
}, 'ack_digest');
|
|
675
|
-
return sign({
|
|
676
|
-
schema: 'lattice.adapter_activate_response.v1',
|
|
677
|
-
request_id: request.request_id,
|
|
678
|
-
ready_ack: ack,
|
|
679
|
-
observed_pointer_digest: request.committed_epoch_digest,
|
|
680
|
-
response_digest: '',
|
|
681
|
-
}, 'response_digest');
|
|
682
|
-
}
|
|
683
|
-
if (operation === 'release') {
|
|
684
|
-
const armed = [];
|
|
685
|
-
for (const stagedDigest of request.staged_lease_digests) {
|
|
686
|
-
const staged = stagedLeases.get(stagedDigest);
|
|
687
|
-
if (staged === undefined) {
|
|
688
|
-
fail('WORK_ORDER_LEASE_REJECTED', 'releaseが未知のstaged leaseを含む');
|
|
689
|
-
}
|
|
690
|
-
const lease = armStagedWriteLease(staged, {
|
|
691
|
-
releaseBarrierDigest: request.release_barrier_digest,
|
|
692
|
-
gateGeneration: request.gate_generation,
|
|
693
|
-
});
|
|
694
|
-
stagedLeases.delete(stagedDigest);
|
|
695
|
-
armedLeases.set(lease.lease_digest, lease);
|
|
696
|
-
armed.push(lease.lease_digest);
|
|
697
|
-
}
|
|
698
|
-
armed.sort();
|
|
699
|
-
const ack = sign({
|
|
700
|
-
schema: 'lattice.adapter_release_ack.v1',
|
|
701
|
-
ack_id: `release-${request.release_barrier_digest.slice(0, 24)}`,
|
|
702
|
-
registration_digest: request.registration_digest,
|
|
703
|
-
controller_id: controllerId,
|
|
704
|
-
run_id: bootstrap.run_id,
|
|
705
|
-
plan_epoch: currentEpoch,
|
|
706
|
-
release_barrier_digest: request.release_barrier_digest,
|
|
707
|
-
gate_generation: request.gate_generation,
|
|
708
|
-
armed_lease_digests: armed,
|
|
709
|
-
supervisor_session_nonce_digest: sessionNonceDigest,
|
|
710
|
-
ack_digest: '',
|
|
711
|
-
}, 'ack_digest');
|
|
712
|
-
return sign({
|
|
713
|
-
schema: 'lattice.adapter_release_response.v1',
|
|
714
|
-
request_id: request.request_id,
|
|
715
|
-
release_ack: ack,
|
|
716
|
-
armed_lease_digests: armed,
|
|
717
|
-
observed_gate_generation: request.gate_generation,
|
|
718
|
-
response_digest: '',
|
|
719
|
-
}, 'response_digest');
|
|
720
|
-
}
|
|
721
|
-
if (operation === 'dispatch') {
|
|
722
|
-
const { packet, write_lease: writeLease } = request;
|
|
723
|
-
if (!validateExecutorPacket(packet) || !validateArmedWriteLease(writeLease)
|
|
724
|
-
|| !armedLeases.has(writeLease.lease_digest)
|
|
725
|
-
|| writeLease.run_id !== bootstrap.run_id
|
|
726
|
-
|| writeLease.todo_id !== packet.todo_id
|
|
727
|
-
|| writeLease.plan_epoch !== packet.plan_epoch
|
|
728
|
-
|| writeLease.packet_digest !== packet.packet_digest
|
|
729
|
-
|| writeLease.controller_registration_digest !== request.registration_digest
|
|
730
|
-
|| writeLease.supervisor_session_nonce_digest !== sessionNonceDigest) {
|
|
731
|
-
fail('WORK_ORDER_PACKET_REJECTED', 'dispatch packetとarmed leaseのbindingが不正');
|
|
732
|
-
}
|
|
733
|
-
const priorHandle = todoToHandle.get(packet.todo_id);
|
|
734
|
-
if (priorHandle !== undefined) {
|
|
735
|
-
const prior = tasks.get(priorHandle);
|
|
736
|
-
if (prior.packet.packet_digest === packet.packet_digest
|
|
737
|
-
&& prior.lease.lease_digest === writeLease.lease_digest) {
|
|
738
|
-
return sign({
|
|
739
|
-
schema: 'lattice.adapter_dispatch_response.v2',
|
|
740
|
-
request_id: request.request_id,
|
|
741
|
-
executor_handle: prior.executorHandle,
|
|
742
|
-
worktree_id: prior.worktreeId,
|
|
743
|
-
packet_digest: packet.packet_digest,
|
|
744
|
-
lease_digest: writeLease.lease_digest,
|
|
745
|
-
worker_process: workerProcessOf(prior),
|
|
746
|
-
response_digest: '',
|
|
747
|
-
}, 'response_digest');
|
|
748
|
-
}
|
|
749
|
-
if (prior.state !== 'held') {
|
|
750
|
-
fail('WORK_ORDER_DUPLICATE_DISPATCH', '同じTODOへ異なるpacketをdispatchできない');
|
|
751
|
-
}
|
|
752
|
-
// 長寿命worker poolのprocessを殺さず、barrierで止めた同じ席を再開して
|
|
753
|
-
// successor orderへ載せ直す。process寿命は外部adapterの所有である。
|
|
754
|
-
try { process.kill(-prior.worker.process_group_id, 'SIGCONT'); }
|
|
755
|
-
catch (error) {
|
|
756
|
-
fail('WORK_ORDER_EXECUTION_FAILED', 'held workerを再開できない', {
|
|
757
|
-
pid: prior.worker.pid,
|
|
758
|
-
reason: String(error?.code ?? error?.message ?? error),
|
|
759
|
-
});
|
|
760
|
-
}
|
|
761
|
-
prior.state = 'superseded';
|
|
762
|
-
todoToHandle.delete(packet.todo_id);
|
|
763
|
-
}
|
|
764
|
-
await readAndValidateGate(canonicalRunDir, writeLease);
|
|
765
|
-
// canonical repoではなく自分の木へ書く。共有rootでは、書き込みの帰属をrootから
|
|
766
|
-
// 決められないので早期警報もcheckpoint判定も成立しない。木はsupervisorが
|
|
767
|
-
// dispatch前に用意する(監視を張るのがdispatchより前でなければ観測を取り逃す)。
|
|
768
|
-
const worktreePath = workOrderWorktreePath({ runDir: canonicalRunDir, packet });
|
|
769
|
-
try {
|
|
770
|
-
const info = await lstat(worktreePath);
|
|
771
|
-
if (!info.isDirectory()) throw new TypeError('not a directory');
|
|
772
|
-
} catch {
|
|
773
|
-
fail('WORK_ORDER_EXECUTION_FAILED', 'supervisorが用意したworktreeが無い', {
|
|
774
|
-
worktree_path: worktreePath,
|
|
775
|
-
});
|
|
776
|
-
}
|
|
777
|
-
const executorHandle = `work-order-${packet.packet_digest.slice(0, 24)}`;
|
|
778
|
-
const worktreeId = workOrderWorktreeId(packet);
|
|
779
|
-
// **dispatchで作業を終わらせない。** 終わらせると、走行中のTODOが1つも存在しない
|
|
780
|
-
// runになり、実行時の観測——書き込みを見て、他のworkerとの重なりを掴む——が
|
|
781
|
-
// 原理的に成立しない。dispatchは作業を起こして返り、完了はobserveが拾う。
|
|
782
|
-
const progress = {
|
|
783
|
-
schema: 'lattice.work_order_adapter_progress.v1',
|
|
784
|
-
run_id: bootstrap.run_id,
|
|
785
|
-
todo_id: packet.todo_id,
|
|
786
|
-
executor_handle: executorHandle,
|
|
787
|
-
worktree_id: worktreeId,
|
|
788
|
-
state: 'running',
|
|
789
|
-
};
|
|
790
|
-
const worktreeReal = await realpath(worktreePath);
|
|
791
|
-
// **workerは別processで起こす。** controller自身のprocessで作業すると、holdが
|
|
792
|
-
// 要求する静止を証明できない——止めれば応答できず、止めなければ証明できない。
|
|
793
|
-
// `detached`で独立process groupへ置く。同じgroupにcontrollerが居ると、直接OS観測が
|
|
794
|
-
// 「未記録process group memberを検出」として正しく落とす。
|
|
795
|
-
const spawned = await spawnWorkOrderWorker({
|
|
796
|
-
packet, worktreePath: worktreeReal,
|
|
797
|
-
spoolDir: behavior.spoolDir,
|
|
798
|
-
});
|
|
799
|
-
const task = {
|
|
800
|
-
packet: structuredClone(packet),
|
|
801
|
-
lease: structuredClone(writeLease),
|
|
802
|
-
executorHandle,
|
|
803
|
-
worktreeId,
|
|
804
|
-
worktreePath: worktreeReal,
|
|
805
|
-
receipt: null,
|
|
806
|
-
payloadDigest: digestArtifact(progress),
|
|
807
|
-
state: 'running',
|
|
808
|
-
failure: null,
|
|
809
|
-
settled: null,
|
|
810
|
-
worker: spawned,
|
|
811
|
-
};
|
|
812
|
-
tasks.set(executorHandle, task);
|
|
813
|
-
todoToHandle.set(packet.todo_id, executorHandle);
|
|
814
|
-
task.settled = (async () => {
|
|
815
|
-
try {
|
|
816
|
-
const { observedDiff, checkpointDigest } = await spawned.completed;
|
|
817
|
-
const receipt = sign({
|
|
818
|
-
schema: 'lattice.executor_receipt.v1',
|
|
819
|
-
receipt_id: `receipt-${packet.packet_digest.slice(0, 24)}`,
|
|
820
|
-
executor_handle: executorHandle,
|
|
821
|
-
worktree_id: worktreeId,
|
|
822
|
-
base_sha: packet.base_sha,
|
|
823
|
-
plan_epoch: packet.plan_epoch,
|
|
824
|
-
packet_digest: packet.packet_digest,
|
|
825
|
-
todo_id: packet.todo_id,
|
|
826
|
-
checkpoint_digest: checkpointDigest,
|
|
827
|
-
observed_diff: observedDiff,
|
|
828
|
-
receipt_digest: '',
|
|
829
|
-
}, 'receipt_digest');
|
|
830
|
-
if (!validateExecutorReceipt(receipt)) {
|
|
831
|
-
throw new TypeError('生成receiptがexecutor contractを満たさない');
|
|
832
|
-
}
|
|
833
|
-
task.payloadDigest = await persistReceipt(receipt);
|
|
834
|
-
task.receipt = receipt;
|
|
835
|
-
task.state = 'terminal';
|
|
836
|
-
} catch (error) {
|
|
837
|
-
// 失敗を走行中のまま放置しない。observeがtypedに落ちる形へ残す。
|
|
838
|
-
task.failure = String(error?.detail?.reason ?? error?.message ?? error);
|
|
839
|
-
}
|
|
840
|
-
})();
|
|
841
|
-
return sign({
|
|
842
|
-
schema: 'lattice.adapter_dispatch_response.v2',
|
|
843
|
-
request_id: request.request_id,
|
|
844
|
-
executor_handle: executorHandle,
|
|
845
|
-
worktree_id: worktreeId,
|
|
846
|
-
packet_digest: packet.packet_digest,
|
|
847
|
-
lease_digest: writeLease.lease_digest,
|
|
848
|
-
worker_process: workerProcessOf(task),
|
|
849
|
-
response_digest: '',
|
|
850
|
-
}, 'response_digest');
|
|
851
|
-
}
|
|
852
|
-
if (operation === 'observe') {
|
|
853
|
-
const task = tasks.get(request.executor_handle);
|
|
854
|
-
if (task === undefined
|
|
855
|
-
|| task.packet.plan_epoch !== request.expected_epoch
|
|
856
|
-
|| task.lease.lease_digest !== request.expected_lease_digest) {
|
|
857
|
-
fail('WORK_ORDER_OBSERVATION_REJECTED', 'observe bindingがdispatch記録と一致しない');
|
|
858
|
-
}
|
|
859
|
-
// 走行中に落ちた作業を「まだ走っている」と言い続けない。
|
|
860
|
-
if (task.failure !== null) {
|
|
861
|
-
fail('WORK_ORDER_EXECUTION_FAILED', 'worker実行が失敗した', { reason: task.failure });
|
|
862
|
-
}
|
|
863
|
-
const observation = sign({
|
|
864
|
-
schema: 'lattice.adapter_observation.v1',
|
|
865
|
-
state: task.state,
|
|
866
|
-
executor_handle: task.executorHandle,
|
|
867
|
-
plan_epoch: task.packet.plan_epoch,
|
|
868
|
-
lease_digest: task.lease.lease_digest,
|
|
869
|
-
payload_digest: task.payloadDigest,
|
|
870
|
-
observation_digest: '',
|
|
871
|
-
}, 'observation_digest');
|
|
872
|
-
return sign({
|
|
873
|
-
schema: 'lattice.adapter_observe_response.v1',
|
|
874
|
-
request_id: request.request_id,
|
|
875
|
-
observation,
|
|
876
|
-
observation_digest: observation.observation_digest,
|
|
877
|
-
response_digest: '',
|
|
878
|
-
}, 'response_digest');
|
|
879
|
-
}
|
|
880
|
-
if (operation === 'inventory') {
|
|
881
|
-
const runningBindings = [...tasks.values()]
|
|
882
|
-
.filter((task) => task.state === 'running')
|
|
883
|
-
.map((task) => ({
|
|
884
|
-
todo_id: task.packet.todo_id,
|
|
885
|
-
executor_handle: task.executorHandle,
|
|
886
|
-
worktree_id: task.worktreeId,
|
|
887
|
-
plan_epoch: task.packet.plan_epoch,
|
|
888
|
-
packet_digest: task.packet.packet_digest,
|
|
889
|
-
write_lease_id: task.lease.lease_id,
|
|
890
|
-
controller_registration_digest: request.registration_digest,
|
|
891
|
-
}))
|
|
892
|
-
.sort((left, right) => left.todo_id.localeCompare(right.todo_id));
|
|
893
|
-
return sign({
|
|
894
|
-
schema: 'lattice.adapter_running_inventory_response.v1',
|
|
895
|
-
request_id: request.request_id,
|
|
896
|
-
running_bindings: runningBindings,
|
|
897
|
-
inventory_digest: digestArtifact(runningBindings),
|
|
898
|
-
response_digest: '',
|
|
899
|
-
}, 'response_digest');
|
|
900
|
-
}
|
|
901
|
-
if (operation === 'barrier') {
|
|
902
|
-
const acks = [];
|
|
903
|
-
for (const binding of request.running_bindings) {
|
|
904
|
-
const task = tasks.get(binding.executor_handle);
|
|
905
|
-
if (task === undefined || task.packet.todo_id !== binding.todo_id) {
|
|
906
|
-
fail('WORK_ORDER_BARRIER_REJECTED', 'barrierが未知のrunning bindingを含む');
|
|
907
|
-
}
|
|
908
|
-
// **barrierは静止の宣言である。** PGID全体を止め、現在の全memberを観測する。
|
|
909
|
-
// ack前に失敗した時は外部所有の席を凍結したまま残さず再開する。
|
|
910
|
-
const quiesced = await quiesceWorkOrderTask(task);
|
|
911
|
-
task.state = 'held';
|
|
912
|
-
acks.push(sign({
|
|
913
|
-
schema: 'lattice.executor_quiescence_ack.v1',
|
|
914
|
-
ack_id: `barrier-${binding.packet_digest.slice(0, 24)}`,
|
|
915
|
-
run_id: bootstrap.run_id,
|
|
916
|
-
todo_id: binding.todo_id,
|
|
917
|
-
executor_handle: binding.executor_handle,
|
|
918
|
-
worktree_id: binding.worktree_id,
|
|
919
|
-
plan_epoch: binding.plan_epoch,
|
|
920
|
-
packet_digest: binding.packet_digest,
|
|
921
|
-
write_lease_id: binding.write_lease_id,
|
|
922
|
-
barrier_control_digest: request.barrier_control_digest,
|
|
923
|
-
final_checkpoint_digest: quiesced.checkpoint.checkpoint_digest,
|
|
924
|
-
process_observation_digest: quiesced.processObservationDigest,
|
|
925
|
-
worktree_fingerprint_digest: quiesced.worktreeFingerprintDigest,
|
|
926
|
-
supervisor_session_nonce_digest: sessionNonceDigest,
|
|
927
|
-
ack_digest: '',
|
|
928
|
-
}, 'ack_digest'));
|
|
929
|
-
}
|
|
930
|
-
return sign({
|
|
931
|
-
schema: 'lattice.adapter_barrier_response.v1',
|
|
932
|
-
request_id: request.request_id,
|
|
933
|
-
barrier_id: request.barrier_id,
|
|
934
|
-
quiescence_acks: acks,
|
|
935
|
-
response_digest: '',
|
|
936
|
-
}, 'response_digest');
|
|
937
|
-
}
|
|
938
|
-
if (operation === 'rebind') {
|
|
939
|
-
const rebind = request.rebind_packet;
|
|
940
|
-
const task = tasks.get(rebind.executor_handle);
|
|
941
|
-
if (task === undefined || task.worktreeId !== rebind.worktree_id
|
|
942
|
-
|| task.packet.todo_id !== rebind.todo_id) {
|
|
943
|
-
fail('WORK_ORDER_REBIND_REJECTED', 'rebind packetが既存taskへ帰属しない');
|
|
944
|
-
}
|
|
945
|
-
const staged = request.staged_lease;
|
|
946
|
-
if (staged.todo_id !== rebind.todo_id || staged.plan_epoch !== rebind.new_plan_epoch
|
|
947
|
-
|| staged.packet_digest !== rebind.packet_digest) {
|
|
948
|
-
fail('WORK_ORDER_REBIND_REJECTED', 'rebind packetとstaged leaseが一致しない');
|
|
949
|
-
}
|
|
950
|
-
stagedLeases.set(staged.lease_digest, structuredClone(staged));
|
|
951
|
-
task.packet.plan_epoch = rebind.new_plan_epoch;
|
|
952
|
-
currentEpoch = rebind.new_plan_epoch;
|
|
953
|
-
const ack = sign({
|
|
954
|
-
schema: 'lattice.executor_epoch_rebind_ack.v1',
|
|
955
|
-
ack_id: `rebind-${rebind.packet_digest.slice(0, 24)}`,
|
|
956
|
-
run_id: bootstrap.run_id,
|
|
957
|
-
todo_id: rebind.todo_id,
|
|
958
|
-
executor_handle: rebind.executor_handle,
|
|
959
|
-
worktree_id: rebind.worktree_id,
|
|
960
|
-
predecessor_epoch: rebind.new_plan_epoch - 1,
|
|
961
|
-
successor_epoch: rebind.new_plan_epoch,
|
|
962
|
-
// **holdされたworkerにreceiptは無い。** 作業を終えていないから止められているので、
|
|
963
|
-
// 完了の記録を前提にできない。前任packetのdigestはdispatch時のpacketが持っている。
|
|
964
|
-
predecessor_packet_digest: task.packet.packet_digest,
|
|
965
|
-
rebind_packet_digest: rebind.packet_digest,
|
|
966
|
-
new_write_lease_id: staged.lease_id,
|
|
967
|
-
supervisor_session_nonce_digest: sessionNonceDigest,
|
|
968
|
-
ack_digest: '',
|
|
969
|
-
}, 'ack_digest');
|
|
970
|
-
return sign({
|
|
971
|
-
schema: 'lattice.adapter_rebind_response.v1',
|
|
972
|
-
request_id: request.request_id,
|
|
973
|
-
rebind_ack: ack,
|
|
974
|
-
staged_lease_digest: staged.lease_digest,
|
|
975
|
-
response_digest: '',
|
|
976
|
-
}, 'response_digest');
|
|
977
|
-
}
|
|
978
|
-
if (operation === 'revoke') {
|
|
979
|
-
const available = new Set([...stagedLeases.keys(), ...armedLeases.keys()]);
|
|
980
|
-
if (request.lease_digests.some((digest) => !available.has(digest))) {
|
|
981
|
-
fail('WORK_ORDER_REVOKE_REJECTED', 'revokeが未知のleaseを含む');
|
|
982
|
-
}
|
|
983
|
-
for (const leaseDigest of request.lease_digests) {
|
|
984
|
-
stagedLeases.delete(leaseDigest);
|
|
985
|
-
armedLeases.delete(leaseDigest);
|
|
986
|
-
}
|
|
987
|
-
for (const task of tasks.values()) {
|
|
988
|
-
if (request.lease_digests.includes(task.lease.lease_digest)) task.state = 'held';
|
|
989
|
-
}
|
|
990
|
-
return sign({
|
|
991
|
-
schema: 'lattice.adapter_revoke_response.v1',
|
|
992
|
-
request_id: request.request_id,
|
|
993
|
-
revoked_lease_digests: [...request.lease_digests],
|
|
994
|
-
residual_processes: [],
|
|
995
|
-
response_digest: '',
|
|
996
|
-
}, 'response_digest');
|
|
997
|
-
}
|
|
998
|
-
fail('WORK_ORDER_UNKNOWN_OPERATION', `未知のcontroller operation: ${operation}`);
|
|
999
|
-
};
|
|
1000
|
-
|
|
1001
|
-
return Object.freeze({
|
|
1002
|
-
bootstrap: structuredClone(bootstrap),
|
|
1003
|
-
controllerId,
|
|
1004
|
-
controllerSessionNonce,
|
|
1005
|
-
descriptor: structuredClone(descriptor),
|
|
1006
|
-
heartbeat,
|
|
1007
|
-
reapWorkers,
|
|
1008
|
-
leaseSetDigest() {
|
|
1009
|
-
return digestArtifact([...stagedLeases.keys(), ...armedLeases.keys()].sort());
|
|
1010
|
-
},
|
|
1011
|
-
handshake(request) {
|
|
1012
|
-
if (!validateHandshakeRequest(request, bootstrap)) {
|
|
1013
|
-
fail('WORK_ORDER_HANDSHAKE_INVALID', 'handshake requestがbootstrapへbindしない');
|
|
1014
|
-
}
|
|
1015
|
-
return sign({
|
|
1016
|
-
schema: 'lattice.adapter_controller_handshake_response.v1',
|
|
1017
|
-
request_id: request.request_id,
|
|
1018
|
-
run_id: bootstrap.run_id,
|
|
1019
|
-
challenge_digest: digestArtifact(request.challenge),
|
|
1020
|
-
controller_session_nonce: controllerSessionNonce,
|
|
1021
|
-
descriptor,
|
|
1022
|
-
response_digest: '',
|
|
1023
|
-
}, 'response_digest');
|
|
1024
|
-
},
|
|
1025
|
-
async request(operation, request) {
|
|
1026
|
-
if (!CONTROLLER_OPERATIONS.includes(operation)) {
|
|
1027
|
-
fail('WORK_ORDER_UNKNOWN_OPERATION', `未知のcontroller operation: ${operation}`);
|
|
1028
|
-
}
|
|
1029
|
-
return responseFor(operation, request);
|
|
1030
|
-
},
|
|
1031
|
-
});
|
|
1032
|
-
}
|
|
1033
|
-
|
|
1034
|
-
async function readBootstrapFd(fd = 3) {
|
|
1035
|
-
const bytes = readFileSync(fd);
|
|
1036
|
-
if (bytes.length === 0 || bytes.length > MAX_DOCUMENT_BYTES) {
|
|
1037
|
-
fail('WORK_ORDER_BOOTSTRAP_INVALID', 'bootstrap sizeが不正');
|
|
1038
|
-
}
|
|
1039
|
-
const text = bytes.toString('utf8');
|
|
1040
|
-
if (!text.endsWith('\n') || text.indexOf('\n') !== text.length - 1) {
|
|
1041
|
-
fail('WORK_ORDER_BOOTSTRAP_INVALID', 'bootstrapはJSON 1 documentでなければならない');
|
|
1042
|
-
}
|
|
1043
|
-
let value;
|
|
1044
|
-
try {
|
|
1045
|
-
value = JSON.parse(text);
|
|
1046
|
-
} catch {
|
|
1047
|
-
fail('WORK_ORDER_BOOTSTRAP_INVALID', 'bootstrap JSONが不正');
|
|
1048
|
-
}
|
|
1049
|
-
return value;
|
|
1050
|
-
}
|
|
1051
|
-
|
|
1052
|
-
export async function runWorkOrderAdapterController({
|
|
1053
|
-
runDir = process.cwd(),
|
|
1054
|
-
bootstrap = null,
|
|
1055
|
-
stderr = process.stderr,
|
|
1056
|
-
} = {}) {
|
|
1057
|
-
const resolvedBootstrap = bootstrap ?? await readBootstrapFd();
|
|
1058
|
-
const canonicalRunDir = await realpath(runDir);
|
|
1059
|
-
const repoRoot = await realpath(path.resolve(canonicalRunDir, '..', '..', '..'));
|
|
1060
|
-
const controller = await createWorkOrderAdapterController({
|
|
1061
|
-
bootstrap: resolvedBootstrap,
|
|
1062
|
-
runDir: canonicalRunDir,
|
|
1063
|
-
repoRoot,
|
|
1064
|
-
});
|
|
1065
|
-
const socketPath = path.join(canonicalRunDir, resolvedBootstrap.controller_socket_ref);
|
|
1066
|
-
const controllerDirectory = path.dirname(socketPath);
|
|
1067
|
-
await mkdir(controllerDirectory, { recursive: true, mode: 0o700 });
|
|
1068
|
-
await chmod(controllerDirectory, 0o700);
|
|
1069
|
-
let heartbeatTimer = null;
|
|
1070
|
-
let persistentSocket = null;
|
|
1071
|
-
let heartbeatSequence = 0;
|
|
1072
|
-
let registrationDigest = null;
|
|
1073
|
-
const send = (socket, document) => {
|
|
1074
|
-
socket.write(`${canonicalizeArtifact(document)}\n`);
|
|
1075
|
-
};
|
|
1076
|
-
const startHeartbeat = (socket, digest) => {
|
|
1077
|
-
persistentSocket = socket;
|
|
1078
|
-
registrationDigest = digest;
|
|
1079
|
-
if (heartbeatTimer !== null) return;
|
|
1080
|
-
heartbeatTimer = setInterval(() => {
|
|
1081
|
-
if (persistentSocket === null || persistentSocket.destroyed) return;
|
|
1082
|
-
heartbeatSequence += 1;
|
|
1083
|
-
send(persistentSocket, sign({
|
|
1084
|
-
schema: 'lattice.adapter_controller_heartbeat.v1',
|
|
1085
|
-
controller_id: controller.controllerId,
|
|
1086
|
-
registration_digest: registrationDigest,
|
|
1087
|
-
supervisor_session_nonce: resolvedBootstrap.supervisor_session_nonce,
|
|
1088
|
-
sequence: heartbeatSequence,
|
|
1089
|
-
lease_set_digest: controller.leaseSetDigest(),
|
|
1090
|
-
heartbeat_digest: '',
|
|
1091
|
-
}, 'heartbeat_digest'));
|
|
1092
|
-
}, controller.heartbeat.interval_ms);
|
|
1093
|
-
heartbeatTimer.unref();
|
|
1094
|
-
};
|
|
1095
|
-
const server = net.createServer((socket) => {
|
|
1096
|
-
socket.setEncoding('utf8');
|
|
1097
|
-
let buffer = '';
|
|
1098
|
-
let processing = Promise.resolve();
|
|
1099
|
-
socket.on('data', (chunk) => {
|
|
1100
|
-
buffer += chunk;
|
|
1101
|
-
if (Buffer.byteLength(buffer, 'utf8') > MAX_DOCUMENT_BYTES) {
|
|
1102
|
-
send(socket, controllerErrorArtifact(
|
|
1103
|
-
new WorkOrderAdapterControllerError(
|
|
1104
|
-
'WORK_ORDER_DOCUMENT_TOO_LARGE',
|
|
1105
|
-
'controller document上限を超えた',
|
|
1106
|
-
),
|
|
1107
|
-
));
|
|
1108
|
-
socket.destroy();
|
|
1109
|
-
return;
|
|
1110
|
-
}
|
|
1111
|
-
while (buffer.includes('\n')) {
|
|
1112
|
-
const newline = buffer.indexOf('\n');
|
|
1113
|
-
const line = buffer.slice(0, newline);
|
|
1114
|
-
buffer = buffer.slice(newline + 1);
|
|
1115
|
-
processing = processing.then(async () => {
|
|
1116
|
-
let request;
|
|
1117
|
-
try {
|
|
1118
|
-
request = JSON.parse(line);
|
|
1119
|
-
if (request.schema === 'lattice.adapter_controller_handshake_request.v1') {
|
|
1120
|
-
send(socket, controller.handshake(request));
|
|
1121
|
-
return;
|
|
1122
|
-
}
|
|
1123
|
-
const operation = REQUEST_SCHEMA_TO_OPERATION[request.schema];
|
|
1124
|
-
if (operation === undefined) {
|
|
1125
|
-
fail('WORK_ORDER_UNKNOWN_OPERATION', `未知のrequest schema: ${String(request.schema)}`);
|
|
1126
|
-
}
|
|
1127
|
-
startHeartbeat(socket, request.registration_digest);
|
|
1128
|
-
const response = await controller.request(operation, request);
|
|
1129
|
-
send(socket, response);
|
|
1130
|
-
} catch (error) {
|
|
1131
|
-
send(socket, controllerErrorArtifact(error, request?.request_id));
|
|
1132
|
-
stderr.write(`${canonicalizeArtifact(controllerErrorArtifact(error, request?.request_id))}\n`);
|
|
1133
|
-
socket.destroy();
|
|
1134
|
-
}
|
|
1135
|
-
});
|
|
1136
|
-
}
|
|
1137
|
-
});
|
|
1138
|
-
socket.on('close', () => {
|
|
1139
|
-
if (socket === persistentSocket) {
|
|
1140
|
-
clearInterval(heartbeatTimer);
|
|
1141
|
-
heartbeatTimer = null;
|
|
1142
|
-
persistentSocket = null;
|
|
1143
|
-
}
|
|
1144
|
-
});
|
|
1145
|
-
});
|
|
1146
|
-
server.on('error', (error) => {
|
|
1147
|
-
stderr.write(`${canonicalizeArtifact(controllerErrorArtifact(error))}\n`);
|
|
1148
|
-
});
|
|
1149
|
-
await new Promise((resolve, reject) => {
|
|
1150
|
-
server.once('error', reject);
|
|
1151
|
-
// AF_UNIXのpath長上限へ一時repoのabsolute prefixを混入させない。supervisorも
|
|
1152
|
-
// run storeをcwdにして同じ固定relative refへ接続する。
|
|
1153
|
-
server.listen(resolvedBootstrap.controller_socket_ref, () => {
|
|
1154
|
-
server.off('error', reject);
|
|
1155
|
-
resolve();
|
|
1156
|
-
});
|
|
1157
|
-
});
|
|
1158
|
-
await chmod(socketPath, 0o600);
|
|
1159
|
-
let closed = false;
|
|
1160
|
-
const close = async () => {
|
|
1161
|
-
if (closed) return;
|
|
1162
|
-
closed = true;
|
|
1163
|
-
// 外部worker poolはadapter側が所有する。controllerはsocketだけを畳む。
|
|
1164
|
-
controller.reapWorkers();
|
|
1165
|
-
clearInterval(heartbeatTimer);
|
|
1166
|
-
persistentSocket?.destroy();
|
|
1167
|
-
await new Promise((resolve) => server.close(resolve));
|
|
1168
|
-
await rm(socketPath, { force: true });
|
|
1169
|
-
};
|
|
1170
|
-
return Object.freeze({ controller, socketPath, close });
|
|
1171
|
-
}
|
|
1
|
+
import net from 'node:net';
|
|
2
|
+
import { createHash, randomBytes } from 'node:crypto';
|
|
3
|
+
import { execFile } from 'node:child_process';
|
|
4
|
+
import { constants as fsConstants, readFileSync } from 'node:fs';
|
|
5
|
+
import {
|
|
6
|
+
chmod,
|
|
7
|
+
lstat,
|
|
8
|
+
mkdir,
|
|
9
|
+
open,
|
|
10
|
+
readFile,
|
|
11
|
+
realpath,
|
|
12
|
+
rename,
|
|
13
|
+
rm,
|
|
14
|
+
} from 'node:fs/promises';
|
|
15
|
+
import path from 'node:path';
|
|
16
|
+
import { promisify } from 'node:util';
|
|
17
|
+
|
|
18
|
+
import { canonicalizeArtifact, digestArtifact } from './artifact-contracts.mjs';
|
|
19
|
+
import {
|
|
20
|
+
armStagedWriteLease,
|
|
21
|
+
CONTROLLER_OPERATIONS,
|
|
22
|
+
validateArmedWriteLease,
|
|
23
|
+
validateControllerRequest,
|
|
24
|
+
validateRuntimeAdapterCapabilities,
|
|
25
|
+
validateRuntimeHeartbeatPolicy,
|
|
26
|
+
} from './runtime-controller-protocol.mjs';
|
|
27
|
+
import {
|
|
28
|
+
selfDigest,
|
|
29
|
+
validateExecutorPacket,
|
|
30
|
+
validateExecutorReceipt,
|
|
31
|
+
} from './runtime-contracts.mjs';
|
|
32
|
+
import { captureWorktreeDiff } from './runtime-diff-observer.mjs';
|
|
33
|
+
import { validateSupervisorWriteGate } from './runtime-gate-store.mjs';
|
|
34
|
+
import { observeManagedProcessStartIdentity } from './runtime-managed-supervisor.mjs';
|
|
35
|
+
import {
|
|
36
|
+
scriptedWorktreeId as workOrderWorktreeId,
|
|
37
|
+
scriptedWorktreePath as workOrderWorktreePath,
|
|
38
|
+
} from './runtime-scripted-worktree.mjs';
|
|
39
|
+
import {
|
|
40
|
+
createRunWorkOrder,
|
|
41
|
+
validateRunWorkOrder,
|
|
42
|
+
validateRunWorkReport,
|
|
43
|
+
} from './runtime-work-order-contracts.mjs';
|
|
44
|
+
|
|
45
|
+
const MAX_DOCUMENT_BYTES = 8_388_608;
|
|
46
|
+
const SHA256 = /^[0-9a-f]{64}$/u;
|
|
47
|
+
const ID = /^[0-9A-Za-z](?:[0-9A-Za-z._-]{0,127})$/u;
|
|
48
|
+
const execFileAsync = promisify(execFile);
|
|
49
|
+
const REQUEST_SCHEMA_TO_OPERATION = Object.freeze(Object.fromEntries([
|
|
50
|
+
['lattice.adapter_dispatch_request.v1', 'dispatch'],
|
|
51
|
+
['lattice.adapter_observe_request.v1', 'observe'],
|
|
52
|
+
['lattice.adapter_running_inventory_request.v1', 'inventory'],
|
|
53
|
+
['lattice.adapter_barrier_request.v1', 'barrier'],
|
|
54
|
+
['lattice.adapter_rebind_request.v1', 'rebind'],
|
|
55
|
+
['lattice.adapter_prepare_request.v1', 'prepare'],
|
|
56
|
+
['lattice.adapter_activate_request.v1', 'activate'],
|
|
57
|
+
['lattice.adapter_release_request.v1', 'release'],
|
|
58
|
+
['lattice.adapter_revoke_request.v1', 'revoke'],
|
|
59
|
+
]));
|
|
60
|
+
|
|
61
|
+
export class WorkOrderAdapterControllerError extends Error {
|
|
62
|
+
constructor(code, message, detail = {}) {
|
|
63
|
+
super(message);
|
|
64
|
+
this.name = 'WorkOrderAdapterControllerError';
|
|
65
|
+
this.code = code;
|
|
66
|
+
this.detail = detail;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function fail(code, message, detail) {
|
|
71
|
+
throw new WorkOrderAdapterControllerError(code, message, detail);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function plain(value) {
|
|
75
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
|
76
|
+
&& Object.getPrototypeOf(value) === Object.prototype;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function exact(value, fields) {
|
|
80
|
+
if (!plain(value)) return false;
|
|
81
|
+
const actual = Object.keys(value).sort();
|
|
82
|
+
const expected = [...fields].sort();
|
|
83
|
+
return actual.length === expected.length
|
|
84
|
+
&& actual.every((field, index) => field === expected[index]);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function sign(value, field) {
|
|
88
|
+
value[field] = '';
|
|
89
|
+
value[field] = selfDigest(value, field);
|
|
90
|
+
return value;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function sha256Bytes(bytes) {
|
|
94
|
+
return createHash('sha256').update(bytes).digest('hex');
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function controllerErrorArtifact(error, requestId = null) {
|
|
98
|
+
const artifact = {
|
|
99
|
+
schema: 'lattice.work_order_adapter_error.v1',
|
|
100
|
+
code: error?.code ?? 'WORK_ORDER_CONTROLLER_FAILED',
|
|
101
|
+
message: String(error?.message ?? error),
|
|
102
|
+
request_id: typeof requestId === 'string' ? requestId : null,
|
|
103
|
+
detail: plain(error?.detail) ? structuredClone(error.detail) : {},
|
|
104
|
+
error_digest: '',
|
|
105
|
+
};
|
|
106
|
+
artifact.error_digest = selfDigest(artifact, 'error_digest');
|
|
107
|
+
return artifact;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function validateBootstrap(value) {
|
|
111
|
+
return exact(value, [
|
|
112
|
+
'schema',
|
|
113
|
+
'request_id',
|
|
114
|
+
'run_id',
|
|
115
|
+
'controller_socket_ref',
|
|
116
|
+
'supervisor_socket_ref',
|
|
117
|
+
'supervisor_session_nonce',
|
|
118
|
+
'bootstrap_digest',
|
|
119
|
+
])
|
|
120
|
+
&& value.schema === 'lattice.adapter_controller_bootstrap.v1'
|
|
121
|
+
&& ID.test(value.request_id ?? '')
|
|
122
|
+
&& ID.test(value.run_id ?? '')
|
|
123
|
+
&& typeof value.controller_socket_ref === 'string'
|
|
124
|
+
&& /^supervisor\/controllers\/[0-9A-Za-z][0-9A-Za-z._-]{0,127}\.sock$/u
|
|
125
|
+
.test(value.controller_socket_ref)
|
|
126
|
+
&& value.supervisor_socket_ref === 'supervisor/control.sock'
|
|
127
|
+
&& typeof value.supervisor_session_nonce === 'string'
|
|
128
|
+
&& value.supervisor_session_nonce.length >= 32
|
|
129
|
+
&& SHA256.test(value.bootstrap_digest ?? '')
|
|
130
|
+
&& selfDigest(value, 'bootstrap_digest') === value.bootstrap_digest;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function validateHandshakeRequest(value, bootstrap) {
|
|
134
|
+
return exact(value, [
|
|
135
|
+
'schema',
|
|
136
|
+
'request_id',
|
|
137
|
+
'run_id',
|
|
138
|
+
'supervisor_session_nonce',
|
|
139
|
+
'controller_socket_ref',
|
|
140
|
+
'challenge',
|
|
141
|
+
'request_digest',
|
|
142
|
+
])
|
|
143
|
+
&& value.schema === 'lattice.adapter_controller_handshake_request.v1'
|
|
144
|
+
&& ID.test(value.request_id ?? '')
|
|
145
|
+
&& value.run_id === bootstrap.run_id
|
|
146
|
+
&& value.supervisor_session_nonce === bootstrap.supervisor_session_nonce
|
|
147
|
+
&& value.controller_socket_ref === bootstrap.controller_socket_ref
|
|
148
|
+
&& typeof value.challenge === 'string'
|
|
149
|
+
&& value.challenge.length >= 32
|
|
150
|
+
&& SHA256.test(value.request_digest ?? '')
|
|
151
|
+
&& selfDigest(value, 'request_digest') === value.request_digest;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
async function durableReplaceBytes(filePath, bytes, mode = 0o600) {
|
|
155
|
+
const directory = path.dirname(filePath);
|
|
156
|
+
await mkdir(directory, { recursive: true, mode: 0o700 });
|
|
157
|
+
const temporaryPath = path.join(directory, `.${path.basename(filePath)}-${process.pid}.tmp`);
|
|
158
|
+
let handle;
|
|
159
|
+
try {
|
|
160
|
+
handle = await open(
|
|
161
|
+
temporaryPath,
|
|
162
|
+
fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_WRONLY,
|
|
163
|
+
mode,
|
|
164
|
+
);
|
|
165
|
+
await handle.writeFile(bytes);
|
|
166
|
+
await handle.sync();
|
|
167
|
+
await handle.close();
|
|
168
|
+
handle = null;
|
|
169
|
+
await rename(temporaryPath, filePath);
|
|
170
|
+
await chmod(filePath, mode);
|
|
171
|
+
const directoryHandle = await open(directory, fsConstants.O_RDONLY);
|
|
172
|
+
// Windowsはdirectory handleのfsyncを許さず常にEPERM/EINVALを返す(Node仕様)。
|
|
173
|
+
// win32のこの2値だけ許容し、他OS・他エラーは従来どおり失敗させる。
|
|
174
|
+
try {
|
|
175
|
+
await directoryHandle.sync();
|
|
176
|
+
} catch (error) {
|
|
177
|
+
if (process.platform !== 'win32' || !['EPERM', 'EINVAL'].includes(error?.code)) throw error;
|
|
178
|
+
} finally {
|
|
179
|
+
await directoryHandle.close();
|
|
180
|
+
}
|
|
181
|
+
} finally {
|
|
182
|
+
await handle?.close().catch(() => {});
|
|
183
|
+
await rm(temporaryPath, { force: true }).catch(() => {});
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
async function readAndValidateGate(runDir, writeLease) {
|
|
188
|
+
const gatePath = path.join(runDir, 'supervisor', 'write-gate.json');
|
|
189
|
+
let gate;
|
|
190
|
+
try {
|
|
191
|
+
gate = JSON.parse(await readFile(gatePath, 'utf8'));
|
|
192
|
+
} catch (error) {
|
|
193
|
+
fail('WORK_ORDER_WRITE_GATE_REJECTED', '中央write gateを読めない', {
|
|
194
|
+
message: String(error?.message ?? error),
|
|
195
|
+
});
|
|
196
|
+
}
|
|
197
|
+
if (!validateSupervisorWriteGate(gate)
|
|
198
|
+
|| gate.run_id !== writeLease.run_id
|
|
199
|
+
|| gate.plan_epoch !== writeLease.plan_epoch
|
|
200
|
+
|| gate.gate_generation !== writeLease.gate_generation
|
|
201
|
+
|| gate.release_barrier_digest !== writeLease.release_barrier_digest
|
|
202
|
+
|| !gate.armed_lease_digests.includes(writeLease.lease_digest)) {
|
|
203
|
+
fail('WORK_ORDER_WRITE_GATE_REJECTED', '中央write gateがarmed leaseを認可しない');
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** 登録時のdigestへ束縛されたwork-order spool設定を読む。 */
|
|
208
|
+
async function readWorkOrderBehavior(repoRoot, capabilitiesDigest) {
|
|
209
|
+
const descriptorPath = path.join(repoRoot, '.lattice', 'runtime', 'adapter-registry',
|
|
210
|
+
'descriptors', 'work-order.json');
|
|
211
|
+
let descriptor;
|
|
212
|
+
try {
|
|
213
|
+
descriptor = JSON.parse(await readFile(descriptorPath, 'utf8'));
|
|
214
|
+
} catch { fail('WORK_ORDER_BOOTSTRAP_INVALID', 'work-order adapterが未登録'); }
|
|
215
|
+
if (typeof descriptor?.config_ref !== 'string' || typeof descriptor.config_digest !== 'string') {
|
|
216
|
+
fail('WORK_ORDER_BOOTSTRAP_INVALID', 'work-order adapter descriptorが不正');
|
|
217
|
+
}
|
|
218
|
+
if (descriptor.capabilities_digest !== capabilitiesDigest) {
|
|
219
|
+
fail(
|
|
220
|
+
'WORK_ORDER_CAPABILITIES_MISMATCH',
|
|
221
|
+
'work-order adapterをregistration input v2で再登録する必要がある',
|
|
222
|
+
{
|
|
223
|
+
registered_capabilities_digest: descriptor.capabilities_digest ?? null,
|
|
224
|
+
controller_capabilities_digest: capabilitiesDigest,
|
|
225
|
+
},
|
|
226
|
+
);
|
|
227
|
+
}
|
|
228
|
+
const configPath = path.join(repoRoot, ...descriptor.config_ref.split('/'));
|
|
229
|
+
let bytes;
|
|
230
|
+
try {
|
|
231
|
+
bytes = await readFile(configPath);
|
|
232
|
+
} catch {
|
|
233
|
+
fail('WORK_ORDER_BOOTSTRAP_INVALID', '登録済みadapter configを読めない', {
|
|
234
|
+
config_ref: descriptor.config_ref,
|
|
235
|
+
});
|
|
236
|
+
}
|
|
237
|
+
if (sha256Bytes(bytes) !== descriptor.config_digest) {
|
|
238
|
+
fail('WORK_ORDER_BOOTSTRAP_INVALID', 'adapter configが登録時のdigestと一致しない', {
|
|
239
|
+
config_ref: descriptor.config_ref,
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
let config;
|
|
243
|
+
try {
|
|
244
|
+
config = JSON.parse(bytes.toString('utf8'));
|
|
245
|
+
} catch {
|
|
246
|
+
fail('WORK_ORDER_BOOTSTRAP_INVALID', 'adapter configのJSONが不正');
|
|
247
|
+
}
|
|
248
|
+
if (!exact(config, ['schema', 'spool_dir'])
|
|
249
|
+
|| config.schema !== 'lattice.work_order_adapter_config.v1'
|
|
250
|
+
|| typeof config.spool_dir !== 'string'
|
|
251
|
+
|| !path.isAbsolute(config.spool_dir)) {
|
|
252
|
+
fail('WORK_ORDER_BOOTSTRAP_INVALID', 'work-order adapter configがexact contractを満たさない');
|
|
253
|
+
}
|
|
254
|
+
let spoolDir;
|
|
255
|
+
try { spoolDir = await realpath(config.spool_dir); }
|
|
256
|
+
catch { fail('WORK_ORDER_BOOTSTRAP_INVALID', 'spool_dirを解決できない'); }
|
|
257
|
+
const info = await lstat(spoolDir);
|
|
258
|
+
if (!info.isDirectory() || info.isSymbolicLink()) {
|
|
259
|
+
fail('WORK_ORDER_BOOTSTRAP_INVALID', 'spool_dirがreal directoryでない');
|
|
260
|
+
}
|
|
261
|
+
for (const child of ['orders', 'reports']) {
|
|
262
|
+
const directory = path.join(spoolDir, child);
|
|
263
|
+
await mkdir(directory, { recursive: true, mode: 0o700 });
|
|
264
|
+
await chmod(directory, 0o700);
|
|
265
|
+
const childInfo = await lstat(directory);
|
|
266
|
+
if (!childInfo.isDirectory() || childInfo.isSymbolicLink()
|
|
267
|
+
|| await realpath(directory) !== directory) {
|
|
268
|
+
fail('WORK_ORDER_BOOTSTRAP_INVALID', `${child} directoryがcanonicalでない`);
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
return { spoolDir };
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
const sleep = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
|
|
275
|
+
|
|
276
|
+
async function readWorkReport(reportPath, packetDigest) {
|
|
277
|
+
let info;
|
|
278
|
+
try { info = await lstat(reportPath); }
|
|
279
|
+
catch (error) {
|
|
280
|
+
if (error?.code === 'ENOENT') return null;
|
|
281
|
+
throw error;
|
|
282
|
+
}
|
|
283
|
+
if (!info.isFile() || info.isSymbolicLink() || (info.mode & 0o077) !== 0
|
|
284
|
+
|| info.size <= 0 || info.size > MAX_DOCUMENT_BYTES
|
|
285
|
+
|| await realpath(reportPath) !== reportPath) {
|
|
286
|
+
fail('WORK_ORDER_REPORT_INVALID', 'work reportがprivate canonical regular fileでない');
|
|
287
|
+
}
|
|
288
|
+
let bytes;
|
|
289
|
+
try { bytes = await readFile(reportPath); }
|
|
290
|
+
catch (error) { throw error; }
|
|
291
|
+
let report;
|
|
292
|
+
try { report = JSON.parse(bytes.toString('utf8')); }
|
|
293
|
+
catch { fail('WORK_ORDER_REPORT_INVALID', 'work reportのJSONが不正'); }
|
|
294
|
+
if (!validateRunWorkReport(report)
|
|
295
|
+
|| report.packet_digest !== packetDigest
|
|
296
|
+
|| bytes.toString('utf8') !== `${canonicalizeArtifact(report)}\n`) {
|
|
297
|
+
fail('WORK_ORDER_REPORT_INVALID', 'work reportがpacketへexact bindしない');
|
|
298
|
+
}
|
|
299
|
+
return report;
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
async function waitForWorkReport(reportPath, packetDigest) {
|
|
303
|
+
for (;;) {
|
|
304
|
+
const report = await readWorkReport(reportPath, packetDigest);
|
|
305
|
+
if (report !== null) return report;
|
|
306
|
+
await sleep(50);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
function processIdentity(record) {
|
|
311
|
+
const identity = {
|
|
312
|
+
schema: 'lattice.process_start_identity.v1',
|
|
313
|
+
platform: process.platform,
|
|
314
|
+
pid: record.pid,
|
|
315
|
+
started_identity: record.startedIdentity,
|
|
316
|
+
identity_digest: '',
|
|
317
|
+
};
|
|
318
|
+
identity.identity_digest = selfDigest(identity, 'identity_digest');
|
|
319
|
+
return identity;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
function quiescedState(rawState) {
|
|
323
|
+
if (rawState.startsWith('T')) return 'stopped';
|
|
324
|
+
if (rawState.startsWith('Z')) return 'zombie';
|
|
325
|
+
return null;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
async function observeWorkerProcessTree(pid, { requireDescendantsInRootGroup = true } = {}) {
|
|
329
|
+
let stdout;
|
|
330
|
+
try {
|
|
331
|
+
({ stdout } = await execFileAsync('/bin/ps', [
|
|
332
|
+
'-axo', 'pid=,ppid=,pgid=,state=,lstart=',
|
|
333
|
+
], {
|
|
334
|
+
encoding: 'utf8',
|
|
335
|
+
maxBuffer: 8 * 1024 * 1024,
|
|
336
|
+
}));
|
|
337
|
+
} catch { fail('WORK_ORDER_REPORT_INVALID', 'worker_pidをOS観測できない'); }
|
|
338
|
+
const records = new Map();
|
|
339
|
+
for (const rawLine of stdout.split('\n')) {
|
|
340
|
+
const line = rawLine.trim();
|
|
341
|
+
if (line.length === 0) continue;
|
|
342
|
+
const match = line.match(/^(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.+)$/u);
|
|
343
|
+
if (!match) fail('WORK_ORDER_REPORT_INVALID', 'ps観測結果を解析できない');
|
|
344
|
+
const record = {
|
|
345
|
+
pid: Number(match[1]),
|
|
346
|
+
parentPid: Number(match[2]),
|
|
347
|
+
processGroupId: Number(match[3]),
|
|
348
|
+
rawState: match[4],
|
|
349
|
+
startedIdentity: match[5].trim(),
|
|
350
|
+
};
|
|
351
|
+
if (![record.pid, record.processGroupId].every((value) => Number.isSafeInteger(value)
|
|
352
|
+
&& value > 0) || !Number.isSafeInteger(record.parentPid) || record.parentPid < 0
|
|
353
|
+
|| record.startedIdentity.length === 0 || records.has(record.pid)) {
|
|
354
|
+
fail('WORK_ORDER_REPORT_INVALID', 'ps process recordが不正');
|
|
355
|
+
}
|
|
356
|
+
records.set(record.pid, record);
|
|
357
|
+
}
|
|
358
|
+
const root = records.get(pid);
|
|
359
|
+
if (root === undefined) fail('WORK_ORDER_REPORT_INVALID', 'worker_pidが存在しない');
|
|
360
|
+
if (root.processGroupId !== root.pid) {
|
|
361
|
+
fail('WORK_ORDER_REPORT_INVALID', 'worker rootがprocess group leaderでない');
|
|
362
|
+
}
|
|
363
|
+
const descendantPids = new Set();
|
|
364
|
+
let changed = true;
|
|
365
|
+
while (changed) {
|
|
366
|
+
changed = false;
|
|
367
|
+
for (const record of records.values()) {
|
|
368
|
+
if (record.pid !== pid && !descendantPids.has(record.pid)
|
|
369
|
+
&& (record.parentPid === pid || descendantPids.has(record.parentPid))) {
|
|
370
|
+
descendantPids.add(record.pid);
|
|
371
|
+
changed = true;
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
const descendants = [...descendantPids].map((childPid) => records.get(childPid))
|
|
376
|
+
.sort((left, right) => left.pid - right.pid);
|
|
377
|
+
if (requireDescendantsInRootGroup
|
|
378
|
+
&& descendants.some((record) => record.processGroupId !== root.processGroupId)) {
|
|
379
|
+
fail('WORK_ORDER_REPORT_INVALID', 'worker childがrootと別process groupに居る');
|
|
380
|
+
}
|
|
381
|
+
const expectedGroup = new Set([pid, ...descendantPids]);
|
|
382
|
+
if ([...records.values()].some((record) => record.processGroupId === root.processGroupId
|
|
383
|
+
&& !expectedGroup.has(record.pid))) {
|
|
384
|
+
fail('WORK_ORDER_REPORT_INVALID', 'worker process groupを無関係processと共有している');
|
|
385
|
+
}
|
|
386
|
+
return {
|
|
387
|
+
pid: root.pid,
|
|
388
|
+
parentPid: root.parentPid,
|
|
389
|
+
processGroupId: root.processGroupId,
|
|
390
|
+
processStartIdentity: processIdentity(root),
|
|
391
|
+
rawState: root.rawState,
|
|
392
|
+
observationChildren: descendants.map((record) => ({
|
|
393
|
+
pid: record.pid,
|
|
394
|
+
parent_pid: record.parentPid,
|
|
395
|
+
process_start_identity_digest: processIdentity(record).identity_digest,
|
|
396
|
+
process_group_id: record.processGroupId,
|
|
397
|
+
state: quiescedState(record.rawState),
|
|
398
|
+
})),
|
|
399
|
+
};
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
export const runtimeWorkOrderControllerInternal = Object.freeze({
|
|
403
|
+
observeWorkerProcessTree,
|
|
404
|
+
});
|
|
405
|
+
|
|
406
|
+
/** orderを書き、bridgeのreportを状態遷移の合図として待つ。 */
|
|
407
|
+
export async function spawnWorkOrderWorker({ packet, worktreePath, spoolDir }) {
|
|
408
|
+
const order = createRunWorkOrder({ packet, worktreePath });
|
|
409
|
+
if (!validateRunWorkOrder(order)) fail('WORK_ORDER_ORDER_INVALID', 'work order生成に失敗した');
|
|
410
|
+
const orderPath = path.join(spoolDir, 'orders', `${packet.packet_digest}.json`);
|
|
411
|
+
const reportPath = path.join(spoolDir, 'reports', `${packet.packet_digest}.json`);
|
|
412
|
+
// spoolはrunを跨いで再利用される。古いdoneを新dispatchの完了として読まないよう、
|
|
413
|
+
// reportを消してからorderをpublishする(bridgeはorderのrename後にしか動かない)。
|
|
414
|
+
await rm(reportPath, { force: true });
|
|
415
|
+
await durableReplaceBytes(orderPath, Buffer.from(`${canonicalizeArtifact(order)}\n`));
|
|
416
|
+
// 辞退はbridge内で再配車し、受諾した席が決まるまでreportを出さない。
|
|
417
|
+
// controller側の任意timeoutでactivateを失敗させず、実席pidだけをimmutable bindする。
|
|
418
|
+
const firstReport = await waitForWorkReport(reportPath, packet.packet_digest);
|
|
419
|
+
// dispatch時は席が稼働中で、一過性tool childが別PGIDへ出る。ここで束縛するのは
|
|
420
|
+
// 停止対象rootのstart identityとPGIDだけ。全子孫の同一PGID・静止はbarrierで検証する。
|
|
421
|
+
const processTree = await observeWorkerProcessTree(firstReport.worker_pid, {
|
|
422
|
+
requireDescendantsInRootGroup: false,
|
|
423
|
+
});
|
|
424
|
+
const completed = (async () => {
|
|
425
|
+
let report = firstReport;
|
|
426
|
+
while (report.state !== 'done') {
|
|
427
|
+
await sleep(50);
|
|
428
|
+
report = await waitForWorkReport(reportPath, packet.packet_digest);
|
|
429
|
+
if (report.worker_pid !== firstReport.worker_pid) {
|
|
430
|
+
fail('WORK_ORDER_REPORT_INVALID', 'work reportのworker_pidが途中で変化した');
|
|
431
|
+
}
|
|
432
|
+
}
|
|
433
|
+
const checkpoint = await captureWorktreeDiff({
|
|
434
|
+
worktreePath,
|
|
435
|
+
baseSha: packet.base_sha,
|
|
436
|
+
});
|
|
437
|
+
return {
|
|
438
|
+
observedDiff: checkpoint.diff.entries.map(({ path: changedPath, change }) => ({
|
|
439
|
+
path: changedPath,
|
|
440
|
+
change,
|
|
441
|
+
})),
|
|
442
|
+
checkpointDigest: checkpoint.checkpoint_digest,
|
|
443
|
+
};
|
|
444
|
+
})();
|
|
445
|
+
return {
|
|
446
|
+
pid: firstReport.worker_pid,
|
|
447
|
+
process_group_id: processTree.processGroupId,
|
|
448
|
+
process_start_identity: processTree.processStartIdentity,
|
|
449
|
+
process_membership: 'dynamic_group',
|
|
450
|
+
completed,
|
|
451
|
+
};
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
async function quiesceWorkOrderTask(task) {
|
|
455
|
+
try {
|
|
456
|
+
process.kill(-task.worker.process_group_id, 'SIGSTOP');
|
|
457
|
+
} catch (error) {
|
|
458
|
+
fail('WORK_ORDER_BARRIER_REJECTED', 'worker process groupを止められない', {
|
|
459
|
+
pid: task.worker.pid, reason: String(error?.code ?? error?.message ?? error),
|
|
460
|
+
});
|
|
461
|
+
}
|
|
462
|
+
try {
|
|
463
|
+
let processTree = null;
|
|
464
|
+
for (let attempt = 0; attempt < 100; attempt += 1) {
|
|
465
|
+
processTree = await observeWorkerProcessTree(task.worker.pid);
|
|
466
|
+
if (quiescedState(processTree.rawState) !== null
|
|
467
|
+
&& processTree.observationChildren.every((child) => child.state !== null)) break;
|
|
468
|
+
processTree = null;
|
|
469
|
+
await sleep(10);
|
|
470
|
+
}
|
|
471
|
+
if (processTree === null
|
|
472
|
+
|| processTree.processGroupId !== task.worker.process_group_id
|
|
473
|
+
|| processTree.processStartIdentity.identity_digest
|
|
474
|
+
!== task.worker.process_start_identity.identity_digest) {
|
|
475
|
+
fail('WORK_ORDER_BARRIER_REJECTED', 'worker process集合を静止・再認証できない');
|
|
476
|
+
}
|
|
477
|
+
const checkpoint = await captureWorktreeDiff({
|
|
478
|
+
worktreePath: task.worktreePath, baseSha: task.packet.base_sha,
|
|
479
|
+
});
|
|
480
|
+
return {
|
|
481
|
+
checkpoint,
|
|
482
|
+
processObservationDigest: digestArtifact({
|
|
483
|
+
schema: 'lattice.direct_process_observation.v2',
|
|
484
|
+
root: {
|
|
485
|
+
pid: task.worker.pid,
|
|
486
|
+
parent_pid: processTree.parentPid,
|
|
487
|
+
process_start_identity_digest: task.worker.process_start_identity.identity_digest,
|
|
488
|
+
process_group_id: task.worker.process_group_id,
|
|
489
|
+
state: quiescedState(processTree.rawState),
|
|
490
|
+
},
|
|
491
|
+
children: processTree.observationChildren,
|
|
492
|
+
process_group_id: task.worker.process_group_id,
|
|
493
|
+
quiesced: true,
|
|
494
|
+
}),
|
|
495
|
+
worktreeFingerprintDigest: digestArtifact({
|
|
496
|
+
schema: 'lattice.direct_worktree_fingerprint.v1',
|
|
497
|
+
worktree_id: task.worktreeId,
|
|
498
|
+
worktree_realpath: task.worktreePath,
|
|
499
|
+
checkpoint_digest: checkpoint.checkpoint_digest,
|
|
500
|
+
}),
|
|
501
|
+
};
|
|
502
|
+
} catch (error) {
|
|
503
|
+
try { process.kill(-task.worker.process_group_id, 'SIGCONT'); }
|
|
504
|
+
catch { /* 元のbarrier失敗を保持する */ }
|
|
505
|
+
task.state = 'running';
|
|
506
|
+
throw error;
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
export function createWorkOrderAdapterCapabilities() {
|
|
511
|
+
const capabilities = sign({
|
|
512
|
+
schema: 'lattice.runtime_adapter_capabilities.v2',
|
|
513
|
+
operations: [...CONTROLLER_OPERATIONS],
|
|
514
|
+
process_observation: true,
|
|
515
|
+
worktree_fingerprint: true,
|
|
516
|
+
staged_write_lease: true,
|
|
517
|
+
durable_dispatch: true,
|
|
518
|
+
host_driven_epoch: true,
|
|
519
|
+
capabilities_digest: '',
|
|
520
|
+
}, 'capabilities_digest');
|
|
521
|
+
if (!validateRuntimeAdapterCapabilities(capabilities)) {
|
|
522
|
+
fail('WORK_ORDER_CONTROLLER_INVALID', 'capabilities生成に失敗した');
|
|
523
|
+
}
|
|
524
|
+
return capabilities;
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
export async function createWorkOrderAdapterController({
|
|
528
|
+
bootstrap,
|
|
529
|
+
runDir,
|
|
530
|
+
repoRoot,
|
|
531
|
+
controllerSessionNonce = randomBytes(32).toString('hex'),
|
|
532
|
+
}) {
|
|
533
|
+
if (!validateBootstrap(bootstrap)) {
|
|
534
|
+
fail('WORK_ORDER_BOOTSTRAP_INVALID', 'controller bootstrapが不正');
|
|
535
|
+
}
|
|
536
|
+
const canonicalRunDir = await realpath(runDir);
|
|
537
|
+
const canonicalRepoRoot = await realpath(repoRoot);
|
|
538
|
+
if (path.resolve(canonicalRunDir, '..', '..', '..') !== canonicalRepoRoot) {
|
|
539
|
+
fail('WORK_ORDER_BOOTSTRAP_INVALID', 'run storeとrepo rootのbindingが不正');
|
|
540
|
+
}
|
|
541
|
+
const controllerId = path.basename(bootstrap.controller_socket_ref, '.sock');
|
|
542
|
+
if (!controllerId.startsWith('work-order-')
|
|
543
|
+
|| typeof controllerSessionNonce !== 'string'
|
|
544
|
+
|| controllerSessionNonce.length < 32) {
|
|
545
|
+
fail('WORK_ORDER_BOOTSTRAP_INVALID', 'work-order controller identityが不正');
|
|
546
|
+
}
|
|
547
|
+
const capabilities = createWorkOrderAdapterCapabilities();
|
|
548
|
+
const heartbeat = sign({
|
|
549
|
+
schema: 'lattice.runtime_heartbeat_policy.v1',
|
|
550
|
+
interval_ms: 1_000,
|
|
551
|
+
ttl_ms: 10_000,
|
|
552
|
+
disconnect_revokes_immediately: true,
|
|
553
|
+
policy_digest: '',
|
|
554
|
+
}, 'policy_digest');
|
|
555
|
+
if (!validateRuntimeHeartbeatPolicy(heartbeat)) {
|
|
556
|
+
fail('WORK_ORDER_CONTROLLER_INVALID', 'heartbeat policy生成に失敗した');
|
|
557
|
+
}
|
|
558
|
+
const descriptor = sign({
|
|
559
|
+
schema: 'lattice.runtime_adapter_controller_descriptor.v1',
|
|
560
|
+
controller_id: controllerId,
|
|
561
|
+
adapter_kind: 'work-order',
|
|
562
|
+
pid: process.pid,
|
|
563
|
+
process_start_identity: await observeManagedProcessStartIdentity(process.pid),
|
|
564
|
+
socket_ref: bootstrap.controller_socket_ref,
|
|
565
|
+
controller_session_nonce_digest: digestArtifact(controllerSessionNonce),
|
|
566
|
+
capabilities,
|
|
567
|
+
heartbeat,
|
|
568
|
+
descriptor_digest: '',
|
|
569
|
+
}, 'descriptor_digest');
|
|
570
|
+
const sessionNonceDigest = digestArtifact(bootstrap.supervisor_session_nonce);
|
|
571
|
+
const behavior = await readWorkOrderBehavior(
|
|
572
|
+
canonicalRepoRoot,
|
|
573
|
+
capabilities.capabilities_digest,
|
|
574
|
+
);
|
|
575
|
+
const stagedLeases = new Map();
|
|
576
|
+
const armedLeases = new Map();
|
|
577
|
+
const preparedPackets = new Map();
|
|
578
|
+
const tasks = new Map();
|
|
579
|
+
const todoToHandle = new Map();
|
|
580
|
+
let currentEpoch = 1;
|
|
581
|
+
let registrationDigest = null;
|
|
582
|
+
|
|
583
|
+
// worker poolのprocess寿命は外部adapterが所有する。controller closeで席を殺さない。
|
|
584
|
+
const reapWorkers = () => {};
|
|
585
|
+
|
|
586
|
+
const workerProcessOf = (task) => ({
|
|
587
|
+
pid: task.worker.pid,
|
|
588
|
+
process_group_id: task.worker.process_group_id,
|
|
589
|
+
process_start_identity: structuredClone(task.worker.process_start_identity),
|
|
590
|
+
process_membership: task.worker.process_membership,
|
|
591
|
+
});
|
|
592
|
+
|
|
593
|
+
const persistReceipt = async (receipt) => {
|
|
594
|
+
const payloadDigest = digestArtifact(receipt);
|
|
595
|
+
const receiptPath = path.join(
|
|
596
|
+
canonicalRunDir,
|
|
597
|
+
'controllers',
|
|
598
|
+
controllerId,
|
|
599
|
+
'receipts',
|
|
600
|
+
`${payloadDigest}.json`,
|
|
601
|
+
);
|
|
602
|
+
await durableReplaceBytes(
|
|
603
|
+
receiptPath,
|
|
604
|
+
Buffer.from(`${canonicalizeArtifact(receipt)}\n`),
|
|
605
|
+
);
|
|
606
|
+
return payloadDigest;
|
|
607
|
+
};
|
|
608
|
+
|
|
609
|
+
const responseFor = async (operation, request) => {
|
|
610
|
+
if (!validateControllerRequest(operation, request)) {
|
|
611
|
+
fail('WORK_ORDER_REQUEST_INVALID', `${operation} requestがprotocol contractを満たさない`);
|
|
612
|
+
}
|
|
613
|
+
if (registrationDigest !== null && request.registration_digest !== registrationDigest) {
|
|
614
|
+
fail('WORK_ORDER_REGISTRATION_MISMATCH', 'controller registration digestが変化した');
|
|
615
|
+
}
|
|
616
|
+
registrationDigest = request.registration_digest;
|
|
617
|
+
if (operation === 'prepare') {
|
|
618
|
+
const packet = request.executor_packet;
|
|
619
|
+
const lease = request.staged_lease;
|
|
620
|
+
if (lease.run_id !== bootstrap.run_id || lease.todo_id !== packet.todo_id
|
|
621
|
+
|| lease.plan_epoch !== packet.plan_epoch || lease.packet_digest !== packet.packet_digest
|
|
622
|
+
|| lease.controller_registration_digest !== request.registration_digest
|
|
623
|
+
|| lease.supervisor_session_nonce_digest !== sessionNonceDigest) {
|
|
624
|
+
fail('WORK_ORDER_LEASE_REJECTED', 'prepare packetとstaged leaseのbindingが不正');
|
|
625
|
+
}
|
|
626
|
+
const prior = stagedLeases.get(lease.lease_digest);
|
|
627
|
+
if (prior !== undefined
|
|
628
|
+
&& canonicalizeArtifact(prior) !== canonicalizeArtifact(lease)) {
|
|
629
|
+
fail('WORK_ORDER_LEASE_REJECTED', '同じdigestのstaged lease bytesが変化した');
|
|
630
|
+
}
|
|
631
|
+
stagedLeases.set(lease.lease_digest, structuredClone(lease));
|
|
632
|
+
preparedPackets.set(packet.packet_digest, structuredClone(packet));
|
|
633
|
+
currentEpoch = packet.plan_epoch;
|
|
634
|
+
const ack = sign({
|
|
635
|
+
schema: 'lattice.adapter_prepare_ack.v1',
|
|
636
|
+
ack_id: `prepare-${packet.packet_digest.slice(0, 24)}`,
|
|
637
|
+
registration_digest: request.registration_digest,
|
|
638
|
+
controller_id: controllerId,
|
|
639
|
+
run_id: bootstrap.run_id,
|
|
640
|
+
todo_id: packet.todo_id,
|
|
641
|
+
plan_epoch: packet.plan_epoch,
|
|
642
|
+
packet_digest: packet.packet_digest,
|
|
643
|
+
staged_lease_digest: lease.lease_digest,
|
|
644
|
+
supervisor_session_nonce_digest: sessionNonceDigest,
|
|
645
|
+
ack_digest: '',
|
|
646
|
+
}, 'ack_digest');
|
|
647
|
+
return sign({
|
|
648
|
+
schema: 'lattice.adapter_prepare_response.v1',
|
|
649
|
+
request_id: request.request_id,
|
|
650
|
+
prepare_ack: ack,
|
|
651
|
+
staged_lease_digest: lease.lease_digest,
|
|
652
|
+
response_digest: '',
|
|
653
|
+
}, 'response_digest');
|
|
654
|
+
}
|
|
655
|
+
if (operation === 'activate') {
|
|
656
|
+
if (request.staged_lease_digests.some((digest) => !stagedLeases.has(digest))) {
|
|
657
|
+
fail('WORK_ORDER_LEASE_REJECTED', 'activateが未知のstaged leaseを含む');
|
|
658
|
+
}
|
|
659
|
+
const epochs = new Set(request.staged_lease_digests
|
|
660
|
+
.map((digest) => stagedLeases.get(digest).plan_epoch));
|
|
661
|
+
if (epochs.size > 1) fail('WORK_ORDER_LEASE_REJECTED', 'activate leaseのepochが混在する');
|
|
662
|
+
if (epochs.size === 1) currentEpoch = [...epochs][0];
|
|
663
|
+
const ack = sign({
|
|
664
|
+
schema: 'lattice.adapter_ready_ack.v1',
|
|
665
|
+
ack_id: `ready-${request.activation_digest.slice(0, 24)}`,
|
|
666
|
+
registration_digest: request.registration_digest,
|
|
667
|
+
controller_id: controllerId,
|
|
668
|
+
run_id: bootstrap.run_id,
|
|
669
|
+
plan_epoch: currentEpoch,
|
|
670
|
+
activation_digest: request.activation_digest,
|
|
671
|
+
staged_lease_digests: [...request.staged_lease_digests],
|
|
672
|
+
supervisor_session_nonce_digest: sessionNonceDigest,
|
|
673
|
+
ack_digest: '',
|
|
674
|
+
}, 'ack_digest');
|
|
675
|
+
return sign({
|
|
676
|
+
schema: 'lattice.adapter_activate_response.v1',
|
|
677
|
+
request_id: request.request_id,
|
|
678
|
+
ready_ack: ack,
|
|
679
|
+
observed_pointer_digest: request.committed_epoch_digest,
|
|
680
|
+
response_digest: '',
|
|
681
|
+
}, 'response_digest');
|
|
682
|
+
}
|
|
683
|
+
if (operation === 'release') {
|
|
684
|
+
const armed = [];
|
|
685
|
+
for (const stagedDigest of request.staged_lease_digests) {
|
|
686
|
+
const staged = stagedLeases.get(stagedDigest);
|
|
687
|
+
if (staged === undefined) {
|
|
688
|
+
fail('WORK_ORDER_LEASE_REJECTED', 'releaseが未知のstaged leaseを含む');
|
|
689
|
+
}
|
|
690
|
+
const lease = armStagedWriteLease(staged, {
|
|
691
|
+
releaseBarrierDigest: request.release_barrier_digest,
|
|
692
|
+
gateGeneration: request.gate_generation,
|
|
693
|
+
});
|
|
694
|
+
stagedLeases.delete(stagedDigest);
|
|
695
|
+
armedLeases.set(lease.lease_digest, lease);
|
|
696
|
+
armed.push(lease.lease_digest);
|
|
697
|
+
}
|
|
698
|
+
armed.sort();
|
|
699
|
+
const ack = sign({
|
|
700
|
+
schema: 'lattice.adapter_release_ack.v1',
|
|
701
|
+
ack_id: `release-${request.release_barrier_digest.slice(0, 24)}`,
|
|
702
|
+
registration_digest: request.registration_digest,
|
|
703
|
+
controller_id: controllerId,
|
|
704
|
+
run_id: bootstrap.run_id,
|
|
705
|
+
plan_epoch: currentEpoch,
|
|
706
|
+
release_barrier_digest: request.release_barrier_digest,
|
|
707
|
+
gate_generation: request.gate_generation,
|
|
708
|
+
armed_lease_digests: armed,
|
|
709
|
+
supervisor_session_nonce_digest: sessionNonceDigest,
|
|
710
|
+
ack_digest: '',
|
|
711
|
+
}, 'ack_digest');
|
|
712
|
+
return sign({
|
|
713
|
+
schema: 'lattice.adapter_release_response.v1',
|
|
714
|
+
request_id: request.request_id,
|
|
715
|
+
release_ack: ack,
|
|
716
|
+
armed_lease_digests: armed,
|
|
717
|
+
observed_gate_generation: request.gate_generation,
|
|
718
|
+
response_digest: '',
|
|
719
|
+
}, 'response_digest');
|
|
720
|
+
}
|
|
721
|
+
if (operation === 'dispatch') {
|
|
722
|
+
const { packet, write_lease: writeLease } = request;
|
|
723
|
+
if (!validateExecutorPacket(packet) || !validateArmedWriteLease(writeLease)
|
|
724
|
+
|| !armedLeases.has(writeLease.lease_digest)
|
|
725
|
+
|| writeLease.run_id !== bootstrap.run_id
|
|
726
|
+
|| writeLease.todo_id !== packet.todo_id
|
|
727
|
+
|| writeLease.plan_epoch !== packet.plan_epoch
|
|
728
|
+
|| writeLease.packet_digest !== packet.packet_digest
|
|
729
|
+
|| writeLease.controller_registration_digest !== request.registration_digest
|
|
730
|
+
|| writeLease.supervisor_session_nonce_digest !== sessionNonceDigest) {
|
|
731
|
+
fail('WORK_ORDER_PACKET_REJECTED', 'dispatch packetとarmed leaseのbindingが不正');
|
|
732
|
+
}
|
|
733
|
+
const priorHandle = todoToHandle.get(packet.todo_id);
|
|
734
|
+
if (priorHandle !== undefined) {
|
|
735
|
+
const prior = tasks.get(priorHandle);
|
|
736
|
+
if (prior.packet.packet_digest === packet.packet_digest
|
|
737
|
+
&& prior.lease.lease_digest === writeLease.lease_digest) {
|
|
738
|
+
return sign({
|
|
739
|
+
schema: 'lattice.adapter_dispatch_response.v2',
|
|
740
|
+
request_id: request.request_id,
|
|
741
|
+
executor_handle: prior.executorHandle,
|
|
742
|
+
worktree_id: prior.worktreeId,
|
|
743
|
+
packet_digest: packet.packet_digest,
|
|
744
|
+
lease_digest: writeLease.lease_digest,
|
|
745
|
+
worker_process: workerProcessOf(prior),
|
|
746
|
+
response_digest: '',
|
|
747
|
+
}, 'response_digest');
|
|
748
|
+
}
|
|
749
|
+
if (prior.state !== 'held') {
|
|
750
|
+
fail('WORK_ORDER_DUPLICATE_DISPATCH', '同じTODOへ異なるpacketをdispatchできない');
|
|
751
|
+
}
|
|
752
|
+
// 長寿命worker poolのprocessを殺さず、barrierで止めた同じ席を再開して
|
|
753
|
+
// successor orderへ載せ直す。process寿命は外部adapterの所有である。
|
|
754
|
+
try { process.kill(-prior.worker.process_group_id, 'SIGCONT'); }
|
|
755
|
+
catch (error) {
|
|
756
|
+
fail('WORK_ORDER_EXECUTION_FAILED', 'held workerを再開できない', {
|
|
757
|
+
pid: prior.worker.pid,
|
|
758
|
+
reason: String(error?.code ?? error?.message ?? error),
|
|
759
|
+
});
|
|
760
|
+
}
|
|
761
|
+
prior.state = 'superseded';
|
|
762
|
+
todoToHandle.delete(packet.todo_id);
|
|
763
|
+
}
|
|
764
|
+
await readAndValidateGate(canonicalRunDir, writeLease);
|
|
765
|
+
// canonical repoではなく自分の木へ書く。共有rootでは、書き込みの帰属をrootから
|
|
766
|
+
// 決められないので早期警報もcheckpoint判定も成立しない。木はsupervisorが
|
|
767
|
+
// dispatch前に用意する(監視を張るのがdispatchより前でなければ観測を取り逃す)。
|
|
768
|
+
const worktreePath = workOrderWorktreePath({ runDir: canonicalRunDir, packet });
|
|
769
|
+
try {
|
|
770
|
+
const info = await lstat(worktreePath);
|
|
771
|
+
if (!info.isDirectory()) throw new TypeError('not a directory');
|
|
772
|
+
} catch {
|
|
773
|
+
fail('WORK_ORDER_EXECUTION_FAILED', 'supervisorが用意したworktreeが無い', {
|
|
774
|
+
worktree_path: worktreePath,
|
|
775
|
+
});
|
|
776
|
+
}
|
|
777
|
+
const executorHandle = `work-order-${packet.packet_digest.slice(0, 24)}`;
|
|
778
|
+
const worktreeId = workOrderWorktreeId(packet);
|
|
779
|
+
// **dispatchで作業を終わらせない。** 終わらせると、走行中のTODOが1つも存在しない
|
|
780
|
+
// runになり、実行時の観測——書き込みを見て、他のworkerとの重なりを掴む——が
|
|
781
|
+
// 原理的に成立しない。dispatchは作業を起こして返り、完了はobserveが拾う。
|
|
782
|
+
const progress = {
|
|
783
|
+
schema: 'lattice.work_order_adapter_progress.v1',
|
|
784
|
+
run_id: bootstrap.run_id,
|
|
785
|
+
todo_id: packet.todo_id,
|
|
786
|
+
executor_handle: executorHandle,
|
|
787
|
+
worktree_id: worktreeId,
|
|
788
|
+
state: 'running',
|
|
789
|
+
};
|
|
790
|
+
const worktreeReal = await realpath(worktreePath);
|
|
791
|
+
// **workerは別processで起こす。** controller自身のprocessで作業すると、holdが
|
|
792
|
+
// 要求する静止を証明できない——止めれば応答できず、止めなければ証明できない。
|
|
793
|
+
// `detached`で独立process groupへ置く。同じgroupにcontrollerが居ると、直接OS観測が
|
|
794
|
+
// 「未記録process group memberを検出」として正しく落とす。
|
|
795
|
+
const spawned = await spawnWorkOrderWorker({
|
|
796
|
+
packet, worktreePath: worktreeReal,
|
|
797
|
+
spoolDir: behavior.spoolDir,
|
|
798
|
+
});
|
|
799
|
+
const task = {
|
|
800
|
+
packet: structuredClone(packet),
|
|
801
|
+
lease: structuredClone(writeLease),
|
|
802
|
+
executorHandle,
|
|
803
|
+
worktreeId,
|
|
804
|
+
worktreePath: worktreeReal,
|
|
805
|
+
receipt: null,
|
|
806
|
+
payloadDigest: digestArtifact(progress),
|
|
807
|
+
state: 'running',
|
|
808
|
+
failure: null,
|
|
809
|
+
settled: null,
|
|
810
|
+
worker: spawned,
|
|
811
|
+
};
|
|
812
|
+
tasks.set(executorHandle, task);
|
|
813
|
+
todoToHandle.set(packet.todo_id, executorHandle);
|
|
814
|
+
task.settled = (async () => {
|
|
815
|
+
try {
|
|
816
|
+
const { observedDiff, checkpointDigest } = await spawned.completed;
|
|
817
|
+
const receipt = sign({
|
|
818
|
+
schema: 'lattice.executor_receipt.v1',
|
|
819
|
+
receipt_id: `receipt-${packet.packet_digest.slice(0, 24)}`,
|
|
820
|
+
executor_handle: executorHandle,
|
|
821
|
+
worktree_id: worktreeId,
|
|
822
|
+
base_sha: packet.base_sha,
|
|
823
|
+
plan_epoch: packet.plan_epoch,
|
|
824
|
+
packet_digest: packet.packet_digest,
|
|
825
|
+
todo_id: packet.todo_id,
|
|
826
|
+
checkpoint_digest: checkpointDigest,
|
|
827
|
+
observed_diff: observedDiff,
|
|
828
|
+
receipt_digest: '',
|
|
829
|
+
}, 'receipt_digest');
|
|
830
|
+
if (!validateExecutorReceipt(receipt)) {
|
|
831
|
+
throw new TypeError('生成receiptがexecutor contractを満たさない');
|
|
832
|
+
}
|
|
833
|
+
task.payloadDigest = await persistReceipt(receipt);
|
|
834
|
+
task.receipt = receipt;
|
|
835
|
+
task.state = 'terminal';
|
|
836
|
+
} catch (error) {
|
|
837
|
+
// 失敗を走行中のまま放置しない。observeがtypedに落ちる形へ残す。
|
|
838
|
+
task.failure = String(error?.detail?.reason ?? error?.message ?? error);
|
|
839
|
+
}
|
|
840
|
+
})();
|
|
841
|
+
return sign({
|
|
842
|
+
schema: 'lattice.adapter_dispatch_response.v2',
|
|
843
|
+
request_id: request.request_id,
|
|
844
|
+
executor_handle: executorHandle,
|
|
845
|
+
worktree_id: worktreeId,
|
|
846
|
+
packet_digest: packet.packet_digest,
|
|
847
|
+
lease_digest: writeLease.lease_digest,
|
|
848
|
+
worker_process: workerProcessOf(task),
|
|
849
|
+
response_digest: '',
|
|
850
|
+
}, 'response_digest');
|
|
851
|
+
}
|
|
852
|
+
if (operation === 'observe') {
|
|
853
|
+
const task = tasks.get(request.executor_handle);
|
|
854
|
+
if (task === undefined
|
|
855
|
+
|| task.packet.plan_epoch !== request.expected_epoch
|
|
856
|
+
|| task.lease.lease_digest !== request.expected_lease_digest) {
|
|
857
|
+
fail('WORK_ORDER_OBSERVATION_REJECTED', 'observe bindingがdispatch記録と一致しない');
|
|
858
|
+
}
|
|
859
|
+
// 走行中に落ちた作業を「まだ走っている」と言い続けない。
|
|
860
|
+
if (task.failure !== null) {
|
|
861
|
+
fail('WORK_ORDER_EXECUTION_FAILED', 'worker実行が失敗した', { reason: task.failure });
|
|
862
|
+
}
|
|
863
|
+
const observation = sign({
|
|
864
|
+
schema: 'lattice.adapter_observation.v1',
|
|
865
|
+
state: task.state,
|
|
866
|
+
executor_handle: task.executorHandle,
|
|
867
|
+
plan_epoch: task.packet.plan_epoch,
|
|
868
|
+
lease_digest: task.lease.lease_digest,
|
|
869
|
+
payload_digest: task.payloadDigest,
|
|
870
|
+
observation_digest: '',
|
|
871
|
+
}, 'observation_digest');
|
|
872
|
+
return sign({
|
|
873
|
+
schema: 'lattice.adapter_observe_response.v1',
|
|
874
|
+
request_id: request.request_id,
|
|
875
|
+
observation,
|
|
876
|
+
observation_digest: observation.observation_digest,
|
|
877
|
+
response_digest: '',
|
|
878
|
+
}, 'response_digest');
|
|
879
|
+
}
|
|
880
|
+
if (operation === 'inventory') {
|
|
881
|
+
const runningBindings = [...tasks.values()]
|
|
882
|
+
.filter((task) => task.state === 'running')
|
|
883
|
+
.map((task) => ({
|
|
884
|
+
todo_id: task.packet.todo_id,
|
|
885
|
+
executor_handle: task.executorHandle,
|
|
886
|
+
worktree_id: task.worktreeId,
|
|
887
|
+
plan_epoch: task.packet.plan_epoch,
|
|
888
|
+
packet_digest: task.packet.packet_digest,
|
|
889
|
+
write_lease_id: task.lease.lease_id,
|
|
890
|
+
controller_registration_digest: request.registration_digest,
|
|
891
|
+
}))
|
|
892
|
+
.sort((left, right) => left.todo_id.localeCompare(right.todo_id));
|
|
893
|
+
return sign({
|
|
894
|
+
schema: 'lattice.adapter_running_inventory_response.v1',
|
|
895
|
+
request_id: request.request_id,
|
|
896
|
+
running_bindings: runningBindings,
|
|
897
|
+
inventory_digest: digestArtifact(runningBindings),
|
|
898
|
+
response_digest: '',
|
|
899
|
+
}, 'response_digest');
|
|
900
|
+
}
|
|
901
|
+
if (operation === 'barrier') {
|
|
902
|
+
const acks = [];
|
|
903
|
+
for (const binding of request.running_bindings) {
|
|
904
|
+
const task = tasks.get(binding.executor_handle);
|
|
905
|
+
if (task === undefined || task.packet.todo_id !== binding.todo_id) {
|
|
906
|
+
fail('WORK_ORDER_BARRIER_REJECTED', 'barrierが未知のrunning bindingを含む');
|
|
907
|
+
}
|
|
908
|
+
// **barrierは静止の宣言である。** PGID全体を止め、現在の全memberを観測する。
|
|
909
|
+
// ack前に失敗した時は外部所有の席を凍結したまま残さず再開する。
|
|
910
|
+
const quiesced = await quiesceWorkOrderTask(task);
|
|
911
|
+
task.state = 'held';
|
|
912
|
+
acks.push(sign({
|
|
913
|
+
schema: 'lattice.executor_quiescence_ack.v1',
|
|
914
|
+
ack_id: `barrier-${binding.packet_digest.slice(0, 24)}`,
|
|
915
|
+
run_id: bootstrap.run_id,
|
|
916
|
+
todo_id: binding.todo_id,
|
|
917
|
+
executor_handle: binding.executor_handle,
|
|
918
|
+
worktree_id: binding.worktree_id,
|
|
919
|
+
plan_epoch: binding.plan_epoch,
|
|
920
|
+
packet_digest: binding.packet_digest,
|
|
921
|
+
write_lease_id: binding.write_lease_id,
|
|
922
|
+
barrier_control_digest: request.barrier_control_digest,
|
|
923
|
+
final_checkpoint_digest: quiesced.checkpoint.checkpoint_digest,
|
|
924
|
+
process_observation_digest: quiesced.processObservationDigest,
|
|
925
|
+
worktree_fingerprint_digest: quiesced.worktreeFingerprintDigest,
|
|
926
|
+
supervisor_session_nonce_digest: sessionNonceDigest,
|
|
927
|
+
ack_digest: '',
|
|
928
|
+
}, 'ack_digest'));
|
|
929
|
+
}
|
|
930
|
+
return sign({
|
|
931
|
+
schema: 'lattice.adapter_barrier_response.v1',
|
|
932
|
+
request_id: request.request_id,
|
|
933
|
+
barrier_id: request.barrier_id,
|
|
934
|
+
quiescence_acks: acks,
|
|
935
|
+
response_digest: '',
|
|
936
|
+
}, 'response_digest');
|
|
937
|
+
}
|
|
938
|
+
if (operation === 'rebind') {
|
|
939
|
+
const rebind = request.rebind_packet;
|
|
940
|
+
const task = tasks.get(rebind.executor_handle);
|
|
941
|
+
if (task === undefined || task.worktreeId !== rebind.worktree_id
|
|
942
|
+
|| task.packet.todo_id !== rebind.todo_id) {
|
|
943
|
+
fail('WORK_ORDER_REBIND_REJECTED', 'rebind packetが既存taskへ帰属しない');
|
|
944
|
+
}
|
|
945
|
+
const staged = request.staged_lease;
|
|
946
|
+
if (staged.todo_id !== rebind.todo_id || staged.plan_epoch !== rebind.new_plan_epoch
|
|
947
|
+
|| staged.packet_digest !== rebind.packet_digest) {
|
|
948
|
+
fail('WORK_ORDER_REBIND_REJECTED', 'rebind packetとstaged leaseが一致しない');
|
|
949
|
+
}
|
|
950
|
+
stagedLeases.set(staged.lease_digest, structuredClone(staged));
|
|
951
|
+
task.packet.plan_epoch = rebind.new_plan_epoch;
|
|
952
|
+
currentEpoch = rebind.new_plan_epoch;
|
|
953
|
+
const ack = sign({
|
|
954
|
+
schema: 'lattice.executor_epoch_rebind_ack.v1',
|
|
955
|
+
ack_id: `rebind-${rebind.packet_digest.slice(0, 24)}`,
|
|
956
|
+
run_id: bootstrap.run_id,
|
|
957
|
+
todo_id: rebind.todo_id,
|
|
958
|
+
executor_handle: rebind.executor_handle,
|
|
959
|
+
worktree_id: rebind.worktree_id,
|
|
960
|
+
predecessor_epoch: rebind.new_plan_epoch - 1,
|
|
961
|
+
successor_epoch: rebind.new_plan_epoch,
|
|
962
|
+
// **holdされたworkerにreceiptは無い。** 作業を終えていないから止められているので、
|
|
963
|
+
// 完了の記録を前提にできない。前任packetのdigestはdispatch時のpacketが持っている。
|
|
964
|
+
predecessor_packet_digest: task.packet.packet_digest,
|
|
965
|
+
rebind_packet_digest: rebind.packet_digest,
|
|
966
|
+
new_write_lease_id: staged.lease_id,
|
|
967
|
+
supervisor_session_nonce_digest: sessionNonceDigest,
|
|
968
|
+
ack_digest: '',
|
|
969
|
+
}, 'ack_digest');
|
|
970
|
+
return sign({
|
|
971
|
+
schema: 'lattice.adapter_rebind_response.v1',
|
|
972
|
+
request_id: request.request_id,
|
|
973
|
+
rebind_ack: ack,
|
|
974
|
+
staged_lease_digest: staged.lease_digest,
|
|
975
|
+
response_digest: '',
|
|
976
|
+
}, 'response_digest');
|
|
977
|
+
}
|
|
978
|
+
if (operation === 'revoke') {
|
|
979
|
+
const available = new Set([...stagedLeases.keys(), ...armedLeases.keys()]);
|
|
980
|
+
if (request.lease_digests.some((digest) => !available.has(digest))) {
|
|
981
|
+
fail('WORK_ORDER_REVOKE_REJECTED', 'revokeが未知のleaseを含む');
|
|
982
|
+
}
|
|
983
|
+
for (const leaseDigest of request.lease_digests) {
|
|
984
|
+
stagedLeases.delete(leaseDigest);
|
|
985
|
+
armedLeases.delete(leaseDigest);
|
|
986
|
+
}
|
|
987
|
+
for (const task of tasks.values()) {
|
|
988
|
+
if (request.lease_digests.includes(task.lease.lease_digest)) task.state = 'held';
|
|
989
|
+
}
|
|
990
|
+
return sign({
|
|
991
|
+
schema: 'lattice.adapter_revoke_response.v1',
|
|
992
|
+
request_id: request.request_id,
|
|
993
|
+
revoked_lease_digests: [...request.lease_digests],
|
|
994
|
+
residual_processes: [],
|
|
995
|
+
response_digest: '',
|
|
996
|
+
}, 'response_digest');
|
|
997
|
+
}
|
|
998
|
+
fail('WORK_ORDER_UNKNOWN_OPERATION', `未知のcontroller operation: ${operation}`);
|
|
999
|
+
};
|
|
1000
|
+
|
|
1001
|
+
return Object.freeze({
|
|
1002
|
+
bootstrap: structuredClone(bootstrap),
|
|
1003
|
+
controllerId,
|
|
1004
|
+
controllerSessionNonce,
|
|
1005
|
+
descriptor: structuredClone(descriptor),
|
|
1006
|
+
heartbeat,
|
|
1007
|
+
reapWorkers,
|
|
1008
|
+
leaseSetDigest() {
|
|
1009
|
+
return digestArtifact([...stagedLeases.keys(), ...armedLeases.keys()].sort());
|
|
1010
|
+
},
|
|
1011
|
+
handshake(request) {
|
|
1012
|
+
if (!validateHandshakeRequest(request, bootstrap)) {
|
|
1013
|
+
fail('WORK_ORDER_HANDSHAKE_INVALID', 'handshake requestがbootstrapへbindしない');
|
|
1014
|
+
}
|
|
1015
|
+
return sign({
|
|
1016
|
+
schema: 'lattice.adapter_controller_handshake_response.v1',
|
|
1017
|
+
request_id: request.request_id,
|
|
1018
|
+
run_id: bootstrap.run_id,
|
|
1019
|
+
challenge_digest: digestArtifact(request.challenge),
|
|
1020
|
+
controller_session_nonce: controllerSessionNonce,
|
|
1021
|
+
descriptor,
|
|
1022
|
+
response_digest: '',
|
|
1023
|
+
}, 'response_digest');
|
|
1024
|
+
},
|
|
1025
|
+
async request(operation, request) {
|
|
1026
|
+
if (!CONTROLLER_OPERATIONS.includes(operation)) {
|
|
1027
|
+
fail('WORK_ORDER_UNKNOWN_OPERATION', `未知のcontroller operation: ${operation}`);
|
|
1028
|
+
}
|
|
1029
|
+
return responseFor(operation, request);
|
|
1030
|
+
},
|
|
1031
|
+
});
|
|
1032
|
+
}
|
|
1033
|
+
|
|
1034
|
+
async function readBootstrapFd(fd = 3) {
|
|
1035
|
+
const bytes = readFileSync(fd);
|
|
1036
|
+
if (bytes.length === 0 || bytes.length > MAX_DOCUMENT_BYTES) {
|
|
1037
|
+
fail('WORK_ORDER_BOOTSTRAP_INVALID', 'bootstrap sizeが不正');
|
|
1038
|
+
}
|
|
1039
|
+
const text = bytes.toString('utf8');
|
|
1040
|
+
if (!text.endsWith('\n') || text.indexOf('\n') !== text.length - 1) {
|
|
1041
|
+
fail('WORK_ORDER_BOOTSTRAP_INVALID', 'bootstrapはJSON 1 documentでなければならない');
|
|
1042
|
+
}
|
|
1043
|
+
let value;
|
|
1044
|
+
try {
|
|
1045
|
+
value = JSON.parse(text);
|
|
1046
|
+
} catch {
|
|
1047
|
+
fail('WORK_ORDER_BOOTSTRAP_INVALID', 'bootstrap JSONが不正');
|
|
1048
|
+
}
|
|
1049
|
+
return value;
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
export async function runWorkOrderAdapterController({
|
|
1053
|
+
runDir = process.cwd(),
|
|
1054
|
+
bootstrap = null,
|
|
1055
|
+
stderr = process.stderr,
|
|
1056
|
+
} = {}) {
|
|
1057
|
+
const resolvedBootstrap = bootstrap ?? await readBootstrapFd();
|
|
1058
|
+
const canonicalRunDir = await realpath(runDir);
|
|
1059
|
+
const repoRoot = await realpath(path.resolve(canonicalRunDir, '..', '..', '..'));
|
|
1060
|
+
const controller = await createWorkOrderAdapterController({
|
|
1061
|
+
bootstrap: resolvedBootstrap,
|
|
1062
|
+
runDir: canonicalRunDir,
|
|
1063
|
+
repoRoot,
|
|
1064
|
+
});
|
|
1065
|
+
const socketPath = path.join(canonicalRunDir, resolvedBootstrap.controller_socket_ref);
|
|
1066
|
+
const controllerDirectory = path.dirname(socketPath);
|
|
1067
|
+
await mkdir(controllerDirectory, { recursive: true, mode: 0o700 });
|
|
1068
|
+
await chmod(controllerDirectory, 0o700);
|
|
1069
|
+
let heartbeatTimer = null;
|
|
1070
|
+
let persistentSocket = null;
|
|
1071
|
+
let heartbeatSequence = 0;
|
|
1072
|
+
let registrationDigest = null;
|
|
1073
|
+
const send = (socket, document) => {
|
|
1074
|
+
socket.write(`${canonicalizeArtifact(document)}\n`);
|
|
1075
|
+
};
|
|
1076
|
+
const startHeartbeat = (socket, digest) => {
|
|
1077
|
+
persistentSocket = socket;
|
|
1078
|
+
registrationDigest = digest;
|
|
1079
|
+
if (heartbeatTimer !== null) return;
|
|
1080
|
+
heartbeatTimer = setInterval(() => {
|
|
1081
|
+
if (persistentSocket === null || persistentSocket.destroyed) return;
|
|
1082
|
+
heartbeatSequence += 1;
|
|
1083
|
+
send(persistentSocket, sign({
|
|
1084
|
+
schema: 'lattice.adapter_controller_heartbeat.v1',
|
|
1085
|
+
controller_id: controller.controllerId,
|
|
1086
|
+
registration_digest: registrationDigest,
|
|
1087
|
+
supervisor_session_nonce: resolvedBootstrap.supervisor_session_nonce,
|
|
1088
|
+
sequence: heartbeatSequence,
|
|
1089
|
+
lease_set_digest: controller.leaseSetDigest(),
|
|
1090
|
+
heartbeat_digest: '',
|
|
1091
|
+
}, 'heartbeat_digest'));
|
|
1092
|
+
}, controller.heartbeat.interval_ms);
|
|
1093
|
+
heartbeatTimer.unref();
|
|
1094
|
+
};
|
|
1095
|
+
const server = net.createServer((socket) => {
|
|
1096
|
+
socket.setEncoding('utf8');
|
|
1097
|
+
let buffer = '';
|
|
1098
|
+
let processing = Promise.resolve();
|
|
1099
|
+
socket.on('data', (chunk) => {
|
|
1100
|
+
buffer += chunk;
|
|
1101
|
+
if (Buffer.byteLength(buffer, 'utf8') > MAX_DOCUMENT_BYTES) {
|
|
1102
|
+
send(socket, controllerErrorArtifact(
|
|
1103
|
+
new WorkOrderAdapterControllerError(
|
|
1104
|
+
'WORK_ORDER_DOCUMENT_TOO_LARGE',
|
|
1105
|
+
'controller document上限を超えた',
|
|
1106
|
+
),
|
|
1107
|
+
));
|
|
1108
|
+
socket.destroy();
|
|
1109
|
+
return;
|
|
1110
|
+
}
|
|
1111
|
+
while (buffer.includes('\n')) {
|
|
1112
|
+
const newline = buffer.indexOf('\n');
|
|
1113
|
+
const line = buffer.slice(0, newline);
|
|
1114
|
+
buffer = buffer.slice(newline + 1);
|
|
1115
|
+
processing = processing.then(async () => {
|
|
1116
|
+
let request;
|
|
1117
|
+
try {
|
|
1118
|
+
request = JSON.parse(line);
|
|
1119
|
+
if (request.schema === 'lattice.adapter_controller_handshake_request.v1') {
|
|
1120
|
+
send(socket, controller.handshake(request));
|
|
1121
|
+
return;
|
|
1122
|
+
}
|
|
1123
|
+
const operation = REQUEST_SCHEMA_TO_OPERATION[request.schema];
|
|
1124
|
+
if (operation === undefined) {
|
|
1125
|
+
fail('WORK_ORDER_UNKNOWN_OPERATION', `未知のrequest schema: ${String(request.schema)}`);
|
|
1126
|
+
}
|
|
1127
|
+
startHeartbeat(socket, request.registration_digest);
|
|
1128
|
+
const response = await controller.request(operation, request);
|
|
1129
|
+
send(socket, response);
|
|
1130
|
+
} catch (error) {
|
|
1131
|
+
send(socket, controllerErrorArtifact(error, request?.request_id));
|
|
1132
|
+
stderr.write(`${canonicalizeArtifact(controllerErrorArtifact(error, request?.request_id))}\n`);
|
|
1133
|
+
socket.destroy();
|
|
1134
|
+
}
|
|
1135
|
+
});
|
|
1136
|
+
}
|
|
1137
|
+
});
|
|
1138
|
+
socket.on('close', () => {
|
|
1139
|
+
if (socket === persistentSocket) {
|
|
1140
|
+
clearInterval(heartbeatTimer);
|
|
1141
|
+
heartbeatTimer = null;
|
|
1142
|
+
persistentSocket = null;
|
|
1143
|
+
}
|
|
1144
|
+
});
|
|
1145
|
+
});
|
|
1146
|
+
server.on('error', (error) => {
|
|
1147
|
+
stderr.write(`${canonicalizeArtifact(controllerErrorArtifact(error))}\n`);
|
|
1148
|
+
});
|
|
1149
|
+
await new Promise((resolve, reject) => {
|
|
1150
|
+
server.once('error', reject);
|
|
1151
|
+
// AF_UNIXのpath長上限へ一時repoのabsolute prefixを混入させない。supervisorも
|
|
1152
|
+
// run storeをcwdにして同じ固定relative refへ接続する。
|
|
1153
|
+
server.listen(resolvedBootstrap.controller_socket_ref, () => {
|
|
1154
|
+
server.off('error', reject);
|
|
1155
|
+
resolve();
|
|
1156
|
+
});
|
|
1157
|
+
});
|
|
1158
|
+
await chmod(socketPath, 0o600);
|
|
1159
|
+
let closed = false;
|
|
1160
|
+
const close = async () => {
|
|
1161
|
+
if (closed) return;
|
|
1162
|
+
closed = true;
|
|
1163
|
+
// 外部worker poolはadapter側が所有する。controllerはsocketだけを畳む。
|
|
1164
|
+
controller.reapWorkers();
|
|
1165
|
+
clearInterval(heartbeatTimer);
|
|
1166
|
+
persistentSocket?.destroy();
|
|
1167
|
+
await new Promise((resolve) => server.close(resolve));
|
|
1168
|
+
await rm(socketPath, { force: true });
|
|
1169
|
+
};
|
|
1170
|
+
return Object.freeze({ controller, socketPath, close });
|
|
1171
|
+
}
|