@quolu/lattice 0.30.1 → 0.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* scripted executorのworker process(ADR 0143 Decision 9)。
|
|
4
|
+
*
|
|
5
|
+
* **別processであること自体が要件である。** holdは静止の証明を要求し、直接OS観測は
|
|
6
|
+
* executorのprocessが実際に停止していることまで確かめる。controller自身のprocessで
|
|
7
|
+
* 作業していると、止めれば応答できず、止めなければ証明できない。
|
|
8
|
+
*
|
|
9
|
+
* 書いたあとも生きたまま待つ。作業を終えて消えてしまうと、barrierが掛かった時に
|
|
10
|
+
* 止めるべきprocessが存在せず、静止を証明する相手が居なくなる。
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { readFile } from 'node:fs/promises';
|
|
14
|
+
|
|
15
|
+
import { executePacket } from '../src/runtime-scripted-adapter-controller.mjs';
|
|
16
|
+
|
|
17
|
+
const MAX_JOB_BYTES = 8_388_608;
|
|
18
|
+
|
|
19
|
+
function fail(reason) {
|
|
20
|
+
process.stderr.write(`${JSON.stringify({
|
|
21
|
+
schema: 'lattice.scripted_worker_error.v1', reason,
|
|
22
|
+
})}\n`);
|
|
23
|
+
process.exit(1);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const jobPath = process.argv[2];
|
|
27
|
+
if (typeof jobPath !== 'string' || jobPath.length === 0) fail('job pathが渡されていない');
|
|
28
|
+
|
|
29
|
+
let job;
|
|
30
|
+
try {
|
|
31
|
+
const bytes = await readFile(jobPath);
|
|
32
|
+
if (bytes.length > MAX_JOB_BYTES) fail('jobが上限byteを超える');
|
|
33
|
+
job = JSON.parse(bytes.toString('utf8'));
|
|
34
|
+
} catch (error) {
|
|
35
|
+
fail(`jobを読めない: ${String(error?.message ?? error)}`);
|
|
36
|
+
}
|
|
37
|
+
if (job?.schema !== 'lattice.scripted_worker_job.v1') fail('job schemaが不正');
|
|
38
|
+
|
|
39
|
+
let result;
|
|
40
|
+
try {
|
|
41
|
+
result = await executePacket({
|
|
42
|
+
packet: job.packet,
|
|
43
|
+
repoRoot: job.worktree_path,
|
|
44
|
+
extraWrites: job.extra_writes ?? [],
|
|
45
|
+
});
|
|
46
|
+
} catch (error) {
|
|
47
|
+
fail(`実行に失敗した: ${String(error?.detail?.reason ?? error?.message ?? error)}`);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
// 完了はstdoutの1行で報せる。controllerはこれを受けてreceiptを組む。
|
|
51
|
+
process.stdout.write(`${JSON.stringify({
|
|
52
|
+
schema: 'lattice.scripted_worker_result.v1',
|
|
53
|
+
observed_diff: result.observedDiff,
|
|
54
|
+
checkpoint_digest: result.checkpointDigest,
|
|
55
|
+
})}\n`);
|
|
56
|
+
|
|
57
|
+
// 生きたまま待つ。ここがrunの「作業中」であり、barrierはこのprocessを止めて静止を証明する。
|
|
58
|
+
const holdMs = Number.isSafeInteger(job.hold_ms) && job.hold_ms > 0 ? job.hold_ms : 0;
|
|
59
|
+
if (holdMs > 0) await new Promise((resolve) => { setTimeout(resolve, holdMs); });
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@quolu/lattice",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.32.0",
|
|
4
4
|
"description": "Schedulability compiler for multi-agent development: observe real code boundaries, refactor the conflicting seam, recompile the plan for parallel execution",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Quo / クオ at kitepon.dev",
|
package/src/runtime-cli.mjs
CHANGED
|
@@ -19,7 +19,7 @@ import { fileURLToPath } from 'node:url';
|
|
|
19
19
|
|
|
20
20
|
import { canonicalizeArtifact, digestArtifact } from './artifact-contracts.mjs';
|
|
21
21
|
import { collectSensorEvidence } from './sensor-adapter.mjs';
|
|
22
|
-
import { detectCheckpointFindings } from './runtime-diff-observer.mjs';
|
|
22
|
+
import { captureWorktreeDiff, detectCheckpointFindings } from './runtime-diff-observer.mjs';
|
|
23
23
|
import {
|
|
24
24
|
buildIoEscalation, createRunSentinel, probeIoWarning, syncSentinelWatches,
|
|
25
25
|
} from './runtime-io-sentinel.mjs';
|
|
@@ -139,6 +139,9 @@ const KNOWN_ADAPTERS = Object.freeze(['scripted', 'isolated-worktree', 'actual-a
|
|
|
139
139
|
* 超えたら`rejected`として理由ごとjournalへ残す(黙って見送らない)。
|
|
140
140
|
*/
|
|
141
141
|
const IO_ESCALATION_LOCK_TIMEOUT_MS = 15_000;
|
|
142
|
+
/** 走行中workerの完了を待つ上限と間隔。待てないと走行中の観測が成立しない。 */
|
|
143
|
+
const SCRIPTED_OBSERVE_TIMEOUT_MS = 120_000;
|
|
144
|
+
const SCRIPTED_OBSERVE_POLL_MS = 20;
|
|
142
145
|
|
|
143
146
|
class CliContractError extends Error {
|
|
144
147
|
constructor(code, message, detail) {
|
|
@@ -750,13 +753,14 @@ async function driveInitialScriptedManagedEpoch({
|
|
|
750
753
|
initialEvents,
|
|
751
754
|
controlEvents,
|
|
752
755
|
sentinel = null,
|
|
756
|
+
preDispatchBindings = null,
|
|
757
|
+
drainEscalations = null,
|
|
753
758
|
}) {
|
|
754
759
|
let events = [...initialEvents];
|
|
755
760
|
const { plan, manifests, executor_packets: packets } = committed.bundle;
|
|
756
761
|
const controllerId = activation.controllerDescriptor.controller_id;
|
|
757
762
|
const registrationDigest = activation.registration.registration_digest;
|
|
758
763
|
const sessionNonceDigest = digestArtifact(activation.sessionNonce);
|
|
759
|
-
const processGroupId = activation.childPid;
|
|
760
764
|
for (;;) {
|
|
761
765
|
const frontier = computeReadyFrontier({ plan, events }).dispatchable;
|
|
762
766
|
if (frontier.length === 0) break;
|
|
@@ -777,9 +781,13 @@ async function driveInitialScriptedManagedEpoch({
|
|
|
777
781
|
// 木がTODOごとに分かれて初めて、書き込みの帰属をrootから決められる。
|
|
778
782
|
const worktreeByTodo = new Map();
|
|
779
783
|
for (const todoId of frontier) {
|
|
780
|
-
|
|
784
|
+
const worktreePath = await ensureScriptedWorktree({
|
|
781
785
|
repoRoot, runDir, packet: packets[todoId],
|
|
782
|
-
})
|
|
786
|
+
});
|
|
787
|
+
worktreeByTodo.set(todoId, worktreePath);
|
|
788
|
+
preDispatchBindings?.set(todoId, {
|
|
789
|
+
worktree_path: worktreePath, base_sha: packets[todoId].base_sha,
|
|
790
|
+
});
|
|
783
791
|
}
|
|
784
792
|
syncSentinelWatches({ sentinel, runningTodoIds: [...frontier],
|
|
785
793
|
rootOf: (todoId) => worktreeByTodo.get(todoId) });
|
|
@@ -854,12 +862,19 @@ async function driveInitialScriptedManagedEpoch({
|
|
|
854
862
|
controller_session_nonce_digest:
|
|
855
863
|
activation.controllerDescriptor.controller_session_nonce_digest,
|
|
856
864
|
direct_os_observation_binding: {
|
|
857
|
-
|
|
858
|
-
|
|
865
|
+
// **controllerではなくworker processを指す。** holdは静止の証明を要求し、
|
|
866
|
+
// 直接OS観測はここで名指しされたprocessが実際に停止していることを確かめる。
|
|
867
|
+
// controllerを指していた頃は、止めれば応答できず止めなければ証明できなかった。
|
|
868
|
+
process_pid: response.worker_process.pid,
|
|
869
|
+
process_group_id: response.worker_process.process_group_id,
|
|
859
870
|
process_start_identity:
|
|
860
|
-
structuredClone(
|
|
871
|
+
structuredClone(response.worker_process.process_start_identity),
|
|
872
|
+
// workerはさらに子を持たない。空配列は「子が居ない」という主張であり、
|
|
873
|
+
// 直接OS観測は実測と突き合わせて未記録のchildが居ないことまで確かめる。
|
|
874
|
+
process_children: [],
|
|
861
875
|
// TODOごとの木を指す。ここがrepo rootだった頃、帰属はrootから決まらなかった。
|
|
862
876
|
worktree_path: worktreeByTodo.get(packet.todo_id),
|
|
877
|
+
worktree_realpath: worktreeByTodo.get(packet.todo_id),
|
|
863
878
|
base_sha: packet.base_sha,
|
|
864
879
|
},
|
|
865
880
|
};
|
|
@@ -869,15 +884,19 @@ async function driveInitialScriptedManagedEpoch({
|
|
|
869
884
|
event.kind === 'executor_dispatched'
|
|
870
885
|
&& event.payload?.executor_handle === executorHandle
|
|
871
886
|
));
|
|
887
|
+
// **ここでpollしない。** 待つのは駆動側の仕事である——workerが走っている間が
|
|
888
|
+
// 実行時競合を掴める唯一の窓であり、その窓は駆動側がeventsを手元に持っている時
|
|
889
|
+
// にしか安全に触れない。ここで待つと、窓の間ずっと駆動側が止まる。
|
|
872
890
|
const response = await managedSupervisor.route('observe', controllerId, {
|
|
873
891
|
executor_handle: executorHandle,
|
|
874
892
|
expected_epoch: dispatch.plan_epoch,
|
|
875
893
|
expected_lease_digest: dispatch.payload.write_lease_digest,
|
|
876
894
|
});
|
|
895
|
+
if (response.observation.state === 'running') return { state: 'running' };
|
|
877
896
|
if (response.observation.state !== 'terminal') {
|
|
878
897
|
throw new ManagedRuntimeError(
|
|
879
898
|
'ADAPTER_CONTROLLER_UNAVAILABLE',
|
|
880
|
-
`scripted controller
|
|
899
|
+
`scripted controllerが未知のstateを返した: ${response.observation.state}`,
|
|
881
900
|
);
|
|
882
901
|
}
|
|
883
902
|
const receipt = await readScriptedControllerReceipt({
|
|
@@ -911,22 +930,52 @@ async function driveInitialScriptedManagedEpoch({
|
|
|
911
930
|
rootOf: (todoId) => events.findLast((event) => event.kind === 'executor_dispatched'
|
|
912
931
|
&& event.subject?.kind === 'todo' && event.subject.ref === todoId)
|
|
913
932
|
?.payload?.direct_os_observation_binding?.worktree_path });
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
933
|
+
// **workerが走っている間だけが、実行時競合を掴める窓である。** 観測を1件ずつ待たずに
|
|
934
|
+
// 回し、その合間に早期警報のescalationを捌く。捌くのは`replaceEventsAtomically`の
|
|
935
|
+
// 直後だけ——そこでしかdiskとメモリのeventsが一致していない。
|
|
936
|
+
const awaiting = new Set(dispatched.dispatched);
|
|
937
|
+
const observeDeadline = Date.now() + SCRIPTED_OBSERVE_TIMEOUT_MS;
|
|
938
|
+
let frozen = false;
|
|
939
|
+
while (awaiting.size > 0) {
|
|
940
|
+
const drained = await drainEscalations?.(events) ?? null;
|
|
941
|
+
if (drained !== null) {
|
|
942
|
+
events = drained;
|
|
943
|
+
if (projectRuntimeState({ events }).freeze !== null) { frozen = true; break; }
|
|
944
|
+
}
|
|
945
|
+
let progressed = false;
|
|
946
|
+
for (const todoId of [...awaiting]) {
|
|
947
|
+
const observed = await observeExecutor({
|
|
948
|
+
runId: request.request_id,
|
|
949
|
+
todoId,
|
|
950
|
+
plan,
|
|
951
|
+
events,
|
|
952
|
+
adapter: managedAdapter,
|
|
953
|
+
recordedAt: canonicalNow(),
|
|
954
|
+
});
|
|
955
|
+
if (observed.observation.state === 'running') continue;
|
|
956
|
+
events = observed.events;
|
|
957
|
+
await replaceEventsAtomically(runDir, events);
|
|
958
|
+
syncSentinelWatches({ sentinel, runningTodoIds: projectRuntimeState({ events }).running,
|
|
959
|
+
rootOf: (id) => events.findLast((event) => event.kind === 'executor_dispatched'
|
|
960
|
+
&& event.subject?.kind === 'todo' && event.subject.ref === id)
|
|
961
|
+
?.payload?.direct_os_observation_binding?.worktree_path });
|
|
962
|
+
awaiting.delete(todoId);
|
|
963
|
+
progressed = true;
|
|
964
|
+
}
|
|
965
|
+
if (awaiting.size === 0) break;
|
|
966
|
+
if (Date.now() >= observeDeadline) {
|
|
967
|
+
throw new ManagedRuntimeError(
|
|
968
|
+
'ADAPTER_CONTROLLER_UNAVAILABLE',
|
|
969
|
+
`scripted workerが時間内に終わらない: ${[...awaiting].join(',')}`,
|
|
970
|
+
);
|
|
971
|
+
}
|
|
972
|
+
if (!progressed) {
|
|
973
|
+
await new Promise((resolve) => { setTimeout(resolve, SCRIPTED_OBSERVE_POLL_MS); });
|
|
974
|
+
}
|
|
929
975
|
}
|
|
976
|
+
// holdが掛かったらdispatchを進めない。freeze後のreceiptはstaleとして拒否されるので、
|
|
977
|
+
// 裁定へ進むと「受理されなかった」で落ちるだけである。
|
|
978
|
+
if (frozen) return events;
|
|
930
979
|
const adjudicated = adjudicatePendingReceipts({
|
|
931
980
|
runId: request.request_id,
|
|
932
981
|
plan,
|
|
@@ -1738,6 +1787,14 @@ export async function runManagedSupervisorDaemon({
|
|
|
1738
1787
|
* probeが撮るのはgitから読んだ本物のdiffなので、そのままfindingの証拠になる。
|
|
1739
1788
|
* fs eventをfindingへ昇格させる必要が無く、契約を1つも緩めずに済む。
|
|
1740
1789
|
*/
|
|
1790
|
+
/**
|
|
1791
|
+
* dispatch event耐久化より前のobservation binding。
|
|
1792
|
+
*
|
|
1793
|
+
* frontierのworktreeを用意した時点でsupervisorが知っている値を置く。警報はdispatchの
|
|
1794
|
+
* 最中に飛ぶので、eventの耐久化を待つと最も早い観測を取り逃す。
|
|
1795
|
+
*/
|
|
1796
|
+
const preDispatchBindings = new Map();
|
|
1797
|
+
|
|
1741
1798
|
const probeWarning = async (warning) => {
|
|
1742
1799
|
const events = await readBoundedJson(path.join(runDir, 'events.json'), 'run events')
|
|
1743
1800
|
.catch(() => null);
|
|
@@ -1749,7 +1806,12 @@ export async function runManagedSupervisorDaemon({
|
|
|
1749
1806
|
for (const todoId of warning.todo_ids) {
|
|
1750
1807
|
const dispatch = events.findLast((event) => event.kind === 'executor_dispatched'
|
|
1751
1808
|
&& event.subject?.kind === 'todo' && event.subject.ref === todoId);
|
|
1752
|
-
|
|
1809
|
+
// dispatch eventは、frontier全体のdispatchが終わってから一括で耐久化される。
|
|
1810
|
+
// 警報はその最中に飛ぶので、eventだけを頼りにすると**いちばん早い観測を必ず取り逃す**。
|
|
1811
|
+
// supervisorはdispatchの前にworktreeを用意した時点で同じbindingを知っているので、
|
|
1812
|
+
// そちらを先に見る。値はどちらも同一で、早いか遅いかの違いしかない。
|
|
1813
|
+
const binding = dispatch?.payload?.direct_os_observation_binding
|
|
1814
|
+
?? preDispatchBindings.get(todoId);
|
|
1753
1815
|
if (typeof binding?.worktree_path !== 'string' || typeof binding?.base_sha !== 'string') continue;
|
|
1754
1816
|
roots[todoId] = binding.worktree_path;
|
|
1755
1817
|
// 観測できないTODOは「書いていない」へ丸めない。probeIoWarningが未観測として扱う。
|
|
@@ -1775,6 +1837,15 @@ export async function runManagedSupervisorDaemon({
|
|
|
1775
1837
|
* 「両方が書いた」と読めてしまう。これをholdへ繋ぐと、無実のTODOを止める。
|
|
1776
1838
|
* 帰属が立たないならescalationへ進めない。警報と実在の記録はそのまま残す。
|
|
1777
1839
|
*/
|
|
1840
|
+
/**
|
|
1841
|
+
* holdが要求する静止を、この構成で証明できるか。
|
|
1842
|
+
*
|
|
1843
|
+
* 直接OS観測はexecutorのprocessが停止していることを実測する。scripted controllerは
|
|
1844
|
+
* 自分のprocessで作業するので、そのprocessを止めると制御そのものが止まる——止めない限り
|
|
1845
|
+
* 証明できず、止めれば応答できない。**別processのexecutorを持つまでこの穴は埋まらない。**
|
|
1846
|
+
*/
|
|
1847
|
+
const canProveQuiescence = () => true;
|
|
1848
|
+
|
|
1778
1849
|
const attributionIsDistinct = (warning, roots) => {
|
|
1779
1850
|
const paths = warning.todo_ids.map((todoId) => roots[todoId]);
|
|
1780
1851
|
if (paths.some((value) => typeof value !== 'string')) return false;
|
|
@@ -1788,7 +1859,7 @@ export async function runManagedSupervisorDaemon({
|
|
|
1788
1859
|
* 事後に再読して再導出できる主張であることの担保である。fs eventは取りこぼすし再読もできない。
|
|
1789
1860
|
* ここで残すのは「機械が気づいた」という事実だけで、判定の正本はcheckpointのままである。
|
|
1790
1861
|
*
|
|
1791
|
-
*
|
|
1862
|
+
* 記録しない選択肢は無い。気づいたのに黙っている状態を残さない。
|
|
1792
1863
|
*/
|
|
1793
1864
|
const warningDigestOf = (warning) => digestArtifact({
|
|
1794
1865
|
warning_kind: warning.kind, todo_ids: [...warning.todo_ids].sort(), path: warning.path,
|
|
@@ -1818,17 +1889,7 @@ export async function runManagedSupervisorDaemon({
|
|
|
1818
1889
|
*
|
|
1819
1890
|
* @returns {{ok: true, expected_epoch: number}|{ok: false, outcome: string, detail: string}}
|
|
1820
1891
|
*/
|
|
1821
|
-
const
|
|
1822
|
-
let lock;
|
|
1823
|
-
try {
|
|
1824
|
-
lock = await acquireRuntimeLifecycleLock({ runDir,
|
|
1825
|
-
sessionNonceDigest: digestArtifact(sessionNonce), operation: 'finding_record',
|
|
1826
|
-
requestId: `${escalation.escalation_id}.checkpoint`,
|
|
1827
|
-
timeoutMs: IO_ESCALATION_LOCK_TIMEOUT_MS, retryIntervalMs: 25 });
|
|
1828
|
-
} catch (error) {
|
|
1829
|
-
return { ok: false, outcome: 'rejected',
|
|
1830
|
-
detail: `lifecycle lockを取れない: ${error?.code ?? 'RUN_BUSY'}` };
|
|
1831
|
-
}
|
|
1892
|
+
const recordProbeCheckpointsUnlocked = async (escalation, checkpointsByTodo) => {
|
|
1832
1893
|
try {
|
|
1833
1894
|
const active = await readCommittedEpochStore(runDir);
|
|
1834
1895
|
if (active === null) return { ok: false, outcome: 'skipped', detail: 'managed epochが未commit' };
|
|
@@ -1859,6 +1920,23 @@ export async function runManagedSupervisorDaemon({
|
|
|
1859
1920
|
} catch (error) {
|
|
1860
1921
|
return { ok: false, outcome: 'rejected',
|
|
1861
1922
|
detail: `probe checkpointを耐久化できない: ${error?.code ?? String(error?.message ?? error)}` };
|
|
1923
|
+
}
|
|
1924
|
+
};
|
|
1925
|
+
|
|
1926
|
+
/** lifecycle lockを自分で取ってから耐久化する版(epoch駆動の外から呼ぶ時)。 */
|
|
1927
|
+
const durablyRecordProbeCheckpoints = async (escalation, checkpointsByTodo) => {
|
|
1928
|
+
let lock;
|
|
1929
|
+
try {
|
|
1930
|
+
lock = await acquireRuntimeLifecycleLock({ runDir,
|
|
1931
|
+
sessionNonceDigest: digestArtifact(sessionNonce), operation: 'finding_record',
|
|
1932
|
+
requestId: `${escalation.escalation_id}.checkpoint`,
|
|
1933
|
+
timeoutMs: IO_ESCALATION_LOCK_TIMEOUT_MS, retryIntervalMs: 25 });
|
|
1934
|
+
} catch (error) {
|
|
1935
|
+
return { ok: false, outcome: 'rejected',
|
|
1936
|
+
detail: `lifecycle lockを取れない: ${error?.code ?? 'RUN_BUSY'}` };
|
|
1937
|
+
}
|
|
1938
|
+
try {
|
|
1939
|
+
return await recordProbeCheckpointsUnlocked(escalation, checkpointsByTodo);
|
|
1862
1940
|
} finally {
|
|
1863
1941
|
await lock.release();
|
|
1864
1942
|
}
|
|
@@ -1874,7 +1952,10 @@ export async function runManagedSupervisorDaemon({
|
|
|
1874
1952
|
*
|
|
1875
1953
|
* 途中で断られたらそこで止め、理由をjournalへ残す。静かに別経路へ逃げない。
|
|
1876
1954
|
*/
|
|
1877
|
-
const escalateIoWarning = async (warning, probed, packets) => {
|
|
1955
|
+
const escalateIoWarning = async (warning, probed, packets, options = {}) => {
|
|
1956
|
+
// 耐久化と送信の経路だけを差し替える。判断そのものはどちらの文脈でも同一にする——
|
|
1957
|
+
// 分けると「epoch駆動の中と外で違う止め方をする」ことになり、記録が読めなくなる。
|
|
1958
|
+
const { recordCheckpoints = durablyRecordProbeCheckpoints, dispatchControl = null } = options;
|
|
1878
1959
|
const built = buildIoEscalation({ warning, probe: probed,
|
|
1879
1960
|
checkpointsByTodo: probed.checkpoints, packets });
|
|
1880
1961
|
if (built === null) return null;
|
|
@@ -1893,7 +1974,20 @@ export async function runManagedSupervisorDaemon({
|
|
|
1893
1974
|
return decide('skipped',
|
|
1894
1975
|
'worktree rootを共有する構成では書き手を特定できない(帰属はrootだけで決まる)');
|
|
1895
1976
|
}
|
|
1896
|
-
|
|
1977
|
+
// **holdは静止の証明を要求する。** 直接OS観測はexecutorのprocessが実際に停止している
|
|
1978
|
+
// ことまで確かめるが、executorがcontroller自身のprocessである構成では、止めると
|
|
1979
|
+
// 制御そのものが止まるので証明できない。
|
|
1980
|
+
//
|
|
1981
|
+
// ここで止めるのは、conflictがintakeをfreezeするからである。freezeしてholdが通らない
|
|
1982
|
+
// 状態を作ると、runは進むことも畳むこともできなくなる(abandonも静止を要求する)。
|
|
1983
|
+
// 止められないと分かっているなら、freezeさせない方が安全側である。判定はcheckpointが
|
|
1984
|
+
// 従来どおり担う——早期警報は早めるためだけに在るという原則どおり。
|
|
1985
|
+
if (!canProveQuiescence()) {
|
|
1986
|
+
return decide('rejected',
|
|
1987
|
+
'executorがcontroller自身のprocessで走っており、静止を証明できない'
|
|
1988
|
+
+ '(停止すると制御も止まる)。freezeさせるとrunを畳めなくなるので進めない');
|
|
1989
|
+
}
|
|
1990
|
+
const recorded = await recordCheckpoints(escalation, probed.checkpoints);
|
|
1897
1991
|
if (!recorded.ok) return decide(recorded.outcome, recorded.detail);
|
|
1898
1992
|
|
|
1899
1993
|
const submit = async (operation, { artifact = null, artifactDigest = null,
|
|
@@ -1905,13 +1999,15 @@ export async function runManagedSupervisorDaemon({
|
|
|
1905
1999
|
artifact, artifactDigest, checkpointDigest,
|
|
1906
2000
|
expectedEpoch: recorded.expected_epoch, expectedQueueDigest: null }),
|
|
1907
2001
|
});
|
|
1908
|
-
return handler(controlRequest);
|
|
2002
|
+
return dispatchControl === null ? handler(controlRequest) : dispatchControl(controlRequest);
|
|
1909
2003
|
};
|
|
1910
2004
|
const unmetOf = (response) => (response?.result?.unmet ?? []).join('/')
|
|
1911
2005
|
|| String(response?.outcome ?? 'unknown');
|
|
1912
2006
|
|
|
1913
2007
|
const findingResponse = await submit('finding_record', {
|
|
1914
|
-
|
|
2008
|
+
// control operationが要求するのはartifact全体のdigestである。`candidate_digest`は
|
|
2009
|
+
// 自分の欄を除いた自己digestなので、そのまま渡すとbindingが合わず必ず弾かれる。
|
|
2010
|
+
artifact: escalation.candidate, artifactDigest: digestArtifact(escalation.candidate),
|
|
1915
2011
|
checkpointDigest: escalation.checkpoint_digest,
|
|
1916
2012
|
}).catch((error) => ({ outcome: 'rejected',
|
|
1917
2013
|
result: { unmet: [String(error?.code ?? error?.message ?? error)] } }));
|
|
@@ -1935,6 +2031,17 @@ export async function runManagedSupervisorDaemon({
|
|
|
1935
2031
|
return decide('held', `早期警報からhold: ${warning.kind}/${warning.path}`, findingDigest);
|
|
1936
2032
|
};
|
|
1937
2033
|
|
|
2034
|
+
/**
|
|
2035
|
+
* epoch駆動中に届いた、実在確認済みの警報。
|
|
2036
|
+
*
|
|
2037
|
+
* **駆動中はここへ積むだけにする。** epoch駆動はrun eventsをメモリに抱えたままawaitを
|
|
2038
|
+
* またぎ、節目ごとに全体を置換する。その最中に横から追記すると、次の置換で消える——
|
|
2039
|
+
* 静かに記録を失う方向に壊れる。捌くのは駆動側の安全点(disk とメモリが一致している点)
|
|
2040
|
+
* であり、そこはworkerがまだ走っている最中なのでholdが成立する。
|
|
2041
|
+
*/
|
|
2042
|
+
const pendingEscalations = [];
|
|
2043
|
+
let epochDriveActive = false;
|
|
2044
|
+
|
|
1938
2045
|
/**
|
|
1939
2046
|
* 警報1件の全行程(probe → 記録 → escalation)。
|
|
1940
2047
|
*
|
|
@@ -1945,6 +2052,11 @@ export async function runManagedSupervisorDaemon({
|
|
|
1945
2052
|
const handleIoWarning = async (warning, packets) => {
|
|
1946
2053
|
const probed = await probeWarning(warning).catch(() => (
|
|
1947
2054
|
{ outcome: 'unprobed', writers: [], checkpoints: {} }));
|
|
2055
|
+
if (probed.outcome === 'observed' && epochDriveActive) {
|
|
2056
|
+
pendingEscalations.push({ warning, probed, packets });
|
|
2057
|
+
await recordIoWarning(warning, 'observed');
|
|
2058
|
+
return;
|
|
2059
|
+
}
|
|
1948
2060
|
const decided = probed.outcome === 'observed'
|
|
1949
2061
|
? await escalateIoWarning(warning, probed, packets).catch(() => null)
|
|
1950
2062
|
: null;
|
|
@@ -1952,6 +2064,36 @@ export async function runManagedSupervisorDaemon({
|
|
|
1952
2064
|
decided === null || decided.outcome === 'skipped' ? probed.outcome : 'escalated');
|
|
1953
2065
|
};
|
|
1954
2066
|
|
|
2067
|
+
/**
|
|
2068
|
+
* epoch駆動の安全点でescalationを捌く(ADR 0143)。
|
|
2069
|
+
*
|
|
2070
|
+
* 呼ぶのは`replaceEventsAtomically`の直後だけとする。そこではdiskとメモリのeventsが
|
|
2071
|
+
* 一致しているので、control operationがdiskを読み書きしても駆動側の像とずれない。
|
|
2072
|
+
* 捌いた後はdiskを読み直して返す——`conflict`と`hold`はevents.jsonを書き換えるため。
|
|
2073
|
+
*
|
|
2074
|
+
* lifecycle lockは取らない。既に駆動側(activate)が握っている内側であり、ここで
|
|
2075
|
+
* 取り直すと自分自身を待つ。同じ理由で`handler`ではなく`executeControl`を直接使う。
|
|
2076
|
+
*
|
|
2077
|
+
* @returns {Promise<Array|null>} 捌いた結果のevents。何も捌かなければnull。
|
|
2078
|
+
*/
|
|
2079
|
+
const drainPendingEscalations = async (events) => {
|
|
2080
|
+
if (pendingEscalations.length === 0) return null;
|
|
2081
|
+
let current = events;
|
|
2082
|
+
while (pendingEscalations.length > 0) {
|
|
2083
|
+
const { warning, probed, packets } = pendingEscalations.shift();
|
|
2084
|
+
const decided = await escalateIoWarning(warning, probed, packets, {
|
|
2085
|
+
recordCheckpoints: recordProbeCheckpointsUnlocked,
|
|
2086
|
+
dispatchControl: (controlRequest) => executeControl(controlRequest),
|
|
2087
|
+
}).catch((error) => ({ outcome: 'rejected',
|
|
2088
|
+
detail: String(error?.message ?? error), finding_digest: null }));
|
|
2089
|
+
if (decided === null) continue;
|
|
2090
|
+
current = await readBoundedJson(path.join(runDir, 'events.json'), 'run events');
|
|
2091
|
+
// holdが掛かったら、残りの警報は同じfreezeの下にある。二重にholdを試みない。
|
|
2092
|
+
if (projectRuntimeState({ events: current }).freeze !== null) break;
|
|
2093
|
+
}
|
|
2094
|
+
return current;
|
|
2095
|
+
};
|
|
2096
|
+
|
|
1955
2097
|
/** 警報の処理を直列化する鎖。`onWarning`はここへ繋ぐだけにする。 */
|
|
1956
2098
|
let escalationChain = Promise.resolve();
|
|
1957
2099
|
|
|
@@ -2080,17 +2222,24 @@ export async function runManagedSupervisorDaemon({
|
|
|
2080
2222
|
return settled;
|
|
2081
2223
|
},
|
|
2082
2224
|
});
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2225
|
+
epochDriveActive = true;
|
|
2226
|
+
try {
|
|
2227
|
+
await driveInitialScriptedManagedEpoch({
|
|
2228
|
+
runDir,
|
|
2229
|
+
repoRoot,
|
|
2230
|
+
request,
|
|
2231
|
+
committed,
|
|
2232
|
+
activation,
|
|
2233
|
+
managedSupervisor,
|
|
2234
|
+
sentinel,
|
|
2235
|
+
initialEvents: events,
|
|
2236
|
+
controlEvents: () => controlEvents,
|
|
2237
|
+
preDispatchBindings,
|
|
2238
|
+
drainEscalations: drainPendingEscalations,
|
|
2239
|
+
});
|
|
2240
|
+
} finally {
|
|
2241
|
+
epochDriveActive = false;
|
|
2242
|
+
}
|
|
2094
2243
|
}
|
|
2095
2244
|
if (restarting) {
|
|
2096
2245
|
await managedSupervisor.recoveryBarrier({ barrierId: `recovery-${randomUUID()}`,
|
|
@@ -158,7 +158,7 @@ export function validateRuntimeControlEventPayload(kind, value) {
|
|
|
158
158
|
}
|
|
159
159
|
// I/O sentinelの早期警報(ADR 0143)。**findingではない**——検知の正本はcheckpointのままで、
|
|
160
160
|
// これは「早くcheckpointを撮って確かめろ」という引き金の記録である。記録しない選択肢は無い:
|
|
161
|
-
//
|
|
161
|
+
// 機械が何かに気づいたのに黙っている状態を残さない。
|
|
162
162
|
if (kind === 'io_warning_observed') {
|
|
163
163
|
return exact(value, ['warning_kind', 'todo_ids', 'path', 'probe_outcome', 'warning_digest'])
|
|
164
164
|
&& ['io_overlap_warning', 'io_scope_warning'].includes(value.warning_kind)
|
|
@@ -31,8 +31,8 @@ const WIRES = Object.freeze({
|
|
|
31
31
|
dispatch: {
|
|
32
32
|
request: ['schema', 'request_id', 'registration_digest', 'packet', 'write_lease', 'request_digest'],
|
|
33
33
|
requestSchema: 'lattice.adapter_dispatch_request.v1',
|
|
34
|
-
response: ['schema', 'request_id', 'executor_handle', 'worktree_id', 'packet_digest', 'lease_digest', 'response_digest'],
|
|
35
|
-
responseSchema: 'lattice.adapter_dispatch_response.
|
|
34
|
+
response: ['schema', 'request_id', 'executor_handle', 'worktree_id', 'packet_digest', 'lease_digest', 'worker_process', 'response_digest'],
|
|
35
|
+
responseSchema: 'lattice.adapter_dispatch_response.v2',
|
|
36
36
|
},
|
|
37
37
|
observe: {
|
|
38
38
|
request: ['schema', 'request_id', 'registration_digest', 'executor_handle', 'expected_epoch', 'expected_lease_digest', 'request_digest'],
|
|
@@ -338,8 +338,12 @@ export function validateControllerResponse(operation, value, expectedRequestId =
|
|
|
338
338
|
if (!(Boolean(wire) && exact(value, wire.response) && value.schema === wire.responseSchema
|
|
339
339
|
&& identifier(value.request_id) && (expectedRequestId === null || value.request_id === expectedRequestId)
|
|
340
340
|
&& selfValid(value, 'response_digest'))) return false;
|
|
341
|
+
// v2で`worker_process`を必須にした。executorがcontroller自身のprocessだと、holdが
|
|
342
|
+
// 要求する静止を証明できない(止めれば応答できず、止めなければ証明できない)。
|
|
343
|
+
// 誰を止めればよいかをdispatchの時点で名指しさせる。
|
|
341
344
|
if (operation === 'dispatch') return identifier(value.executor_handle) && identifier(value.worktree_id)
|
|
342
|
-
&& digest(value.packet_digest) && digest(value.lease_digest)
|
|
345
|
+
&& digest(value.packet_digest) && digest(value.lease_digest)
|
|
346
|
+
&& validateExpectedWorkerProcess(value.worker_process);
|
|
343
347
|
if (operation === 'observe') return exact(value.observation, ['schema', 'state', 'executor_handle', 'plan_epoch', 'lease_digest', 'payload_digest', 'observation_digest'])
|
|
344
348
|
&& value.observation.schema === 'lattice.adapter_observation.v1'
|
|
345
349
|
&& ['running', 'checkpoint_ready', 'terminal', 'held'].includes(value.observation.state)
|
|
@@ -397,6 +401,20 @@ function validateProtocolRunningBinding(value) {
|
|
|
397
401
|
&& identifier(value.write_lease_id) && digest(value.controller_registration_digest);
|
|
398
402
|
}
|
|
399
403
|
|
|
404
|
+
/**
|
|
405
|
+
* dispatchが名指しするworker process。直接OS観測が期待するchild processと同じ形にする
|
|
406
|
+
* ——照合先の形が分かれると、supervisorとcontrollerが別のものを見ていても気づけない。
|
|
407
|
+
*/
|
|
408
|
+
export function validateExpectedWorkerProcess(value) {
|
|
409
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value)
|
|
410
|
+
&& Object.keys(value).sort().join('\0')
|
|
411
|
+
=== ['pid', 'process_group_id', 'process_start_identity'].sort().join('\0')
|
|
412
|
+
&& Number.isSafeInteger(value.pid) && value.pid > 0
|
|
413
|
+
&& Number.isSafeInteger(value.process_group_id) && value.process_group_id > 0
|
|
414
|
+
&& validateProcessStartIdentity(value.process_start_identity)
|
|
415
|
+
&& value.process_start_identity.pid === value.pid;
|
|
416
|
+
}
|
|
417
|
+
|
|
400
418
|
export function validateQuiescenceAck(value) {
|
|
401
419
|
return exact(value, ['schema', 'ack_id', 'run_id', 'todo_id', 'executor_handle', 'worktree_id', 'plan_epoch', 'packet_digest', 'write_lease_id', 'barrier_control_digest', 'final_checkpoint_digest', 'process_observation_digest', 'worktree_fingerprint_digest', 'supervisor_session_nonce_digest', 'ack_digest'])
|
|
402
420
|
&& value.schema === 'lattice.executor_quiescence_ack.v1'
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import net from 'node:net';
|
|
2
2
|
import { createHash, randomBytes } from 'node:crypto';
|
|
3
|
-
import { execFile } from 'node:child_process';
|
|
3
|
+
import { execFile, spawn } from 'node:child_process';
|
|
4
4
|
import { constants as fsConstants, readFileSync } from 'node:fs';
|
|
5
5
|
import {
|
|
6
6
|
chmod,
|
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
rm,
|
|
14
14
|
} from 'node:fs/promises';
|
|
15
15
|
import path from 'node:path';
|
|
16
|
+
import { fileURLToPath } from 'node:url';
|
|
16
17
|
import { promisify } from 'node:util';
|
|
17
18
|
|
|
18
19
|
import { canonicalizeArtifact, digestArtifact } from './artifact-contracts.mjs';
|
|
@@ -29,6 +30,7 @@ import {
|
|
|
29
30
|
validateExecutorPacket,
|
|
30
31
|
validateExecutorReceipt,
|
|
31
32
|
} from './runtime-contracts.mjs';
|
|
33
|
+
import { captureWorktreeDiff } from './runtime-diff-observer.mjs';
|
|
32
34
|
import { validateSupervisorWriteGate } from './runtime-gate-store.mjs';
|
|
33
35
|
import { observeManagedProcessStartIdentity } from './runtime-managed-supervisor.mjs';
|
|
34
36
|
import { scriptedWorktreeId, scriptedWorktreePath } from './runtime-scripted-worktree.mjs';
|
|
@@ -251,15 +253,25 @@ function deterministicWriteBytes(packet, relativePath) {
|
|
|
251
253
|
})}\n`);
|
|
252
254
|
}
|
|
253
255
|
|
|
254
|
-
|
|
256
|
+
/**
|
|
257
|
+
* packetの宣言writeを実行する。**worker processから呼ばれる。**
|
|
258
|
+
*
|
|
259
|
+
* controllerと同じprocessで走らせていた頃、このrunには走行中のTODOが存在せず、
|
|
260
|
+
* 実行時の観測も静止の証明も原理的に成立しなかった(ADR 0143 Decision 7・9)。
|
|
261
|
+
*/
|
|
262
|
+
export async function executePacket({ packet, repoRoot, extraWrites = [] }) {
|
|
255
263
|
const writes = packet.scope?.writes;
|
|
256
264
|
if (!Array.isArray(writes) || writes.length === 0
|
|
257
265
|
|| writes.some((entry, index) => !safeWritePath(entry)
|
|
258
266
|
|| (index > 0 && writes[index - 1] >= entry))) {
|
|
259
267
|
fail('SCRIPTED_PACKET_REJECTED', 'scope.writesは非空の昇順一意pathでなければならない');
|
|
260
268
|
}
|
|
269
|
+
// 宣言scope外への書き込みは、検知そのものを検証するために要る。writeがすべて宣言内に
|
|
270
|
+
// 収まる限り、実行時competitionは原理的に一度も起きないので、検知経路を実runで通せない。
|
|
271
|
+
// 宣言と実writeが食い違う状態を意図して作れることが、この面の受入条件である。
|
|
272
|
+
const allWrites = [...new Set([...writes, ...extraWrites])].sort();
|
|
261
273
|
const prepared = [];
|
|
262
|
-
for (const relativePath of
|
|
274
|
+
for (const relativePath of allWrites) {
|
|
263
275
|
const target = await requireSafeTarget(repoRoot, relativePath);
|
|
264
276
|
const existedAtBase = await basePathExists(repoRoot, packet.base_sha, relativePath);
|
|
265
277
|
prepared.push({
|
|
@@ -318,6 +330,122 @@ async function readAndValidateGate(runDir, writeLease) {
|
|
|
318
330
|
* supervisor controller protocolを実装する決定論的scripted adapter。
|
|
319
331
|
* 時刻はheartbeat schedulingだけに使い、write bytes/handle/receiptへ混入させない。
|
|
320
332
|
*/
|
|
333
|
+
/**
|
|
334
|
+
* 登録済みadapter configから、このcontrollerの振る舞いを読む。
|
|
335
|
+
*
|
|
336
|
+
* **configはregistrationのdigestへ束縛されている。** repo内の任意のfileを信用するのではなく、
|
|
337
|
+
* `run adapter register`が記録した`config_digest`と一致するbytesだけを受ける。一致しなければ
|
|
338
|
+
* 既定の振る舞いへ落とすのではなく止める——registrationと実configがずれた状態で走ると、
|
|
339
|
+
* 記録されたrunの再現性が壊れる。
|
|
340
|
+
*
|
|
341
|
+
* 読むのは3つだけ:
|
|
342
|
+
* - `hold_ms`: 書き込み後にworkerが走り続ける時間。実行時観測が成立する窓を作る。
|
|
343
|
+
* - `extra_writes`: 宣言scope外へのwrite。競合検知そのものを検証するために要る。
|
|
344
|
+
* - `mode`: 既存の`deterministic`のみ。未知の値は黙って無視せず止める。
|
|
345
|
+
*/
|
|
346
|
+
async function readScriptedBehavior(repoRoot) {
|
|
347
|
+
const descriptorPath = path.join(repoRoot, '.lattice', 'runtime', 'adapter-registry',
|
|
348
|
+
'descriptors', 'scripted.json');
|
|
349
|
+
let descriptor;
|
|
350
|
+
try {
|
|
351
|
+
descriptor = JSON.parse(await readFile(descriptorPath, 'utf8'));
|
|
352
|
+
} catch {
|
|
353
|
+
// 未登録のまま走らせる経路(unit test等)は既定の振る舞いで動かす。
|
|
354
|
+
return { hold_ms: 0, extra_writes: [] };
|
|
355
|
+
}
|
|
356
|
+
if (typeof descriptor?.config_ref !== 'string' || typeof descriptor.config_digest !== 'string') {
|
|
357
|
+
return { hold_ms: 0, extra_writes: [] };
|
|
358
|
+
}
|
|
359
|
+
const configPath = path.join(repoRoot, ...descriptor.config_ref.split('/'));
|
|
360
|
+
let bytes;
|
|
361
|
+
try {
|
|
362
|
+
bytes = await readFile(configPath);
|
|
363
|
+
} catch {
|
|
364
|
+
fail('SCRIPTED_BOOTSTRAP_INVALID', '登録済みadapter configを読めない', {
|
|
365
|
+
config_ref: descriptor.config_ref,
|
|
366
|
+
});
|
|
367
|
+
}
|
|
368
|
+
if (sha256Bytes(bytes) !== descriptor.config_digest) {
|
|
369
|
+
fail('SCRIPTED_BOOTSTRAP_INVALID', 'adapter configが登録時のdigestと一致しない', {
|
|
370
|
+
config_ref: descriptor.config_ref,
|
|
371
|
+
});
|
|
372
|
+
}
|
|
373
|
+
let config;
|
|
374
|
+
try {
|
|
375
|
+
config = JSON.parse(bytes.toString('utf8'));
|
|
376
|
+
} catch {
|
|
377
|
+
fail('SCRIPTED_BOOTSTRAP_INVALID', 'adapter configのJSONが不正');
|
|
378
|
+
}
|
|
379
|
+
if (config?.mode !== undefined && config.mode !== 'deterministic') {
|
|
380
|
+
fail('SCRIPTED_BOOTSTRAP_INVALID', `未知のscripted mode: ${String(config.mode)}`);
|
|
381
|
+
}
|
|
382
|
+
const holdMs = config?.hold_ms ?? 0;
|
|
383
|
+
if (!Number.isSafeInteger(holdMs) || holdMs < 0 || holdMs > 600_000) {
|
|
384
|
+
fail('SCRIPTED_BOOTSTRAP_INVALID', 'hold_msが不正');
|
|
385
|
+
}
|
|
386
|
+
const extraWrites = config?.extra_writes ?? [];
|
|
387
|
+
if (!Array.isArray(extraWrites) || extraWrites.some((entry) => !safeWritePath(entry))) {
|
|
388
|
+
fail('SCRIPTED_BOOTSTRAP_INVALID', 'extra_writesが不正');
|
|
389
|
+
}
|
|
390
|
+
return { hold_ms: holdMs, extra_writes: [...extraWrites] };
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
const WORKER_ENTRYPOINT = fileURLToPath(new URL('../bin/lattice-scripted-worker.mjs', import.meta.url));
|
|
394
|
+
|
|
395
|
+
/**
|
|
396
|
+
* worker processを起こす(ADR 0143 Decision 9)。
|
|
397
|
+
*
|
|
398
|
+
* `detached`で独立process groupへ置く。controllerと同じgroupに居ると、直接OS観測が
|
|
399
|
+
* 「未記録process group memberを検出」として落とす——正しく落ちるので、ここを外さない。
|
|
400
|
+
*
|
|
401
|
+
* 起動直後にprocess start identityを観測して束ねる。pidは再利用されうるので、
|
|
402
|
+
* pidだけでは「同じprocessか」を後から言えない。
|
|
403
|
+
*/
|
|
404
|
+
async function spawnScriptedWorker({ packet, worktreePath, extraWrites, holdMs,
|
|
405
|
+
controllerId, runDir }) {
|
|
406
|
+
const job = {
|
|
407
|
+
schema: 'lattice.scripted_worker_job.v1',
|
|
408
|
+
packet: structuredClone(packet),
|
|
409
|
+
worktree_path: worktreePath,
|
|
410
|
+
extra_writes: [...extraWrites],
|
|
411
|
+
hold_ms: holdMs,
|
|
412
|
+
};
|
|
413
|
+
const jobPath = path.join(runDir, 'controllers', controllerId, 'jobs',
|
|
414
|
+
`${packet.packet_digest}.json`);
|
|
415
|
+
await durableReplaceBytes(jobPath, Buffer.from(`${canonicalizeArtifact(job)}\n`));
|
|
416
|
+
const child = spawn(process.execPath, [WORKER_ENTRYPOINT, jobPath], {
|
|
417
|
+
detached: true, stdio: ['ignore', 'pipe', 'pipe'], cwd: worktreePath,
|
|
418
|
+
});
|
|
419
|
+
if (!Number.isSafeInteger(child.pid)) {
|
|
420
|
+
fail('SCRIPTED_EXECUTION_FAILED', 'worker processを起こせない');
|
|
421
|
+
}
|
|
422
|
+
const identity = await observeManagedProcessStartIdentity(child.pid);
|
|
423
|
+
let stdout = '';
|
|
424
|
+
let stderr = '';
|
|
425
|
+
child.stdout.on('data', (chunk) => { stdout += chunk.toString('utf8'); });
|
|
426
|
+
child.stderr.on('data', (chunk) => { stderr += chunk.toString('utf8'); });
|
|
427
|
+
const completed = new Promise((resolve, reject) => {
|
|
428
|
+
child.once('error', reject);
|
|
429
|
+
child.once('close', (code, signal) => {
|
|
430
|
+
const line = stdout.split('\n').find((entry) => entry.includes('scripted_worker_result'));
|
|
431
|
+
let parsed = null;
|
|
432
|
+
try { parsed = line === undefined ? null : JSON.parse(line); } catch { parsed = null; }
|
|
433
|
+
if (parsed?.schema !== 'lattice.scripted_worker_result.v1') {
|
|
434
|
+
reject(new TypeError(`worker結果を読めない (${signal ?? code}): ${stderr.trim() || '(出力なし)'}`));
|
|
435
|
+
return;
|
|
436
|
+
}
|
|
437
|
+
resolve({ observedDiff: parsed.observed_diff, checkpointDigest: parsed.checkpoint_digest });
|
|
438
|
+
});
|
|
439
|
+
});
|
|
440
|
+
return {
|
|
441
|
+
pid: child.pid,
|
|
442
|
+
process_group_id: child.pid,
|
|
443
|
+
process_start_identity: identity,
|
|
444
|
+
child,
|
|
445
|
+
completed,
|
|
446
|
+
};
|
|
447
|
+
}
|
|
448
|
+
|
|
321
449
|
export async function createScriptedAdapterController({
|
|
322
450
|
bootstrap,
|
|
323
451
|
runDir,
|
|
@@ -373,6 +501,7 @@ export async function createScriptedAdapterController({
|
|
|
373
501
|
descriptor_digest: '',
|
|
374
502
|
}, 'descriptor_digest');
|
|
375
503
|
const sessionNonceDigest = digestArtifact(bootstrap.supervisor_session_nonce);
|
|
504
|
+
const behavior = await readScriptedBehavior(canonicalRepoRoot);
|
|
376
505
|
const stagedLeases = new Map();
|
|
377
506
|
const armedLeases = new Map();
|
|
378
507
|
const preparedPackets = new Map();
|
|
@@ -381,6 +510,13 @@ export async function createScriptedAdapterController({
|
|
|
381
510
|
let currentEpoch = 1;
|
|
382
511
|
let registrationDigest = null;
|
|
383
512
|
|
|
513
|
+
/** dispatch応答が運ぶworker process。誰を止めればよいかをsupervisorへ名指しする。 */
|
|
514
|
+
const workerProcessOf = (task) => ({
|
|
515
|
+
pid: task.worker.pid,
|
|
516
|
+
process_group_id: task.worker.process_group_id,
|
|
517
|
+
process_start_identity: structuredClone(task.worker.process_start_identity),
|
|
518
|
+
});
|
|
519
|
+
|
|
384
520
|
const persistReceipt = async (receipt) => {
|
|
385
521
|
const payloadDigest = digestArtifact(receipt);
|
|
386
522
|
const receiptPath = path.join(
|
|
@@ -529,12 +665,13 @@ export async function createScriptedAdapterController({
|
|
|
529
665
|
fail('SCRIPTED_DUPLICATE_DISPATCH', '同じTODOへ異なるpacketをdispatchできない');
|
|
530
666
|
}
|
|
531
667
|
return sign({
|
|
532
|
-
schema: 'lattice.adapter_dispatch_response.
|
|
668
|
+
schema: 'lattice.adapter_dispatch_response.v2',
|
|
533
669
|
request_id: request.request_id,
|
|
534
670
|
executor_handle: prior.executorHandle,
|
|
535
671
|
worktree_id: prior.worktreeId,
|
|
536
672
|
packet_digest: packet.packet_digest,
|
|
537
673
|
lease_digest: writeLease.lease_digest,
|
|
674
|
+
worker_process: workerProcessOf(prior),
|
|
538
675
|
response_digest: '',
|
|
539
676
|
}, 'response_digest');
|
|
540
677
|
}
|
|
@@ -551,47 +688,78 @@ export async function createScriptedAdapterController({
|
|
|
551
688
|
worktree_path: worktreePath,
|
|
552
689
|
});
|
|
553
690
|
}
|
|
554
|
-
const { observedDiff, checkpointDigest } = await executePacket({
|
|
555
|
-
packet,
|
|
556
|
-
repoRoot: await realpath(worktreePath),
|
|
557
|
-
});
|
|
558
691
|
const executorHandle = `scripted-${packet.packet_digest.slice(0, 24)}`;
|
|
559
692
|
const worktreeId = scriptedWorktreeId(packet);
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
693
|
+
// **dispatchで作業を終わらせない。** 終わらせると、走行中のTODOが1つも存在しない
|
|
694
|
+
// runになり、実行時の観測——書き込みを見て、他のworkerとの重なりを掴む——が
|
|
695
|
+
// 原理的に成立しない。dispatchは作業を起こして返り、完了はobserveが拾う。
|
|
696
|
+
const progress = {
|
|
697
|
+
schema: 'lattice.scripted_adapter_progress.v1',
|
|
698
|
+
run_id: bootstrap.run_id,
|
|
699
|
+
todo_id: packet.todo_id,
|
|
563
700
|
executor_handle: executorHandle,
|
|
564
701
|
worktree_id: worktreeId,
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
}
|
|
576
|
-
const payloadDigest = await persistReceipt(receipt);
|
|
702
|
+
state: 'running',
|
|
703
|
+
};
|
|
704
|
+
const worktreeReal = await realpath(worktreePath);
|
|
705
|
+
// **workerは別processで起こす。** controller自身のprocessで作業すると、holdが
|
|
706
|
+
// 要求する静止を証明できない——止めれば応答できず、止めなければ証明できない。
|
|
707
|
+
// `detached`で独立process groupへ置く。同じgroupにcontrollerが居ると、直接OS観測が
|
|
708
|
+
// 「未記録process group memberを検出」として正しく落とす。
|
|
709
|
+
const spawned = await spawnScriptedWorker({
|
|
710
|
+
packet, worktreePath: worktreeReal, extraWrites: behavior.extra_writes,
|
|
711
|
+
holdMs: behavior.hold_ms, controllerId, runDir: canonicalRunDir,
|
|
712
|
+
});
|
|
577
713
|
const task = {
|
|
578
714
|
packet: structuredClone(packet),
|
|
579
715
|
lease: structuredClone(writeLease),
|
|
580
716
|
executorHandle,
|
|
581
717
|
worktreeId,
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
718
|
+
worktreePath: worktreeReal,
|
|
719
|
+
receipt: null,
|
|
720
|
+
payloadDigest: digestArtifact(progress),
|
|
721
|
+
state: 'running',
|
|
722
|
+
failure: null,
|
|
723
|
+
settled: null,
|
|
724
|
+
worker: spawned,
|
|
585
725
|
};
|
|
586
726
|
tasks.set(executorHandle, task);
|
|
587
727
|
todoToHandle.set(packet.todo_id, executorHandle);
|
|
728
|
+
task.settled = (async () => {
|
|
729
|
+
try {
|
|
730
|
+
const { observedDiff, checkpointDigest } = await spawned.completed;
|
|
731
|
+
const receipt = sign({
|
|
732
|
+
schema: 'lattice.executor_receipt.v1',
|
|
733
|
+
receipt_id: `receipt-${packet.packet_digest.slice(0, 24)}`,
|
|
734
|
+
executor_handle: executorHandle,
|
|
735
|
+
worktree_id: worktreeId,
|
|
736
|
+
base_sha: packet.base_sha,
|
|
737
|
+
plan_epoch: packet.plan_epoch,
|
|
738
|
+
packet_digest: packet.packet_digest,
|
|
739
|
+
todo_id: packet.todo_id,
|
|
740
|
+
checkpoint_digest: checkpointDigest,
|
|
741
|
+
observed_diff: observedDiff,
|
|
742
|
+
receipt_digest: '',
|
|
743
|
+
}, 'receipt_digest');
|
|
744
|
+
if (!validateExecutorReceipt(receipt)) {
|
|
745
|
+
throw new TypeError('生成receiptがexecutor contractを満たさない');
|
|
746
|
+
}
|
|
747
|
+
task.payloadDigest = await persistReceipt(receipt);
|
|
748
|
+
task.receipt = receipt;
|
|
749
|
+
task.state = 'terminal';
|
|
750
|
+
} catch (error) {
|
|
751
|
+
// 失敗を走行中のまま放置しない。observeがtypedに落ちる形へ残す。
|
|
752
|
+
task.failure = String(error?.detail?.reason ?? error?.message ?? error);
|
|
753
|
+
}
|
|
754
|
+
})();
|
|
588
755
|
return sign({
|
|
589
|
-
schema: 'lattice.adapter_dispatch_response.
|
|
756
|
+
schema: 'lattice.adapter_dispatch_response.v2',
|
|
590
757
|
request_id: request.request_id,
|
|
591
758
|
executor_handle: executorHandle,
|
|
592
759
|
worktree_id: worktreeId,
|
|
593
760
|
packet_digest: packet.packet_digest,
|
|
594
761
|
lease_digest: writeLease.lease_digest,
|
|
762
|
+
worker_process: workerProcessOf(task),
|
|
595
763
|
response_digest: '',
|
|
596
764
|
}, 'response_digest');
|
|
597
765
|
}
|
|
@@ -602,6 +770,10 @@ export async function createScriptedAdapterController({
|
|
|
602
770
|
|| task.lease.lease_digest !== request.expected_lease_digest) {
|
|
603
771
|
fail('SCRIPTED_OBSERVATION_REJECTED', 'observe bindingがdispatch記録と一致しない');
|
|
604
772
|
}
|
|
773
|
+
// 走行中に落ちた作業を「まだ走っている」と言い続けない。
|
|
774
|
+
if (task.failure !== null) {
|
|
775
|
+
fail('SCRIPTED_EXECUTION_FAILED', 'worker実行が失敗した', { reason: task.failure });
|
|
776
|
+
}
|
|
605
777
|
const observation = sign({
|
|
606
778
|
schema: 'lattice.adapter_observation.v1',
|
|
607
779
|
state: task.state,
|
|
@@ -641,23 +813,48 @@ export async function createScriptedAdapterController({
|
|
|
641
813
|
}, 'response_digest');
|
|
642
814
|
}
|
|
643
815
|
if (operation === 'barrier') {
|
|
644
|
-
const acks =
|
|
816
|
+
const acks = [];
|
|
817
|
+
for (const binding of request.running_bindings) {
|
|
645
818
|
const task = tasks.get(binding.executor_handle);
|
|
646
819
|
if (task === undefined || task.packet.todo_id !== binding.todo_id) {
|
|
647
820
|
fail('SCRIPTED_BARRIER_REJECTED', 'barrierが未知のrunning bindingを含む');
|
|
648
821
|
}
|
|
822
|
+
// **barrierは静止の宣言である。** worker processを実際に止め、止まった木を読む。
|
|
823
|
+
// 走行中の作業を残したままackを返すと、「止まった」と言いながらworktreeが動き続ける。
|
|
824
|
+
try {
|
|
825
|
+
process.kill(task.worker.pid, 'SIGSTOP');
|
|
826
|
+
} catch (error) {
|
|
827
|
+
fail('SCRIPTED_BARRIER_REJECTED', 'worker processを止められない', {
|
|
828
|
+
pid: task.worker.pid, reason: String(error?.code ?? error?.message ?? error),
|
|
829
|
+
});
|
|
830
|
+
}
|
|
649
831
|
task.state = 'held';
|
|
832
|
+
// supervisorも同じ観測を独立に行い、3つのdigestの一致を要求する。**こちらが別の形で
|
|
833
|
+
// 作ると、両者が別のものを見ていても気づけない。** 形はdirect OS observerの契約に
|
|
834
|
+
// 揃え、入力(自分のprocess・自分の木)だけを独立に観測する。
|
|
835
|
+
const checkpoint = await captureWorktreeDiff({
|
|
836
|
+
worktreePath: task.worktreePath, baseSha: task.packet.base_sha,
|
|
837
|
+
});
|
|
650
838
|
const processObservationDigest = digestArtifact({
|
|
651
|
-
schema: 'lattice.
|
|
652
|
-
|
|
653
|
-
|
|
839
|
+
schema: 'lattice.direct_process_observation.v2',
|
|
840
|
+
root: {
|
|
841
|
+
pid: task.worker.pid,
|
|
842
|
+
parent_pid: process.pid,
|
|
843
|
+
process_start_identity_digest: task.worker.process_start_identity.identity_digest,
|
|
844
|
+
process_group_id: task.worker.process_group_id,
|
|
845
|
+
state: 'stopped',
|
|
846
|
+
},
|
|
847
|
+
children: [],
|
|
848
|
+
process_group_id: task.worker.process_group_id,
|
|
849
|
+
quiesced: true,
|
|
654
850
|
});
|
|
655
851
|
const worktreeFingerprintDigest = digestArtifact({
|
|
656
|
-
schema: 'lattice.
|
|
852
|
+
schema: 'lattice.direct_worktree_fingerprint.v1',
|
|
657
853
|
worktree_id: task.worktreeId,
|
|
658
|
-
|
|
854
|
+
worktree_realpath: task.worktreePath,
|
|
855
|
+
checkpoint_digest: checkpoint.checkpoint_digest,
|
|
659
856
|
});
|
|
660
|
-
|
|
857
|
+
acks.push(sign({
|
|
661
858
|
schema: 'lattice.executor_quiescence_ack.v1',
|
|
662
859
|
ack_id: `barrier-${binding.packet_digest.slice(0, 24)}`,
|
|
663
860
|
run_id: bootstrap.run_id,
|
|
@@ -668,13 +865,13 @@ export async function createScriptedAdapterController({
|
|
|
668
865
|
packet_digest: binding.packet_digest,
|
|
669
866
|
write_lease_id: binding.write_lease_id,
|
|
670
867
|
barrier_control_digest: request.barrier_control_digest,
|
|
671
|
-
final_checkpoint_digest:
|
|
868
|
+
final_checkpoint_digest: checkpoint.checkpoint_digest,
|
|
672
869
|
process_observation_digest: processObservationDigest,
|
|
673
870
|
worktree_fingerprint_digest: worktreeFingerprintDigest,
|
|
674
871
|
supervisor_session_nonce_digest: sessionNonceDigest,
|
|
675
872
|
ack_digest: '',
|
|
676
|
-
}, 'ack_digest');
|
|
677
|
-
}
|
|
873
|
+
}, 'ack_digest'));
|
|
874
|
+
}
|
|
678
875
|
return sign({
|
|
679
876
|
schema: 'lattice.adapter_barrier_response.v1',
|
|
680
877
|
request_id: request.request_id,
|