@quolu/lattice 0.28.0 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +148 -25
- package/README.ja.md +251 -0
- package/README.md +174 -127
- package/package.json +24 -5
- package/src/bridge-daemon.mjs +18 -6
- package/src/runtime-cli.mjs +285 -1
- package/src/runtime-control-store.mjs +30 -0
- package/src/runtime-decision-verifier.mjs +3 -0
- package/src/runtime-diff-observer.mjs +8 -1
- package/src/runtime-engine.mjs +5 -0
- package/src/runtime-io-sentinel.mjs +384 -0
- package/src/runtime-managed-supervisor.mjs +26 -25
- package/src/runtime-scripted-adapter-controller.mjs +15 -2
- package/src/runtime-scripted-worktree.mjs +104 -0
- package/src/runtime-socket-owner.mjs +125 -0
- package/src/todo-cli.mjs +14 -5
- package/src/todo-independence-guidance.mjs +62 -0
- package/src/todo-store.mjs +39 -0
- package/src/witness-scaffold.mjs +60 -11
package/src/runtime-cli.mjs
CHANGED
|
@@ -20,6 +20,10 @@ import { fileURLToPath } from 'node:url';
|
|
|
20
20
|
import { canonicalizeArtifact, digestArtifact } from './artifact-contracts.mjs';
|
|
21
21
|
import { collectSensorEvidence } from './sensor-adapter.mjs';
|
|
22
22
|
import { detectCheckpointFindings } from './runtime-diff-observer.mjs';
|
|
23
|
+
import {
|
|
24
|
+
buildIoEscalation, createRunSentinel, probeIoWarning, syncSentinelWatches,
|
|
25
|
+
} from './runtime-io-sentinel.mjs';
|
|
26
|
+
import { ensureScriptedWorktree, removeScriptedWorktrees } from './runtime-scripted-worktree.mjs';
|
|
23
27
|
import {
|
|
24
28
|
buildRuntimeSeamResolution, readRuntimeFindingRecord, resolveRuntimeSeam,
|
|
25
29
|
validateRuntimeSeamRequest, verifySeamSplitSuccessor,
|
|
@@ -126,6 +130,15 @@ const CLI_ERROR_SCHEMA = 'lattice.cli_error.v2';
|
|
|
126
130
|
const RUN_STORE_ROOT = ['.lattice', 'runs'];
|
|
127
131
|
const RUN_REF = /^\.lattice\/runs\/([0-9A-Za-z](?:[0-9A-Za-z._-]{0,127}))$/u;
|
|
128
132
|
const KNOWN_ADAPTERS = Object.freeze(['scripted', 'isolated-worktree', 'actual-agent']);
|
|
133
|
+
/**
|
|
134
|
+
* 自動escalationがlifecycle lockを待つ上限(ADR 0143)。
|
|
135
|
+
*
|
|
136
|
+
* 待つのは、警報が飛んだ瞬間にactivateやrecompileが走っていることが普通にあるからである。
|
|
137
|
+
* 待たずに諦めると、早期警報がいちばん効く場面——複数TODOが同時に動いている最中——で
|
|
138
|
+
* 必ず取り逃す。上限を置くのは、待ち続けてdaemonの他の仕事を止めないため。
|
|
139
|
+
* 超えたら`rejected`として理由ごとjournalへ残す(黙って見送らない)。
|
|
140
|
+
*/
|
|
141
|
+
const IO_ESCALATION_LOCK_TIMEOUT_MS = 15_000;
|
|
129
142
|
|
|
130
143
|
class CliContractError extends Error {
|
|
131
144
|
constructor(code, message, detail) {
|
|
@@ -736,6 +749,7 @@ async function driveInitialScriptedManagedEpoch({
|
|
|
736
749
|
managedSupervisor,
|
|
737
750
|
initialEvents,
|
|
738
751
|
controlEvents,
|
|
752
|
+
sentinel = null,
|
|
739
753
|
}) {
|
|
740
754
|
let events = [...initialEvents];
|
|
741
755
|
const { plan, manifests, executor_packets: packets } = committed.bundle;
|
|
@@ -758,6 +772,17 @@ async function driveInitialScriptedManagedEpoch({
|
|
|
758
772
|
'initial dispatch leaseのcontrol bindingが無い',
|
|
759
773
|
);
|
|
760
774
|
}
|
|
775
|
+
// workerの木を先に用意し、**dispatchの前から**監視を張る。dispatch応答を待つと、
|
|
776
|
+
// 応答の中で書き込みが終わっている構成では観測対象のイベントが一度も起きない。
|
|
777
|
+
// 木がTODOごとに分かれて初めて、書き込みの帰属をrootから決められる。
|
|
778
|
+
const worktreeByTodo = new Map();
|
|
779
|
+
for (const todoId of frontier) {
|
|
780
|
+
worktreeByTodo.set(todoId, await ensureScriptedWorktree({
|
|
781
|
+
repoRoot, runDir, packet: packets[todoId],
|
|
782
|
+
}));
|
|
783
|
+
}
|
|
784
|
+
syncSentinelWatches({ sentinel, runningTodoIds: [...frontier],
|
|
785
|
+
rootOf: (todoId) => worktreeByTodo.get(todoId) });
|
|
761
786
|
const stagedLeases = [];
|
|
762
787
|
for (const todoId of frontier) {
|
|
763
788
|
const packet = packets[todoId];
|
|
@@ -833,7 +858,8 @@ async function driveInitialScriptedManagedEpoch({
|
|
|
833
858
|
process_group_id: processGroupId,
|
|
834
859
|
process_start_identity:
|
|
835
860
|
structuredClone(activation.controllerDescriptor.process_start_identity),
|
|
836
|
-
|
|
861
|
+
// TODOごとの木を指す。ここがrepo rootだった頃、帰属はrootから決まらなかった。
|
|
862
|
+
worktree_path: worktreeByTodo.get(packet.todo_id),
|
|
837
863
|
base_sha: packet.base_sha,
|
|
838
864
|
},
|
|
839
865
|
};
|
|
@@ -879,6 +905,12 @@ async function driveInitialScriptedManagedEpoch({
|
|
|
879
905
|
}
|
|
880
906
|
events = dispatched.events;
|
|
881
907
|
await replaceEventsAtomically(runDir, events);
|
|
908
|
+
// 走り出した瞬間から見る。checkpointは完了時まで撮られないので、ここを逃すと
|
|
909
|
+
// 早期警報の意味が無くなる。
|
|
910
|
+
syncSentinelWatches({ sentinel, runningTodoIds: projectRuntimeState({ events }).running,
|
|
911
|
+
rootOf: (todoId) => events.findLast((event) => event.kind === 'executor_dispatched'
|
|
912
|
+
&& event.subject?.kind === 'todo' && event.subject.ref === todoId)
|
|
913
|
+
?.payload?.direct_os_observation_binding?.worktree_path });
|
|
882
914
|
for (const todoId of dispatched.dispatched) {
|
|
883
915
|
const observed = await observeExecutor({
|
|
884
916
|
runId: request.request_id,
|
|
@@ -890,6 +922,10 @@ async function driveInitialScriptedManagedEpoch({
|
|
|
890
922
|
});
|
|
891
923
|
events = observed.events;
|
|
892
924
|
await replaceEventsAtomically(runDir, events);
|
|
925
|
+
syncSentinelWatches({ sentinel, runningTodoIds: projectRuntimeState({ events }).running,
|
|
926
|
+
rootOf: (todoId) => events.findLast((event) => event.kind === 'executor_dispatched'
|
|
927
|
+
&& event.subject?.kind === 'todo' && event.subject.ref === todoId)
|
|
928
|
+
?.payload?.direct_os_observation_binding?.worktree_path });
|
|
893
929
|
}
|
|
894
930
|
const adjudicated = adjudicatePendingReceipts({
|
|
895
931
|
runId: request.request_id,
|
|
@@ -1686,6 +1722,7 @@ export async function runManagedSupervisorDaemon({
|
|
|
1686
1722
|
clock: canonicalNow });
|
|
1687
1723
|
let eventStore = requestStore;
|
|
1688
1724
|
let server;
|
|
1725
|
+
let sentinel = null;
|
|
1689
1726
|
|
|
1690
1727
|
const appendControl = async ({ run_id: runId, kind, session_nonce_digest: sessionDigest, payload }) => {
|
|
1691
1728
|
const eventDigest = await eventStore.append({ run_id: runId, kind,
|
|
@@ -1694,6 +1731,230 @@ export async function runManagedSupervisorDaemon({
|
|
|
1694
1731
|
return eventDigest;
|
|
1695
1732
|
};
|
|
1696
1733
|
let gateWriter = null;
|
|
1734
|
+
|
|
1735
|
+
/**
|
|
1736
|
+
* 警報を無停止のcheckpointで確かめる(ADR 0143の二段目)。
|
|
1737
|
+
*
|
|
1738
|
+
* probeが撮るのはgitから読んだ本物のdiffなので、そのままfindingの証拠になる。
|
|
1739
|
+
* fs eventをfindingへ昇格させる必要が無く、契約を1つも緩めずに済む。
|
|
1740
|
+
*/
|
|
1741
|
+
const probeWarning = async (warning) => {
|
|
1742
|
+
const events = await readBoundedJson(path.join(runDir, 'events.json'), 'run events')
|
|
1743
|
+
.catch(() => null);
|
|
1744
|
+
if (!Array.isArray(events)) {
|
|
1745
|
+
return { outcome: 'unprobed', writers: [], checkpoints: {}, roots: {} };
|
|
1746
|
+
}
|
|
1747
|
+
const checkpoints = {};
|
|
1748
|
+
const roots = {};
|
|
1749
|
+
for (const todoId of warning.todo_ids) {
|
|
1750
|
+
const dispatch = events.findLast((event) => event.kind === 'executor_dispatched'
|
|
1751
|
+
&& event.subject?.kind === 'todo' && event.subject.ref === todoId);
|
|
1752
|
+
const binding = dispatch?.payload?.direct_os_observation_binding;
|
|
1753
|
+
if (typeof binding?.worktree_path !== 'string' || typeof binding?.base_sha !== 'string') continue;
|
|
1754
|
+
roots[todoId] = binding.worktree_path;
|
|
1755
|
+
// 観測できないTODOは「書いていない」へ丸めない。probeIoWarningが未観測として扱う。
|
|
1756
|
+
const captured = await captureWorktreeDiff({
|
|
1757
|
+
worktreePath: binding.worktree_path, baseSha: binding.base_sha,
|
|
1758
|
+
}).catch(() => null);
|
|
1759
|
+
if (captured !== null) checkpoints[todoId] = captured;
|
|
1760
|
+
}
|
|
1761
|
+
if (Object.keys(checkpoints).length === 0) {
|
|
1762
|
+
return { outcome: 'unprobed', writers: [], checkpoints, roots };
|
|
1763
|
+
}
|
|
1764
|
+
return { ...probeIoWarning({ warning, checkpointsByTodo: checkpoints }), checkpoints, roots };
|
|
1765
|
+
};
|
|
1766
|
+
|
|
1767
|
+
/**
|
|
1768
|
+
* 書き手を特定できる構成かを確かめる。
|
|
1769
|
+
*
|
|
1770
|
+
* sentinelの帰属はworktree rootだけで決まる(プロセス帰属を持たない)。だから
|
|
1771
|
+
* **複数TODOが同じrootを共有している構成では、誰が書いたかを観測から言えない**。
|
|
1772
|
+
* 実際、管理daemonのscripted構成は全TODOのbindingが同じrepo rootを指す。
|
|
1773
|
+
*
|
|
1774
|
+
* そこでは警報も、probeが撮るcheckpointも、TODO間で区別が付かない——同じ木を2回読んで
|
|
1775
|
+
* 「両方が書いた」と読めてしまう。これをholdへ繋ぐと、無実のTODOを止める。
|
|
1776
|
+
* 帰属が立たないならescalationへ進めない。警報と実在の記録はそのまま残す。
|
|
1777
|
+
*/
|
|
1778
|
+
const attributionIsDistinct = (warning, roots) => {
|
|
1779
|
+
const paths = warning.todo_ids.map((todoId) => roots[todoId]);
|
|
1780
|
+
if (paths.some((value) => typeof value !== 'string')) return false;
|
|
1781
|
+
return new Set(paths).size === paths.length;
|
|
1782
|
+
};
|
|
1783
|
+
|
|
1784
|
+
/**
|
|
1785
|
+
* I/O sentinelの早期警報を耐久化する(ADR 0143)。
|
|
1786
|
+
*
|
|
1787
|
+
* **findingにはしない。** findingの契約はcheckpoint digestを必須にしており、それはfindingが
|
|
1788
|
+
* 事後に再読して再導出できる主張であることの担保である。fs eventは取りこぼすし再読もできない。
|
|
1789
|
+
* ここで残すのは「機械が気づいた」という事実だけで、判定の正本はcheckpointのままである。
|
|
1790
|
+
*
|
|
1791
|
+
* 記録しない選択肢は無い。気づいたのに黙っている状態を残さない(ADR 0130)。
|
|
1792
|
+
*/
|
|
1793
|
+
const warningDigestOf = (warning) => digestArtifact({
|
|
1794
|
+
warning_kind: warning.kind, todo_ids: [...warning.todo_ids].sort(), path: warning.path,
|
|
1795
|
+
});
|
|
1796
|
+
|
|
1797
|
+
const recordIoWarning = async (warning, probeOutcome) => {
|
|
1798
|
+
const payload = {
|
|
1799
|
+
warning_kind: warning.kind,
|
|
1800
|
+
todo_ids: [...warning.todo_ids].sort(),
|
|
1801
|
+
path: warning.path,
|
|
1802
|
+
probe_outcome: probeOutcome,
|
|
1803
|
+
warning_digest: warningDigestOf(warning),
|
|
1804
|
+
};
|
|
1805
|
+
await appendControl({ run_id: request.request_id, kind: 'io_warning_observed',
|
|
1806
|
+
session_nonce_digest: digestArtifact(sessionNonce), payload });
|
|
1807
|
+
};
|
|
1808
|
+
|
|
1809
|
+
/**
|
|
1810
|
+
* probeが撮ったcheckpointをrun eventへ耐久化する(ADR 0143の三段目・前段)。
|
|
1811
|
+
*
|
|
1812
|
+
* `finding_record`はcheckpointがactive epochのevent prefixから解決できることを要求する。
|
|
1813
|
+
* probeのcheckpointは他のcheckpointと同じくgitから読んだ実diffなので、findingの証拠に
|
|
1814
|
+
* そのまま使える。ただし**由来は印として残す**——これはexecutorの申告境界ではなく
|
|
1815
|
+
* supervisorが選んだ走行中の一点であり、receipt裁定のbinding基準に混ぜると、
|
|
1816
|
+
* probe後も書き続けた正当なreceiptがcheckpoint_mismatchで落ちる。
|
|
1817
|
+
* events.jsonは全体置換なので、lifecycle lockの内側でだけ触る。
|
|
1818
|
+
*
|
|
1819
|
+
* @returns {{ok: true, expected_epoch: number}|{ok: false, outcome: string, detail: string}}
|
|
1820
|
+
*/
|
|
1821
|
+
const durablyRecordProbeCheckpoints = async (escalation, checkpointsByTodo) => {
|
|
1822
|
+
let lock;
|
|
1823
|
+
try {
|
|
1824
|
+
lock = await acquireRuntimeLifecycleLock({ runDir,
|
|
1825
|
+
sessionNonceDigest: digestArtifact(sessionNonce), operation: 'finding_record',
|
|
1826
|
+
requestId: `${escalation.escalation_id}.checkpoint`,
|
|
1827
|
+
timeoutMs: IO_ESCALATION_LOCK_TIMEOUT_MS, retryIntervalMs: 25 });
|
|
1828
|
+
} catch (error) {
|
|
1829
|
+
return { ok: false, outcome: 'rejected',
|
|
1830
|
+
detail: `lifecycle lockを取れない: ${error?.code ?? 'RUN_BUSY'}` };
|
|
1831
|
+
}
|
|
1832
|
+
try {
|
|
1833
|
+
const active = await readCommittedEpochStore(runDir);
|
|
1834
|
+
if (active === null) return { ok: false, outcome: 'skipped', detail: 'managed epochが未commit' };
|
|
1835
|
+
const events = await readBoundedJson(path.join(runDir, 'events.json'), 'run events');
|
|
1836
|
+
const state = projectRuntimeState({ events });
|
|
1837
|
+
// 既にfreeze済みなら、この警報が指す競合はもう止まっている。二重にholdを掛けない。
|
|
1838
|
+
if (state.freeze !== null) {
|
|
1839
|
+
return { ok: false, outcome: 'skipped', detail: 'runは既にfreeze済み' };
|
|
1840
|
+
}
|
|
1841
|
+
// 走り終わったTODOはcheckpoint findingが従来どおり捕まえる。早期警報の出番ではない。
|
|
1842
|
+
const running = new Set(state.running);
|
|
1843
|
+
if (!escalation.writers.every((todoId) => running.has(todoId))) {
|
|
1844
|
+
return { ok: false, outcome: 'skipped', detail: '観測したTODOが既にrunningでない' };
|
|
1845
|
+
}
|
|
1846
|
+
let next = events;
|
|
1847
|
+
for (const todoId of escalation.writers) {
|
|
1848
|
+
const checkpoint = checkpointsByTodo[todoId];
|
|
1849
|
+
if (next.some((event) => event.kind === 'checkpoint_observed'
|
|
1850
|
+
&& event.payload?.checkpoint_digest === checkpoint.checkpoint_digest)) continue;
|
|
1851
|
+
next = [...next, buildNextRunEvent({ events: next, runId: request.request_id,
|
|
1852
|
+
kind: 'checkpoint_observed', planEpoch: active.pointer.plan_epoch,
|
|
1853
|
+
subject: { kind: 'todo', ref: todoId },
|
|
1854
|
+
payload: { ...structuredClone(checkpoint), observed_by: 'supervisor_probe' },
|
|
1855
|
+
recordedAt: canonicalNow() })];
|
|
1856
|
+
}
|
|
1857
|
+
if (next !== events) await replaceEventsAtomically(runDir, next);
|
|
1858
|
+
return { ok: true, expected_epoch: active.pointer.plan_epoch };
|
|
1859
|
+
} catch (error) {
|
|
1860
|
+
return { ok: false, outcome: 'rejected',
|
|
1861
|
+
detail: `probe checkpointを耐久化できない: ${error?.code ?? String(error?.message ?? error)}` };
|
|
1862
|
+
} finally {
|
|
1863
|
+
await lock.release();
|
|
1864
|
+
}
|
|
1865
|
+
};
|
|
1866
|
+
|
|
1867
|
+
/**
|
|
1868
|
+
* probeが実在と判定した警報を、既存のhold経路へ入れる(ADR 0143の三段目)。
|
|
1869
|
+
*
|
|
1870
|
+
* **新しい停止経路を作らない。** daemon自身が、hostが叩くのとまったく同じ
|
|
1871
|
+
* `finding_record`→`conflict`→`hold`をcontrol requestとして発行する。findingの再導出も、
|
|
1872
|
+
* epoch束縛も、durable evidenceの照合も、既存handlerがそのまま行う——早期警報が短くするのは
|
|
1873
|
+
* **気づくまでの時間**だけで、通す関門は1つも減らない。
|
|
1874
|
+
*
|
|
1875
|
+
* 途中で断られたらそこで止め、理由をjournalへ残す。静かに別経路へ逃げない。
|
|
1876
|
+
*/
|
|
1877
|
+
const escalateIoWarning = async (warning, probed, packets) => {
|
|
1878
|
+
const built = buildIoEscalation({ warning, probe: probed,
|
|
1879
|
+
checkpointsByTodo: probed.checkpoints, packets });
|
|
1880
|
+
if (built === null) return null;
|
|
1881
|
+
const escalation = { ...built, escalation_id: `io-esc-${randomUUID()}` };
|
|
1882
|
+
const decide = async (outcome, detail, findingDigest = null) => {
|
|
1883
|
+
await appendControl({ run_id: request.request_id, kind: 'io_escalation_decided',
|
|
1884
|
+
session_nonce_digest: digestArtifact(sessionNonce), payload: {
|
|
1885
|
+
warning_digest: warningDigestOf(warning),
|
|
1886
|
+
anchor_todo_id: escalation.anchor_todo_id,
|
|
1887
|
+
checkpoint_digest: escalation.checkpoint_digest,
|
|
1888
|
+
finding_digest: findingDigest, outcome, detail,
|
|
1889
|
+
} });
|
|
1890
|
+
return { outcome, detail, finding_digest: findingDigest };
|
|
1891
|
+
};
|
|
1892
|
+
if (!attributionIsDistinct(warning, probed.roots)) {
|
|
1893
|
+
return decide('skipped',
|
|
1894
|
+
'worktree rootを共有する構成では書き手を特定できない(帰属はrootだけで決まる)');
|
|
1895
|
+
}
|
|
1896
|
+
const recorded = await durablyRecordProbeCheckpoints(escalation, probed.checkpoints);
|
|
1897
|
+
if (!recorded.ok) return decide(recorded.outcome, recorded.detail);
|
|
1898
|
+
|
|
1899
|
+
const submit = async (operation, { artifact = null, artifactDigest = null,
|
|
1900
|
+
checkpointDigest = null } = {}) => {
|
|
1901
|
+
const controlRequest = createRuntimeControlRequest({
|
|
1902
|
+
requestId: `${escalation.escalation_id}.${operation.replace(/_/gu, '-')}`,
|
|
1903
|
+
runId: request.request_id, operation, sessionNonce,
|
|
1904
|
+
payload: controlOperationPayload({ operation, runRef: request.request_id,
|
|
1905
|
+
artifact, artifactDigest, checkpointDigest,
|
|
1906
|
+
expectedEpoch: recorded.expected_epoch, expectedQueueDigest: null }),
|
|
1907
|
+
});
|
|
1908
|
+
return handler(controlRequest);
|
|
1909
|
+
};
|
|
1910
|
+
const unmetOf = (response) => (response?.result?.unmet ?? []).join('/')
|
|
1911
|
+
|| String(response?.outcome ?? 'unknown');
|
|
1912
|
+
|
|
1913
|
+
const findingResponse = await submit('finding_record', {
|
|
1914
|
+
artifact: escalation.candidate, artifactDigest: escalation.candidate.candidate_digest,
|
|
1915
|
+
checkpointDigest: escalation.checkpoint_digest,
|
|
1916
|
+
}).catch((error) => ({ outcome: 'rejected',
|
|
1917
|
+
result: { unmet: [String(error?.code ?? error?.message ?? error)] } }));
|
|
1918
|
+
const findingDigest = findingResponse?.result?.finding_digest ?? null;
|
|
1919
|
+
if (findingResponse?.outcome !== 'completed' || !/^[0-9a-f]{64}$/u.test(findingDigest ?? '')) {
|
|
1920
|
+
return decide('rejected', `finding_recordが通らない: ${unmetOf(findingResponse)}`);
|
|
1921
|
+
}
|
|
1922
|
+
|
|
1923
|
+
const conflictResponse = await submit('conflict', { artifactDigest: findingDigest })
|
|
1924
|
+
.catch((error) => ({ outcome: 'rejected',
|
|
1925
|
+
result: { unmet: [String(error?.code ?? error?.message ?? error)] } }));
|
|
1926
|
+
if (conflictResponse?.outcome !== 'completed') {
|
|
1927
|
+
return decide('rejected', `conflictが通らない: ${unmetOf(conflictResponse)}`, findingDigest);
|
|
1928
|
+
}
|
|
1929
|
+
|
|
1930
|
+
const holdResponse = await submit('hold').catch((error) => ({ outcome: 'rejected',
|
|
1931
|
+
result: { unmet: [String(error?.code ?? error?.message ?? error)] } }));
|
|
1932
|
+
if (holdResponse?.outcome !== 'completed') {
|
|
1933
|
+
return decide('rejected', `holdが通らない: ${unmetOf(holdResponse)}`, findingDigest);
|
|
1934
|
+
}
|
|
1935
|
+
return decide('held', `早期警報からhold: ${warning.kind}/${warning.path}`, findingDigest);
|
|
1936
|
+
};
|
|
1937
|
+
|
|
1938
|
+
/**
|
|
1939
|
+
* 警報1件の全行程(probe → 記録 → escalation)。
|
|
1940
|
+
*
|
|
1941
|
+
* probeが`observed`でも、escalationは走らないことがある(既にfreeze済み、当該TODOが
|
|
1942
|
+
* 走り終わっている)。その差を`probe_outcome`で区別する——`escalated`はhold経路へ
|
|
1943
|
+
* 渡したことまでを言い、渡さなかったなら`observed`のまま残す。
|
|
1944
|
+
*/
|
|
1945
|
+
const handleIoWarning = async (warning, packets) => {
|
|
1946
|
+
const probed = await probeWarning(warning).catch(() => (
|
|
1947
|
+
{ outcome: 'unprobed', writers: [], checkpoints: {} }));
|
|
1948
|
+
const decided = probed.outcome === 'observed'
|
|
1949
|
+
? await escalateIoWarning(warning, probed, packets).catch(() => null)
|
|
1950
|
+
: null;
|
|
1951
|
+
await recordIoWarning(warning,
|
|
1952
|
+
decided === null || decided.outcome === 'skipped' ? probed.outcome : 'escalated');
|
|
1953
|
+
};
|
|
1954
|
+
|
|
1955
|
+
/** 警報の処理を直列化する鎖。`onWarning`はここへ繋ぐだけにする。 */
|
|
1956
|
+
let escalationChain = Promise.resolve();
|
|
1957
|
+
|
|
1697
1958
|
const resolveObservationBinding = async ({ binding }) => {
|
|
1698
1959
|
const events = await readBoundedJson(path.join(runDir, 'events.json'), 'run events');
|
|
1699
1960
|
const dispatch = events.findLast((event) => event.kind === 'executor_dispatched'
|
|
@@ -1805,6 +2066,20 @@ export async function runManagedSupervisorDaemon({
|
|
|
1805
2066
|
await extra.registerWithManagedSupervisor(managedSupervisor);
|
|
1806
2067
|
}
|
|
1807
2068
|
if (!restarting && isDistributedScriptedControllerActivation(activation)) {
|
|
2069
|
+
sentinel = createRunSentinel({
|
|
2070
|
+
packets: committed.bundle.executor_packets,
|
|
2071
|
+
onWarning: (warning) => {
|
|
2072
|
+
// escalationはrun全体の状態(events.json・lifecycle lock・freeze)を触るので、
|
|
2073
|
+
// 警報が同時に何本飛んでも1本ずつ処理する。並べると、同じcheckpointを二重に
|
|
2074
|
+
// 積んだり、holdが掛かった後の警報でもう一度holdを試したりする。
|
|
2075
|
+
const settled = escalationChain.then(() => handleIoWarning(warning,
|
|
2076
|
+
committed.bundle.executor_packets));
|
|
2077
|
+
// 鎖自体は必ずresolvedへ戻す(1件の失敗で以後の警報を止めない)。
|
|
2078
|
+
// 失敗そのものは呼び出し側へ返す方を残し、握り潰さない。
|
|
2079
|
+
escalationChain = settled.catch(() => {});
|
|
2080
|
+
return settled;
|
|
2081
|
+
},
|
|
2082
|
+
});
|
|
1808
2083
|
await driveInitialScriptedManagedEpoch({
|
|
1809
2084
|
runDir,
|
|
1810
2085
|
repoRoot,
|
|
@@ -1812,6 +2087,7 @@ export async function runManagedSupervisorDaemon({
|
|
|
1812
2087
|
committed,
|
|
1813
2088
|
activation,
|
|
1814
2089
|
managedSupervisor,
|
|
2090
|
+
sentinel,
|
|
1815
2091
|
initialEvents: events,
|
|
1816
2092
|
controlEvents: () => controlEvents,
|
|
1817
2093
|
});
|
|
@@ -2577,6 +2853,11 @@ export async function runManagedSupervisorDaemon({
|
|
|
2577
2853
|
recordedAt: canonicalNow() })];
|
|
2578
2854
|
}
|
|
2579
2855
|
await replaceEventsAtomically(runDir, events);
|
|
2856
|
+
// abandonは成果を捨てる決定なので、workerの木も畳む。closeでは畳まない——
|
|
2857
|
+
// 木そのものがrunの成果であり、着地させる前に消したら受理した内容が残らない。
|
|
2858
|
+
if (controlRequest.operation === 'abandon') {
|
|
2859
|
+
await removeScriptedWorktrees({ repoRoot, runDir }).catch(() => null);
|
|
2860
|
+
}
|
|
2580
2861
|
await appendControl({ run_id: request.request_id, kind: 'supervisor_stopped',
|
|
2581
2862
|
session_nonce_digest: digestArtifact(sessionNonce),
|
|
2582
2863
|
payload: { shutdown_result_digest: shutdown.result_digest } });
|
|
@@ -3095,6 +3376,9 @@ export async function runManagedSupervisorDaemon({
|
|
|
3095
3376
|
};
|
|
3096
3377
|
server = await serveRuntimeControlSocket({ socketPath, handler });
|
|
3097
3378
|
registerDaemonCleanup(async (signal) => {
|
|
3379
|
+
// 監視fdを残さない。取り残すとtest fixtureの後片付けが重くなる。
|
|
3380
|
+
sentinel?.close();
|
|
3381
|
+
sentinel = null;
|
|
3098
3382
|
if (activationCommitted && activation !== null) {
|
|
3099
3383
|
await appendControl({ run_id: request.request_id, kind: 'supervisor_stopped',
|
|
3100
3384
|
session_nonce_digest: digestArtifact(sessionNonce), payload: { signal } });
|
|
@@ -156,6 +156,36 @@ export function validateRuntimeControlEventPayload(kind, value) {
|
|
|
156
156
|
if (kind === 'supervisor_recovery_barrier') {
|
|
157
157
|
return exact(value, ['barrier_id']) && identifier(value.barrier_id);
|
|
158
158
|
}
|
|
159
|
+
// I/O sentinelの早期警報(ADR 0143)。**findingではない**——検知の正本はcheckpointのままで、
|
|
160
|
+
// これは「早くcheckpointを撮って確かめろ」という引き金の記録である。記録しない選択肢は無い:
|
|
161
|
+
// 機械が何かに気づいたのに黙っている状態を残さない(ADR 0130)。
|
|
162
|
+
if (kind === 'io_warning_observed') {
|
|
163
|
+
return exact(value, ['warning_kind', 'todo_ids', 'path', 'probe_outcome', 'warning_digest'])
|
|
164
|
+
&& ['io_overlap_warning', 'io_scope_warning'].includes(value.warning_kind)
|
|
165
|
+
&& Array.isArray(value.todo_ids) && value.todo_ids.length >= 1 && value.todo_ids.length <= 256
|
|
166
|
+
&& value.todo_ids.every(identifier)
|
|
167
|
+
&& value.todo_ids.every((id, index) => index === 0 || value.todo_ids[index - 1] < id)
|
|
168
|
+
&& typeof value.path === 'string' && value.path.length > 0 && value.path.length <= 4096
|
|
169
|
+
&& ['observed', 'transient', 'escalated', 'unprobed'].includes(value.probe_outcome)
|
|
170
|
+
&& value.warning_digest === canonicalDigest({
|
|
171
|
+
warning_kind: value.warning_kind, todo_ids: value.todo_ids, path: value.path,
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
// probeを通った警報を既存hold経路へ入れた顛末(ADR 0143の三段目)。
|
|
175
|
+
// 成否どちらも残す。自動escalationが黙って失敗する状態を作らない——失敗の理由は
|
|
176
|
+
// `detail`が持ち、これが無いと「警報は出たのにholdが掛かっていない」を後から説明できない。
|
|
177
|
+
if (kind === 'io_escalation_decided') {
|
|
178
|
+
return exact(value, ['warning_digest', 'anchor_todo_id', 'checkpoint_digest',
|
|
179
|
+
'finding_digest', 'outcome', 'detail'])
|
|
180
|
+
&& digest(value.warning_digest) && identifier(value.anchor_todo_id)
|
|
181
|
+
&& digest(value.checkpoint_digest)
|
|
182
|
+
&& (value.finding_digest === null || digest(value.finding_digest))
|
|
183
|
+
&& ['held', 'rejected', 'skipped'].includes(value.outcome)
|
|
184
|
+
// holdまで通ったなら、どのfindingで止めたかを必ず指す。途中で落ちた時は
|
|
185
|
+
// 記録済みfindingがあればそれを残す(無ければnull)——後追いの手掛かりを捨てない。
|
|
186
|
+
&& (value.outcome !== 'held' || digest(value.finding_digest))
|
|
187
|
+
&& typeof value.detail === 'string' && value.detail.length > 0 && value.detail.length <= 4096;
|
|
188
|
+
}
|
|
159
189
|
return false;
|
|
160
190
|
}
|
|
161
191
|
|
|
@@ -487,6 +487,9 @@ export function recomputeReceiptDecisions(options = {}) {
|
|
|
487
487
|
entry.todo_id === receipt.todo_id
|
|
488
488
|
&& entry.sequence > dispatch.sequence
|
|
489
489
|
&& entry.sequence < receipt.sequence
|
|
490
|
+
// supervisorがI/O警報を確かめるために撮ったcheckpointはexecutorの申告境界では
|
|
491
|
+
// ないので、bindingの基準にしない(ADR 0143。engine側と同一規則)。
|
|
492
|
+
&& entry.payload?.observed_by !== 'supervisor_probe'
|
|
490
493
|
));
|
|
491
494
|
if (observedCheckpoints.length > 0) {
|
|
492
495
|
const last = observedCheckpoints[observedCheckpoints.length - 1].payload;
|
|
@@ -238,7 +238,14 @@ export async function captureWorktreeDiff(options = {}) {
|
|
|
238
238
|
};
|
|
239
239
|
}
|
|
240
240
|
|
|
241
|
-
|
|
241
|
+
/**
|
|
242
|
+
* 宣言writeがobserved pathを覆うか。末尾`/`はprefixとして読む。
|
|
243
|
+
*
|
|
244
|
+
* I/O sentinelの早期警報も同じ述語を使う(ADR 0143)。警報とcheckpoint findingで
|
|
245
|
+
* 述語が分かれると、「警報は出たがcheckpointでは競合にならない」種類のずれが生まれ、
|
|
246
|
+
* どちらが正しいのか誰にも分からなくなる。
|
|
247
|
+
*/
|
|
248
|
+
export function coveredBy(declaredWrites, observedPath) {
|
|
242
249
|
return declaredWrites.some((declared) => (
|
|
243
250
|
declared === observedPath
|
|
244
251
|
|| (declared.endsWith('/') && observedPath.startsWith(declared))
|
package/src/runtime-engine.mjs
CHANGED
|
@@ -619,6 +619,11 @@ export function adjudicatePendingReceipts(options = {}) {
|
|
|
619
619
|
entry.todo_id === receipt.todo_id
|
|
620
620
|
&& entry.sequence > attemptStart
|
|
621
621
|
&& entry.sequence < receipt.sequence
|
|
622
|
+
// supervisorが自分の判断で撮ったcheckpoint(I/O警報のprobe)は、executorの
|
|
623
|
+
// 申告境界ではない。走行中の任意の一点なので、その後も書き続けたexecutorの
|
|
624
|
+
// receiptと一致しないのが正常である。ここへ混ぜると、正当なreceiptが
|
|
625
|
+
// checkpoint_mismatchで落ちる。証拠としては残り、findingの導出には使われる。
|
|
626
|
+
&& entry.payload?.observed_by !== 'supervisor_probe'
|
|
622
627
|
));
|
|
623
628
|
if (observedCheckpoints.length === 0) return false;
|
|
624
629
|
const last = observedCheckpoints[observedCheckpoints.length - 1].payload;
|