@rex0220/ksql-flownet 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.ja.md +81 -0
- package/README.md +117 -0
- package/dist/bundle/bundle-builder.d.ts +49 -0
- package/dist/bundle/bundle-builder.js +208 -0
- package/dist/bundle/index.d.ts +4 -0
- package/dist/bundle/index.js +4 -0
- package/dist/bundle/upload.d.ts +28 -0
- package/dist/bundle/upload.js +72 -0
- package/dist/bundle/zip-store.d.ts +17 -0
- package/dist/bundle/zip-store.js +175 -0
- package/dist/cli/archive-run-command.d.ts +12 -0
- package/dist/cli/archive-run-command.js +131 -0
- package/dist/cli/cancel-run-command.d.ts +9 -0
- package/dist/cli/cancel-run-command.js +77 -0
- package/dist/cli/force-unlock-network-command.d.ts +8 -0
- package/dist/cli/force-unlock-network-command.js +129 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.js +132 -0
- package/dist/cli/plan-command.d.ts +2 -0
- package/dist/cli/plan-command.js +100 -0
- package/dist/cli/poll-requests-command.d.ts +3 -0
- package/dist/cli/poll-requests-command.js +74 -0
- package/dist/cli/record-job-unlock-command.d.ts +11 -0
- package/dist/cli/record-job-unlock-command.js +78 -0
- package/dist/cli/resolve-node-command.d.ts +13 -0
- package/dist/cli/resolve-node-command.js +174 -0
- package/dist/cli/run-network-command.d.ts +14 -0
- package/dist/cli/run-network-command.js +418 -0
- package/dist/cli/status-command.d.ts +5 -0
- package/dist/cli/status-command.js +137 -0
- package/dist/cli/validate-command.d.ts +6 -0
- package/dist/cli/validate-command.js +17 -0
- package/dist/dag/descendants.d.ts +9 -0
- package/dist/dag/descendants.js +67 -0
- package/dist/dag/topological-sort.d.ts +7 -0
- package/dist/dag/topological-sort.js +32 -0
- package/dist/domain/business-key.d.ts +15 -0
- package/dist/domain/business-key.js +249 -0
- package/dist/domain/canonical-lock-key.d.ts +17 -0
- package/dist/domain/canonical-lock-key.js +60 -0
- package/dist/domain/canonical-record-key.d.ts +18 -0
- package/dist/domain/canonical-record-key.js +70 -0
- package/dist/domain/load-network.d.ts +4 -0
- package/dist/domain/load-network.js +47 -0
- package/dist/domain/network-definition.d.ts +47 -0
- package/dist/domain/network-definition.js +7 -0
- package/dist/domain/persistence-model.d.ts +198 -0
- package/dist/domain/persistence-model.js +2 -0
- package/dist/domain/run-aggregate.d.ts +3 -0
- package/dist/domain/run-aggregate.js +15 -0
- package/dist/domain/validate-network-path.d.ts +11 -0
- package/dist/domain/validate-network-path.js +33 -0
- package/dist/domain/validate-network.d.ts +3 -0
- package/dist/domain/validate-network.js +180 -0
- package/dist/executor/attempt-executor.d.ts +43 -0
- package/dist/executor/attempt-executor.js +236 -0
- package/dist/executor/job-log-reader.d.ts +37 -0
- package/dist/executor/job-log-reader.js +91 -0
- package/dist/executor/ksql-flow-cli.d.ts +86 -0
- package/dist/executor/ksql-flow-cli.js +161 -0
- package/dist/executor/preflight.d.ts +41 -0
- package/dist/executor/preflight.js +150 -0
- package/dist/executor/result-classifier.d.ts +69 -0
- package/dist/executor/result-classifier.js +237 -0
- package/dist/executor/run-subprocess.d.ts +64 -0
- package/dist/executor/run-subprocess.js +175 -0
- package/dist/io/input-baseline.d.ts +19 -0
- package/dist/io/input-baseline.js +214 -0
- package/dist/io/io-config.d.ts +6 -0
- package/dist/io/io-config.js +33 -0
- package/dist/io/io-path.d.ts +37 -0
- package/dist/io/io-path.js +279 -0
- package/dist/orchestration/archive-run.d.ts +53 -0
- package/dist/orchestration/archive-run.js +323 -0
- package/dist/orchestration/cancel-request.d.ts +18 -0
- package/dist/orchestration/cancel-request.js +67 -0
- package/dist/orchestration/ensure-run.d.ts +83 -0
- package/dist/orchestration/ensure-run.js +632 -0
- package/dist/orchestration/job-lock-audit.d.ts +24 -0
- package/dist/orchestration/job-lock-audit.js +123 -0
- package/dist/orchestration/orphan-attempt-adjudication.d.ts +39 -0
- package/dist/orchestration/orphan-attempt-adjudication.js +167 -0
- package/dist/orchestration/reconciliation.d.ts +41 -0
- package/dist/orchestration/reconciliation.js +526 -0
- package/dist/orchestration/resolve-node.d.ts +29 -0
- package/dist/orchestration/resolve-node.js +105 -0
- package/dist/orchestration/run-activity.d.ts +13 -0
- package/dist/orchestration/run-activity.js +14 -0
- package/dist/orchestration/sequential-scheduler.d.ts +79 -0
- package/dist/orchestration/sequential-scheduler.js +706 -0
- package/dist/orchestration/status.d.ts +91 -0
- package/dist/orchestration/status.js +171 -0
- package/dist/persistence/in-memory-repository.d.ts +41 -0
- package/dist/persistence/in-memory-repository.js +333 -0
- package/dist/persistence/kintone/client.d.ts +36 -0
- package/dist/persistence/kintone/client.js +93 -0
- package/dist/persistence/kintone/design-notes.d.ts +38 -0
- package/dist/persistence/kintone/design-notes.js +38 -0
- package/dist/persistence/kintone/repository.d.ts +44 -0
- package/dist/persistence/kintone/repository.js +836 -0
- package/dist/persistence/kintone/schema.d.ts +116 -0
- package/dist/persistence/kintone/schema.js +115 -0
- package/dist/persistence/network-lock-reader.d.ts +24 -0
- package/dist/persistence/network-lock-reader.js +43 -0
- package/dist/persistence/network-lock-recovery.d.ts +66 -0
- package/dist/persistence/network-lock-recovery.js +392 -0
- package/dist/persistence/network-lock.d.ts +97 -0
- package/dist/persistence/network-lock.js +408 -0
- package/dist/persistence/repository.d.ts +101 -0
- package/dist/persistence/repository.js +11 -0
- package/dist/persistence/state-transition.d.ts +4 -0
- package/dist/persistence/state-transition.js +25 -0
- package/dist/requests/flownet-child-client.d.ts +78 -0
- package/dist/requests/flownet-child-client.js +243 -0
- package/dist/requests/kintone-request-store.d.ts +44 -0
- package/dist/requests/kintone-request-store.js +202 -0
- package/dist/requests/poll-requests-config.d.ts +26 -0
- package/dist/requests/poll-requests-config.js +105 -0
- package/dist/requests/request-model.d.ts +57 -0
- package/dist/requests/request-model.js +305 -0
- package/dist/requests/request-poller.d.ts +33 -0
- package/dist/requests/request-poller.js +353 -0
- package/dist/requests/request-result.d.ts +18 -0
- package/dist/requests/request-result.js +215 -0
- package/dist/requests/start-request.d.ts +26 -0
- package/dist/requests/start-request.js +83 -0
- package/package.json +63 -0
- package/schemas/.gitkeep +1 -0
- package/schemas/network-definition.schema.json +121 -0
|
@@ -0,0 +1,526 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { computeRunAggregateStatus } from "../domain/run-aggregate.js";
|
|
3
|
+
const TERMINAL_ATTEMPTS = new Set([
|
|
4
|
+
"SUCCESS",
|
|
5
|
+
"FAILED",
|
|
6
|
+
"CANCELLED",
|
|
7
|
+
"UNKNOWN",
|
|
8
|
+
]);
|
|
9
|
+
const TERMINAL_STATES = new Set([
|
|
10
|
+
"SUCCESS",
|
|
11
|
+
"FAILED",
|
|
12
|
+
"BLOCKED",
|
|
13
|
+
"SKIPPED",
|
|
14
|
+
"CANCELLED",
|
|
15
|
+
"UNKNOWN",
|
|
16
|
+
]);
|
|
17
|
+
export class ReconciliationRequiredError extends Error {
|
|
18
|
+
code = "RECONCILIATION_REQUIRED";
|
|
19
|
+
result;
|
|
20
|
+
constructor(result) {
|
|
21
|
+
super(`RECONCILIATION_REQUIRED: ${JSON.stringify(result.inconsistencies)}`);
|
|
22
|
+
this.name = "ReconciliationRequiredError";
|
|
23
|
+
this.result = result;
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
function groupByNode(attempts) {
|
|
27
|
+
const result = new Map();
|
|
28
|
+
for (const attempt of attempts) {
|
|
29
|
+
const values = result.get(attempt.value.node_id) ?? [];
|
|
30
|
+
values.push(attempt);
|
|
31
|
+
result.set(attempt.value.node_id, values);
|
|
32
|
+
}
|
|
33
|
+
return result;
|
|
34
|
+
}
|
|
35
|
+
function resolutionGroups(resolutions) {
|
|
36
|
+
const result = new Map();
|
|
37
|
+
for (const resolution of resolutions) {
|
|
38
|
+
const values = result.get(resolution.value.attempt_id) ?? [];
|
|
39
|
+
values.push(resolution);
|
|
40
|
+
result.set(resolution.value.attempt_id, values);
|
|
41
|
+
}
|
|
42
|
+
return result;
|
|
43
|
+
}
|
|
44
|
+
/** Runs the D-09 inspections without exposing or invoking any repair method. */
|
|
45
|
+
export async function detectReconciliation(repository, runId) {
|
|
46
|
+
const run = await repository.getRun(runId);
|
|
47
|
+
const states = await repository.getNodeStates(runId);
|
|
48
|
+
const attempts = await repository.getAttempts(runId);
|
|
49
|
+
const resolutions = await repository.getResolutions(runId);
|
|
50
|
+
const attemptsByNode = groupByNode(attempts);
|
|
51
|
+
const stateByNode = new Map(states.map((state) => [state.value.node_id, state]));
|
|
52
|
+
const inconsistencies = [];
|
|
53
|
+
const unsafeNodes = new Set();
|
|
54
|
+
for (const [nodeId, nodeAttempts] of attemptsByNode) {
|
|
55
|
+
if (!stateByNode.has(nodeId)) {
|
|
56
|
+
inconsistencies.push({
|
|
57
|
+
code: "ATTEMPT_NODE_STATE_MISSING",
|
|
58
|
+
nodeId,
|
|
59
|
+
detail: "Node Attempts exist without a corresponding Node State",
|
|
60
|
+
attemptIds: nodeAttempts.map(({ value }) => value.node_attempt_id),
|
|
61
|
+
});
|
|
62
|
+
unsafeNodes.add(nodeId);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
for (const state of states) {
|
|
66
|
+
const nodeId = state.value.node_id;
|
|
67
|
+
const nodeAttempts = attemptsByNode.get(nodeId) ?? [];
|
|
68
|
+
const running = nodeAttempts.filter(({ value }) => value.status === "RUNNING");
|
|
69
|
+
if (running.length > 1) {
|
|
70
|
+
inconsistencies.push({
|
|
71
|
+
code: "MULTIPLE_RUNNING_ATTEMPTS",
|
|
72
|
+
nodeId,
|
|
73
|
+
detail: "one node has multiple RUNNING attempts",
|
|
74
|
+
attemptIds: running.map(({ value }) => value.node_attempt_id),
|
|
75
|
+
});
|
|
76
|
+
unsafeNodes.add(nodeId);
|
|
77
|
+
}
|
|
78
|
+
if (state.value.revision !== state.revision) {
|
|
79
|
+
inconsistencies.push({
|
|
80
|
+
code: "REVISION_SERIES_INVERTED",
|
|
81
|
+
nodeId,
|
|
82
|
+
detail: `business revision ${state.value.revision} differs from storage revision ${state.revision}`,
|
|
83
|
+
});
|
|
84
|
+
unsafeNodes.add(nodeId);
|
|
85
|
+
}
|
|
86
|
+
for (const attempt of nodeAttempts) {
|
|
87
|
+
if (attempt.value.state_revision_before === null) {
|
|
88
|
+
inconsistencies.push({
|
|
89
|
+
code: "REVISION_SERIES_UNDETERMINED",
|
|
90
|
+
nodeId,
|
|
91
|
+
detail: `Attempt ${attempt.value.node_attempt_id} has no state_revision_before`,
|
|
92
|
+
attemptIds: [attempt.value.node_attempt_id],
|
|
93
|
+
});
|
|
94
|
+
unsafeNodes.add(nodeId);
|
|
95
|
+
}
|
|
96
|
+
else if (attempt.value.state_revision_before >= state.value.revision) {
|
|
97
|
+
inconsistencies.push({
|
|
98
|
+
code: attempt.value.state_revision_before > state.value.revision
|
|
99
|
+
? "REVISION_SERIES_INVERTED"
|
|
100
|
+
: "REVISION_SERIES_UNDETERMINED",
|
|
101
|
+
nodeId,
|
|
102
|
+
detail: `Attempt ${attempt.value.node_attempt_id} state_revision_before ${attempt.value.state_revision_before} is not older than State revision ${state.value.revision}`,
|
|
103
|
+
attemptIds: [attempt.value.node_attempt_id],
|
|
104
|
+
});
|
|
105
|
+
unsafeNodes.add(nodeId);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
if (state.value.status === "RUNNING" &&
|
|
109
|
+
state.value.active_attempt_id === null) {
|
|
110
|
+
inconsistencies.push({
|
|
111
|
+
code: "ACTIVE_ATTEMPT_ID_MISSING",
|
|
112
|
+
nodeId,
|
|
113
|
+
detail: "RUNNING Node State has no active_attempt_id",
|
|
114
|
+
});
|
|
115
|
+
unsafeNodes.add(nodeId);
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
if (state.value.active_attempt_id === null)
|
|
119
|
+
continue;
|
|
120
|
+
const active = nodeAttempts.filter(({ value }) => value.node_attempt_id === state.value.active_attempt_id);
|
|
121
|
+
if (active.length !== 1) {
|
|
122
|
+
inconsistencies.push({
|
|
123
|
+
code: "ACTIVE_ATTEMPT_NOT_FOUND",
|
|
124
|
+
nodeId,
|
|
125
|
+
detail: `active attempt cardinality is ${active.length}`,
|
|
126
|
+
attemptIds: [state.value.active_attempt_id],
|
|
127
|
+
});
|
|
128
|
+
unsafeNodes.add(nodeId);
|
|
129
|
+
continue;
|
|
130
|
+
}
|
|
131
|
+
const attempt = active[0];
|
|
132
|
+
if (unsafeNodes.has(nodeId))
|
|
133
|
+
continue;
|
|
134
|
+
if (TERMINAL_ATTEMPTS.has(attempt.value.status) &&
|
|
135
|
+
(attempt.value.status === state.value.status ||
|
|
136
|
+
state.value.status === "RUNNING")) {
|
|
137
|
+
inconsistencies.push({
|
|
138
|
+
code: "TERMINAL_ATTEMPT_PENDING_APPLY",
|
|
139
|
+
nodeId,
|
|
140
|
+
detail: "terminal active Attempt has not been applied to Node State",
|
|
141
|
+
attemptIds: [attempt.value.node_attempt_id],
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
else if (TERMINAL_STATES.has(state.value.status) &&
|
|
145
|
+
attempt.value.status === "RUNNING") {
|
|
146
|
+
inconsistencies.push({
|
|
147
|
+
code: "STATE_TERMINAL_ATTEMPT_RUNNING",
|
|
148
|
+
nodeId,
|
|
149
|
+
detail: "Node State is terminal while its active Attempt is RUNNING",
|
|
150
|
+
attemptIds: [attempt.value.node_attempt_id],
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
else if (attempt.value.status !== state.value.status) {
|
|
154
|
+
inconsistencies.push({
|
|
155
|
+
code: "STATE_ATTEMPT_STATUS_MISMATCH",
|
|
156
|
+
nodeId,
|
|
157
|
+
detail: `State ${state.value.status} and Attempt ${attempt.value.status} do not match`,
|
|
158
|
+
attemptIds: [attempt.value.node_attempt_id],
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
const attemptById = new Map(attempts.map((attempt) => [attempt.value.node_attempt_id, attempt]));
|
|
163
|
+
for (const [attemptId, grouped] of resolutionGroups(resolutions)) {
|
|
164
|
+
const attempt = attemptById.get(attemptId);
|
|
165
|
+
if (!attempt)
|
|
166
|
+
continue;
|
|
167
|
+
const outcomes = new Set(grouped.map(({ value }) => value.resolved_outcome));
|
|
168
|
+
const nodeId = attempt.value.node_id;
|
|
169
|
+
if (outcomes.size > 1) {
|
|
170
|
+
inconsistencies.push({
|
|
171
|
+
code: "CONFLICTING_ATTEMPT_RESOLUTIONS",
|
|
172
|
+
nodeId,
|
|
173
|
+
detail: `Attempt ${attemptId} has conflicting Resolution outcomes`,
|
|
174
|
+
attemptIds: [attemptId],
|
|
175
|
+
});
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
if (attempt.value.status !== "UNKNOWN" &&
|
|
179
|
+
attempt.value.status !== "FAILED") {
|
|
180
|
+
inconsistencies.push({
|
|
181
|
+
code: "RESOLUTION_ATTEMPT_NOT_UNKNOWN",
|
|
182
|
+
nodeId,
|
|
183
|
+
detail: `Resolution targets ${attempt.value.status} Attempt ${attemptId}`,
|
|
184
|
+
attemptIds: [attemptId],
|
|
185
|
+
});
|
|
186
|
+
continue;
|
|
187
|
+
}
|
|
188
|
+
const state = stateByNode.get(nodeId);
|
|
189
|
+
if (!state)
|
|
190
|
+
continue;
|
|
191
|
+
if (attempt.value.attempt_no !== state.value.latest_attempt_no ||
|
|
192
|
+
unsafeNodes.has(nodeId)) {
|
|
193
|
+
inconsistencies.push({
|
|
194
|
+
code: "RESOLUTION_NOT_CURRENT_ATTEMPT",
|
|
195
|
+
nodeId,
|
|
196
|
+
detail: `Resolution attempt ${attemptId} is not the uniquely current State attempt`,
|
|
197
|
+
attemptIds: [attemptId],
|
|
198
|
+
});
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
201
|
+
const outcome = grouped[0].value.resolved_outcome;
|
|
202
|
+
if (state.value.status === outcome)
|
|
203
|
+
continue;
|
|
204
|
+
const resolvable = state.value.status === "UNKNOWN" ||
|
|
205
|
+
(state.value.status === "FAILED" &&
|
|
206
|
+
attempt.value.status === "FAILED" &&
|
|
207
|
+
!state.value.idempotent);
|
|
208
|
+
inconsistencies.push({
|
|
209
|
+
code: resolvable
|
|
210
|
+
? "ATTEMPT_RESOLUTION_PENDING_APPLY"
|
|
211
|
+
: "RESOLUTION_STATE_NOT_UNKNOWN",
|
|
212
|
+
nodeId,
|
|
213
|
+
detail: resolvable
|
|
214
|
+
? `Resolution for Attempt ${attemptId} has not been applied to Node State`
|
|
215
|
+
: `Resolution cannot be applied to State ${state.value.status}`,
|
|
216
|
+
attemptIds: [attemptId],
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
const aggregate = computeRunAggregateStatus(states.map(({ value }) => value.status), run.value.started_at);
|
|
220
|
+
if (aggregate !== run.value.status) {
|
|
221
|
+
inconsistencies.push({
|
|
222
|
+
code: "RUN_AGGREGATE_MISMATCH",
|
|
223
|
+
detail: `Run status ${run.value.status} differs from computed ${aggregate}`,
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
return { inconsistencies };
|
|
227
|
+
}
|
|
228
|
+
async function recordRepair(repository, runId, repaired, repair, occurredAt) {
|
|
229
|
+
const auditEventId = randomUUID();
|
|
230
|
+
await repository.appendOperationAudit({
|
|
231
|
+
event_id: auditEventId,
|
|
232
|
+
event_type: "RECONCILIATION_REPAIR",
|
|
233
|
+
repair_type: repair.type,
|
|
234
|
+
run_id: runId,
|
|
235
|
+
target_type: repair.targetType,
|
|
236
|
+
target_id: repair.targetId,
|
|
237
|
+
before: repair.before,
|
|
238
|
+
after: repair.after,
|
|
239
|
+
basis: repair.basis,
|
|
240
|
+
occurred_at: occurredAt,
|
|
241
|
+
});
|
|
242
|
+
repaired.push({ ...repair, auditEventId });
|
|
243
|
+
}
|
|
244
|
+
async function repairState(repository, current, status, finishedAt, reason, resolutionEvent) {
|
|
245
|
+
const now = new Date().toISOString();
|
|
246
|
+
return repository.upsertNodeState({
|
|
247
|
+
value: {
|
|
248
|
+
...current.value,
|
|
249
|
+
status,
|
|
250
|
+
active_attempt_id: null,
|
|
251
|
+
status_reason: reason,
|
|
252
|
+
finished_at: finishedAt,
|
|
253
|
+
updated_at: now,
|
|
254
|
+
},
|
|
255
|
+
expected_revision: current.revision,
|
|
256
|
+
...(resolutionEvent === undefined
|
|
257
|
+
? {}
|
|
258
|
+
: { resolution_event: resolutionEvent }),
|
|
259
|
+
});
|
|
260
|
+
}
|
|
261
|
+
/** D-09 pre-DAG gate. Throws RECONCILIATION_REQUIRED after retaining safe repairs. */
|
|
262
|
+
export async function reconcileRun(repository, runId) {
|
|
263
|
+
const repaired = [];
|
|
264
|
+
const inconsistencies = [];
|
|
265
|
+
const run = await repository.getRun(runId);
|
|
266
|
+
const initialStates = await repository.getNodeStates(runId);
|
|
267
|
+
const attempts = await repository.getAttempts(runId);
|
|
268
|
+
const resolutions = await repository.getResolutions(runId);
|
|
269
|
+
const attemptsByNode = groupByNode(attempts);
|
|
270
|
+
const unsafeNodes = new Set();
|
|
271
|
+
const states = new Map(initialStates.map((state) => [state.value.node_id, state]));
|
|
272
|
+
for (const [nodeId, nodeAttempts] of attemptsByNode) {
|
|
273
|
+
if (!states.has(nodeId)) {
|
|
274
|
+
inconsistencies.push({
|
|
275
|
+
code: "ATTEMPT_NODE_STATE_MISSING",
|
|
276
|
+
nodeId,
|
|
277
|
+
detail: "Node Attempts exist without a corresponding Node State",
|
|
278
|
+
attemptIds: nodeAttempts.map(({ value }) => value.node_attempt_id),
|
|
279
|
+
});
|
|
280
|
+
unsafeNodes.add(nodeId);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
for (const initialState of initialStates) {
|
|
284
|
+
let state = states.get(initialState.value.node_id);
|
|
285
|
+
const nodeAttempts = attemptsByNode.get(state.value.node_id) ?? [];
|
|
286
|
+
const running = nodeAttempts.filter(({ value }) => value.status === "RUNNING");
|
|
287
|
+
if (running.length > 1) {
|
|
288
|
+
inconsistencies.push({
|
|
289
|
+
code: "MULTIPLE_RUNNING_ATTEMPTS",
|
|
290
|
+
nodeId: state.value.node_id,
|
|
291
|
+
detail: "one node has multiple RUNNING attempts",
|
|
292
|
+
attemptIds: running.map(({ value }) => value.node_attempt_id),
|
|
293
|
+
});
|
|
294
|
+
unsafeNodes.add(state.value.node_id);
|
|
295
|
+
}
|
|
296
|
+
if (state.value.revision !== state.revision) {
|
|
297
|
+
inconsistencies.push({
|
|
298
|
+
code: "REVISION_SERIES_INVERTED",
|
|
299
|
+
nodeId: state.value.node_id,
|
|
300
|
+
detail: `business revision ${state.value.revision} differs from storage revision ${state.revision}`,
|
|
301
|
+
});
|
|
302
|
+
unsafeNodes.add(state.value.node_id);
|
|
303
|
+
}
|
|
304
|
+
for (const attempt of nodeAttempts) {
|
|
305
|
+
if (attempt.value.state_revision_before === null) {
|
|
306
|
+
inconsistencies.push({
|
|
307
|
+
code: "REVISION_SERIES_UNDETERMINED",
|
|
308
|
+
nodeId: state.value.node_id,
|
|
309
|
+
detail: `Attempt ${attempt.value.node_attempt_id} has no state_revision_before`,
|
|
310
|
+
attemptIds: [attempt.value.node_attempt_id],
|
|
311
|
+
});
|
|
312
|
+
unsafeNodes.add(state.value.node_id);
|
|
313
|
+
}
|
|
314
|
+
else if (attempt.value.state_revision_before > state.value.revision) {
|
|
315
|
+
inconsistencies.push({
|
|
316
|
+
code: "REVISION_SERIES_INVERTED",
|
|
317
|
+
nodeId: state.value.node_id,
|
|
318
|
+
detail: `Attempt ${attempt.value.node_attempt_id} state_revision_before ${attempt.value.state_revision_before} is not older than State revision ${state.value.revision}`,
|
|
319
|
+
attemptIds: [attempt.value.node_attempt_id],
|
|
320
|
+
});
|
|
321
|
+
unsafeNodes.add(state.value.node_id);
|
|
322
|
+
}
|
|
323
|
+
else if (attempt.value.state_revision_before === state.value.revision) {
|
|
324
|
+
inconsistencies.push({
|
|
325
|
+
code: "REVISION_SERIES_UNDETERMINED",
|
|
326
|
+
nodeId: state.value.node_id,
|
|
327
|
+
detail: `Attempt ${attempt.value.node_attempt_id} was created from State revision ${state.value.revision}, but no later State revision is visible`,
|
|
328
|
+
attemptIds: [attempt.value.node_attempt_id],
|
|
329
|
+
});
|
|
330
|
+
unsafeNodes.add(state.value.node_id);
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
if (state.value.status === "RUNNING" &&
|
|
334
|
+
state.value.active_attempt_id === null) {
|
|
335
|
+
inconsistencies.push({
|
|
336
|
+
code: "ACTIVE_ATTEMPT_ID_MISSING",
|
|
337
|
+
nodeId: state.value.node_id,
|
|
338
|
+
detail: "RUNNING Node State has no active_attempt_id",
|
|
339
|
+
});
|
|
340
|
+
unsafeNodes.add(state.value.node_id);
|
|
341
|
+
continue;
|
|
342
|
+
}
|
|
343
|
+
if (state.value.active_attempt_id === null)
|
|
344
|
+
continue;
|
|
345
|
+
const active = nodeAttempts.filter(({ value }) => value.node_attempt_id === state.value.active_attempt_id);
|
|
346
|
+
if (active.length !== 1) {
|
|
347
|
+
inconsistencies.push({
|
|
348
|
+
code: "ACTIVE_ATTEMPT_NOT_FOUND",
|
|
349
|
+
nodeId: state.value.node_id,
|
|
350
|
+
detail: `active attempt cardinality is ${active.length}`,
|
|
351
|
+
attemptIds: [state.value.active_attempt_id],
|
|
352
|
+
});
|
|
353
|
+
unsafeNodes.add(state.value.node_id);
|
|
354
|
+
continue;
|
|
355
|
+
}
|
|
356
|
+
const attempt = active[0];
|
|
357
|
+
if (unsafeNodes.has(state.value.node_id))
|
|
358
|
+
continue;
|
|
359
|
+
if (attempt.value.status === state.value.status) {
|
|
360
|
+
if (TERMINAL_ATTEMPTS.has(attempt.value.status)) {
|
|
361
|
+
const before = {
|
|
362
|
+
status: state.value.status,
|
|
363
|
+
active_attempt_id: state.value.active_attempt_id,
|
|
364
|
+
revision: state.value.revision,
|
|
365
|
+
};
|
|
366
|
+
state = await repairState(repository, state, state.value.status, attempt.value.finished_at, "reconciled terminal active attempt");
|
|
367
|
+
states.set(state.value.node_id, state);
|
|
368
|
+
await recordRepair(repository, runId, repaired, {
|
|
369
|
+
type: "TERMINAL_ATTEMPT_APPLIED",
|
|
370
|
+
targetType: "NODE_STATE",
|
|
371
|
+
targetId: state.value.node_state_id,
|
|
372
|
+
before,
|
|
373
|
+
after: {
|
|
374
|
+
status: state.value.status,
|
|
375
|
+
active_attempt_id: null,
|
|
376
|
+
revision: state.value.revision,
|
|
377
|
+
},
|
|
378
|
+
basis: `terminal attempt ${attempt.value.node_attempt_id} matched State status; cleared stale active_attempt_id`,
|
|
379
|
+
}, state.value.updated_at);
|
|
380
|
+
}
|
|
381
|
+
continue;
|
|
382
|
+
}
|
|
383
|
+
if (state.value.status === "RUNNING" &&
|
|
384
|
+
TERMINAL_ATTEMPTS.has(attempt.value.status)) {
|
|
385
|
+
const before = {
|
|
386
|
+
status: state.value.status,
|
|
387
|
+
active_attempt_id: state.value.active_attempt_id,
|
|
388
|
+
revision: state.value.revision,
|
|
389
|
+
};
|
|
390
|
+
state = await repairState(repository, state, attempt.value.status, attempt.value.finished_at, "reconciled from terminal active attempt");
|
|
391
|
+
states.set(state.value.node_id, state);
|
|
392
|
+
await recordRepair(repository, runId, repaired, {
|
|
393
|
+
type: "TERMINAL_ATTEMPT_APPLIED",
|
|
394
|
+
targetType: "NODE_STATE",
|
|
395
|
+
targetId: state.value.node_state_id,
|
|
396
|
+
before,
|
|
397
|
+
after: {
|
|
398
|
+
status: state.value.status,
|
|
399
|
+
active_attempt_id: null,
|
|
400
|
+
revision: state.value.revision,
|
|
401
|
+
},
|
|
402
|
+
basis: `terminal attempt ${attempt.value.node_attempt_id} is the unique active attempt`,
|
|
403
|
+
}, state.value.updated_at);
|
|
404
|
+
}
|
|
405
|
+
else if (TERMINAL_STATES.has(state.value.status) &&
|
|
406
|
+
attempt.value.status === "RUNNING") {
|
|
407
|
+
inconsistencies.push({
|
|
408
|
+
code: "STATE_TERMINAL_ATTEMPT_RUNNING",
|
|
409
|
+
nodeId: state.value.node_id,
|
|
410
|
+
detail: "Node State is terminal while its active Attempt is RUNNING",
|
|
411
|
+
attemptIds: [attempt.value.node_attempt_id],
|
|
412
|
+
});
|
|
413
|
+
}
|
|
414
|
+
else {
|
|
415
|
+
inconsistencies.push({
|
|
416
|
+
code: "STATE_ATTEMPT_STATUS_MISMATCH",
|
|
417
|
+
nodeId: state.value.node_id,
|
|
418
|
+
detail: `State ${state.value.status} and Attempt ${attempt.value.status} do not match`,
|
|
419
|
+
attemptIds: [attempt.value.node_attempt_id],
|
|
420
|
+
});
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
const attemptById = new Map(attempts.map((attempt) => [attempt.value.node_attempt_id, attempt]));
|
|
424
|
+
for (const [attemptId, grouped] of resolutionGroups(resolutions)) {
|
|
425
|
+
const attempt = attemptById.get(attemptId);
|
|
426
|
+
if (!attempt)
|
|
427
|
+
continue;
|
|
428
|
+
const outcomes = new Set(grouped.map(({ value }) => value.resolved_outcome));
|
|
429
|
+
if (outcomes.size > 1) {
|
|
430
|
+
inconsistencies.push({
|
|
431
|
+
code: "CONFLICTING_ATTEMPT_RESOLUTIONS",
|
|
432
|
+
nodeId: attempt.value.node_id,
|
|
433
|
+
detail: `Attempt ${attemptId} has conflicting Resolution outcomes`,
|
|
434
|
+
attemptIds: [attemptId],
|
|
435
|
+
});
|
|
436
|
+
continue;
|
|
437
|
+
}
|
|
438
|
+
if (attempt.value.status !== "UNKNOWN" &&
|
|
439
|
+
attempt.value.status !== "FAILED") {
|
|
440
|
+
inconsistencies.push({
|
|
441
|
+
code: "RESOLUTION_ATTEMPT_NOT_UNKNOWN",
|
|
442
|
+
nodeId: attempt.value.node_id,
|
|
443
|
+
detail: `Resolution targets ${attempt.value.status} Attempt ${attemptId}`,
|
|
444
|
+
attemptIds: [attemptId],
|
|
445
|
+
});
|
|
446
|
+
continue;
|
|
447
|
+
}
|
|
448
|
+
const state = states.get(attempt.value.node_id);
|
|
449
|
+
if (!state)
|
|
450
|
+
continue;
|
|
451
|
+
if (attempt.value.attempt_no !== state.value.latest_attempt_no ||
|
|
452
|
+
unsafeNodes.has(state.value.node_id)) {
|
|
453
|
+
inconsistencies.push({
|
|
454
|
+
code: "RESOLUTION_NOT_CURRENT_ATTEMPT",
|
|
455
|
+
nodeId: state.value.node_id,
|
|
456
|
+
detail: `Resolution attempt ${attemptId} is not the uniquely current State attempt`,
|
|
457
|
+
attemptIds: [attemptId],
|
|
458
|
+
});
|
|
459
|
+
continue;
|
|
460
|
+
}
|
|
461
|
+
const outcome = grouped[0].value.resolved_outcome;
|
|
462
|
+
if (state.value.status === outcome)
|
|
463
|
+
continue;
|
|
464
|
+
const resolvableState = state.value.status === "UNKNOWN" ||
|
|
465
|
+
(state.value.status === "FAILED" &&
|
|
466
|
+
attempt.value.status === "FAILED" &&
|
|
467
|
+
!state.value.idempotent);
|
|
468
|
+
if (!resolvableState) {
|
|
469
|
+
inconsistencies.push({
|
|
470
|
+
code: "RESOLUTION_STATE_NOT_UNKNOWN",
|
|
471
|
+
nodeId: state.value.node_id,
|
|
472
|
+
detail: `Resolution cannot be applied to State ${state.value.status}`,
|
|
473
|
+
attemptIds: [attemptId],
|
|
474
|
+
});
|
|
475
|
+
continue;
|
|
476
|
+
}
|
|
477
|
+
const resolution = grouped.reduce((latest, item) => item.value.resolved_at > latest.value.resolved_at ? item : latest);
|
|
478
|
+
const before = {
|
|
479
|
+
status: state.value.status,
|
|
480
|
+
revision: state.value.revision,
|
|
481
|
+
};
|
|
482
|
+
const updated = await repairState(repository, state, outcome, resolution.value.resolved_at, "reconciled from Attempt Resolution", {
|
|
483
|
+
event_type: "ATTEMPT_RESOLVED",
|
|
484
|
+
attempt_id: attemptId,
|
|
485
|
+
resolved_outcome: outcome,
|
|
486
|
+
resolved_at: resolution.value.resolved_at,
|
|
487
|
+
});
|
|
488
|
+
states.set(updated.value.node_id, updated);
|
|
489
|
+
await recordRepair(repository, runId, repaired, {
|
|
490
|
+
type: "ATTEMPT_RESOLUTION_APPLIED",
|
|
491
|
+
targetType: "NODE_STATE",
|
|
492
|
+
targetId: updated.value.node_state_id,
|
|
493
|
+
before,
|
|
494
|
+
after: {
|
|
495
|
+
status: updated.value.status,
|
|
496
|
+
revision: updated.value.revision,
|
|
497
|
+
},
|
|
498
|
+
basis: `Attempt Resolution for ${attemptId} uniquely resolved outcome to ${outcome}`,
|
|
499
|
+
}, updated.value.updated_at);
|
|
500
|
+
}
|
|
501
|
+
const aggregate = computeRunAggregateStatus([...states.values()].map(({ value }) => value.status), run.value.started_at);
|
|
502
|
+
let aggregateUpdated = false;
|
|
503
|
+
if (aggregate !== run.value.status) {
|
|
504
|
+
const now = new Date().toISOString();
|
|
505
|
+
const updated = await repository.updateRunAggregate(runId, run.revision, {
|
|
506
|
+
status: aggregate,
|
|
507
|
+
started_at: run.value.started_at,
|
|
508
|
+
finished_at: run.value.finished_at,
|
|
509
|
+
updated_at: now,
|
|
510
|
+
});
|
|
511
|
+
await recordRepair(repository, runId, repaired, {
|
|
512
|
+
type: "RUN_AGGREGATE_RECOMPUTED",
|
|
513
|
+
targetType: "NETWORK_RUN",
|
|
514
|
+
targetId: runId,
|
|
515
|
+
before: { status: run.value.status, revision: run.revision },
|
|
516
|
+
after: { status: updated.value.status, revision: updated.revision },
|
|
517
|
+
basis: "Network Run status recomputed from the complete Node State set using specification section 10",
|
|
518
|
+
}, now);
|
|
519
|
+
aggregateUpdated = true;
|
|
520
|
+
}
|
|
521
|
+
const result = { repaired, inconsistencies, aggregateUpdated };
|
|
522
|
+
if (inconsistencies.length > 0)
|
|
523
|
+
throw new ReconciliationRequiredError(result);
|
|
524
|
+
return result;
|
|
525
|
+
}
|
|
526
|
+
//# sourceMappingURL=reconciliation.js.map
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { AttemptResolution, NodeState } from "../domain/persistence-model.js";
|
|
2
|
+
import type { PersistenceRepository, Versioned } from "../persistence/repository.js";
|
|
3
|
+
export type ResolutionOutcome = "SUCCESS" | "FAILED" | "CANCELLED";
|
|
4
|
+
export type ResolutionType = AttemptResolution["resolution_type"];
|
|
5
|
+
export interface ResolveNodeInput {
|
|
6
|
+
repository: PersistenceRepository;
|
|
7
|
+
runId: string;
|
|
8
|
+
nodeId: string;
|
|
9
|
+
outcome: ResolutionOutcome;
|
|
10
|
+
resolutionType: ResolutionType;
|
|
11
|
+
reason: string;
|
|
12
|
+
evidenceRef: string;
|
|
13
|
+
servicePrincipal: string;
|
|
14
|
+
requestedBy: string;
|
|
15
|
+
approvedBy: string;
|
|
16
|
+
stopConfirmedBy: string;
|
|
17
|
+
stopEvidenceRef: string;
|
|
18
|
+
now?: () => Date;
|
|
19
|
+
}
|
|
20
|
+
export interface ResolveNodeResult {
|
|
21
|
+
resolution: Versioned<AttemptResolution>;
|
|
22
|
+
nodeState: Versioned<NodeState>;
|
|
23
|
+
}
|
|
24
|
+
export declare class ResolveNodeError extends Error {
|
|
25
|
+
readonly code: string;
|
|
26
|
+
constructor(code: string, message: string);
|
|
27
|
+
}
|
|
28
|
+
export declare function resolveNode(input: ResolveNodeInput): Promise<ResolveNodeResult>;
|
|
29
|
+
//# sourceMappingURL=resolve-node.d.ts.map
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
export class ResolveNodeError extends Error {
|
|
2
|
+
code;
|
|
3
|
+
constructor(code, message) {
|
|
4
|
+
super(message);
|
|
5
|
+
this.name = "ResolveNodeError";
|
|
6
|
+
this.code = code;
|
|
7
|
+
}
|
|
8
|
+
}
|
|
9
|
+
export async function resolveNode(input) {
|
|
10
|
+
validateRequiredAuditInput(input);
|
|
11
|
+
if (input.resolutionType === "NODE_COMPENSATION_COMPLETED" &&
|
|
12
|
+
input.outcome === "SUCCESS") {
|
|
13
|
+
throw new ResolveNodeError("COMPENSATION_CANNOT_SUCCEED", "NODE_COMPENSATION_COMPLETED cannot resolve a node to SUCCESS");
|
|
14
|
+
}
|
|
15
|
+
if (input.outcome === "SUCCESS" &&
|
|
16
|
+
input.resolutionType !== "NODE_MANUAL_COMPLETION_CONFIRMED") {
|
|
17
|
+
throw new ResolveNodeError("MANUAL_COMPLETION_REQUIRED", "SUCCESS requires NODE_MANUAL_COMPLETION_CONFIRMED");
|
|
18
|
+
}
|
|
19
|
+
if (input.resolutionType === "NODE_MANUAL_COMPLETION_CONFIRMED" &&
|
|
20
|
+
input.outcome !== "SUCCESS") {
|
|
21
|
+
throw new ResolveNodeError("MANUAL_COMPLETION_REQUIRES_SUCCESS", "NODE_MANUAL_COMPLETION_CONFIRMED can only resolve to SUCCESS");
|
|
22
|
+
}
|
|
23
|
+
await input.repository.getRun(input.runId);
|
|
24
|
+
const states = (await input.repository.getNodeStates(input.runId)).filter(({ value }) => value.node_id === input.nodeId);
|
|
25
|
+
if (states.length !== 1) {
|
|
26
|
+
throw new ResolveNodeError("NODE_STATE_NOT_UNIQUE", `expected exactly one Node State for ${input.runId}/${input.nodeId}, found ${states.length}`);
|
|
27
|
+
}
|
|
28
|
+
const state = states[0];
|
|
29
|
+
const eligible = state.value.status === "UNKNOWN" ||
|
|
30
|
+
(state.value.status === "FAILED" && !state.value.idempotent);
|
|
31
|
+
if (!eligible) {
|
|
32
|
+
throw new ResolveNodeError("NODE_NOT_RESOLVABLE", `Node State ${state.value.status} is not UNKNOWN or non-idempotent FAILED`);
|
|
33
|
+
}
|
|
34
|
+
const approvedBy = input.approvedBy.trim();
|
|
35
|
+
const requestedBy = input.requestedBy.trim();
|
|
36
|
+
const servicePrincipal = input.servicePrincipal.trim();
|
|
37
|
+
if (!state.value.idempotent &&
|
|
38
|
+
input.outcome === "SUCCESS" &&
|
|
39
|
+
(approvedBy === "" ||
|
|
40
|
+
approvedBy === requestedBy ||
|
|
41
|
+
approvedBy === servicePrincipal)) {
|
|
42
|
+
throw new ResolveNodeError("DISTINCT_APPROVER_REQUIRED", "non-idempotent SUCCESS requires an approved_by distinct from requested_by and service_principal");
|
|
43
|
+
}
|
|
44
|
+
const currentAttempts = (await input.repository.getAttempts(input.runId)).filter(({ value }) => value.node_id === input.nodeId &&
|
|
45
|
+
value.attempt_no === state.value.latest_attempt_no);
|
|
46
|
+
if (currentAttempts.length !== 1) {
|
|
47
|
+
throw new ResolveNodeError("CURRENT_ATTEMPT_NOT_UNIQUE", `expected exactly one latest Attempt for ${input.runId}/${input.nodeId}, found ${currentAttempts.length}`);
|
|
48
|
+
}
|
|
49
|
+
const attempt = currentAttempts[0];
|
|
50
|
+
if (attempt.value.status !== state.value.status) {
|
|
51
|
+
throw new ResolveNodeError("STATE_ATTEMPT_MISMATCH", `Node State ${state.value.status} does not match latest Attempt ${attempt.value.status}`);
|
|
52
|
+
}
|
|
53
|
+
const resolvedAt = (input.now ?? (() => new Date()))().toISOString();
|
|
54
|
+
const resolution = await input.repository.appendResolution({
|
|
55
|
+
event_type: "ATTEMPT_RESOLVED",
|
|
56
|
+
resolution_type: input.resolutionType,
|
|
57
|
+
attempt_id: attempt.value.node_attempt_id,
|
|
58
|
+
resolved_outcome: input.outcome,
|
|
59
|
+
reason: input.reason.trim(),
|
|
60
|
+
evidence_ref: input.evidenceRef.trim(),
|
|
61
|
+
service_principal: servicePrincipal,
|
|
62
|
+
requested_by: requestedBy,
|
|
63
|
+
approved_by: approvedBy,
|
|
64
|
+
stop_confirmed_by: input.stopConfirmedBy.trim(),
|
|
65
|
+
stop_evidence_ref: input.stopEvidenceRef.trim(),
|
|
66
|
+
resolved_at: resolvedAt,
|
|
67
|
+
});
|
|
68
|
+
const nodeState = await input.repository.upsertNodeState({
|
|
69
|
+
expected_revision: state.revision,
|
|
70
|
+
resolution_event: {
|
|
71
|
+
event_type: "ATTEMPT_RESOLVED",
|
|
72
|
+
attempt_id: attempt.value.node_attempt_id,
|
|
73
|
+
resolved_outcome: input.outcome,
|
|
74
|
+
resolved_at: resolvedAt,
|
|
75
|
+
},
|
|
76
|
+
value: {
|
|
77
|
+
...state.value,
|
|
78
|
+
status: input.outcome,
|
|
79
|
+
active_attempt_id: null,
|
|
80
|
+
status_reason: input.resolutionType,
|
|
81
|
+
finished_at: state.value.finished_at ?? resolvedAt,
|
|
82
|
+
updated_at: resolvedAt,
|
|
83
|
+
},
|
|
84
|
+
});
|
|
85
|
+
return { resolution, nodeState };
|
|
86
|
+
}
|
|
87
|
+
function validateRequiredAuditInput(input) {
|
|
88
|
+
const required = [
|
|
89
|
+
["runId", input.runId],
|
|
90
|
+
["nodeId", input.nodeId],
|
|
91
|
+
["reason", input.reason],
|
|
92
|
+
["evidenceRef", input.evidenceRef],
|
|
93
|
+
["servicePrincipal", input.servicePrincipal],
|
|
94
|
+
["requestedBy", input.requestedBy],
|
|
95
|
+
["stopConfirmedBy", input.stopConfirmedBy],
|
|
96
|
+
["stopEvidenceRef", input.stopEvidenceRef],
|
|
97
|
+
];
|
|
98
|
+
const missing = required
|
|
99
|
+
.filter(([, value]) => value.trim() === "")
|
|
100
|
+
.map(([name]) => name);
|
|
101
|
+
if (missing.length > 0) {
|
|
102
|
+
throw new ResolveNodeError("AUDIT_INPUT_REQUIRED", `required audit input is empty: ${missing.join(", ")}`);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
//# sourceMappingURL=resolve-node.js.map
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { CancelRequestState, NetworkRun } from "../domain/persistence-model.js";
|
|
2
|
+
import type { NetworkLockStatus } from "../persistence/network-lock-reader.js";
|
|
3
|
+
export type RunActivity = "LIVE" | "IDLE" | "INTERRUPTED" | "STOPPED";
|
|
4
|
+
export interface ActivityInput {
|
|
5
|
+
readonly status: NetworkRun["status"];
|
|
6
|
+
readonly startedAt: string | null;
|
|
7
|
+
readonly invocationIds: readonly string[];
|
|
8
|
+
readonly lock: NetworkLockStatus | null;
|
|
9
|
+
readonly cancelState: CancelRequestState | null;
|
|
10
|
+
readonly nowMs: number;
|
|
11
|
+
}
|
|
12
|
+
export declare function deriveRunActivity(input: ActivityInput): RunActivity | null;
|
|
13
|
+
//# sourceMappingURL=run-activity.d.ts.map
|