@rex0220/ksql-flownet 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.ja.md +81 -0
- package/README.md +117 -0
- package/dist/bundle/bundle-builder.d.ts +49 -0
- package/dist/bundle/bundle-builder.js +208 -0
- package/dist/bundle/index.d.ts +4 -0
- package/dist/bundle/index.js +4 -0
- package/dist/bundle/upload.d.ts +28 -0
- package/dist/bundle/upload.js +72 -0
- package/dist/bundle/zip-store.d.ts +17 -0
- package/dist/bundle/zip-store.js +175 -0
- package/dist/cli/archive-run-command.d.ts +12 -0
- package/dist/cli/archive-run-command.js +131 -0
- package/dist/cli/cancel-run-command.d.ts +9 -0
- package/dist/cli/cancel-run-command.js +77 -0
- package/dist/cli/force-unlock-network-command.d.ts +8 -0
- package/dist/cli/force-unlock-network-command.js +129 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.js +132 -0
- package/dist/cli/plan-command.d.ts +2 -0
- package/dist/cli/plan-command.js +100 -0
- package/dist/cli/poll-requests-command.d.ts +3 -0
- package/dist/cli/poll-requests-command.js +74 -0
- package/dist/cli/record-job-unlock-command.d.ts +11 -0
- package/dist/cli/record-job-unlock-command.js +78 -0
- package/dist/cli/resolve-node-command.d.ts +13 -0
- package/dist/cli/resolve-node-command.js +174 -0
- package/dist/cli/run-network-command.d.ts +14 -0
- package/dist/cli/run-network-command.js +418 -0
- package/dist/cli/status-command.d.ts +5 -0
- package/dist/cli/status-command.js +137 -0
- package/dist/cli/validate-command.d.ts +6 -0
- package/dist/cli/validate-command.js +17 -0
- package/dist/dag/descendants.d.ts +9 -0
- package/dist/dag/descendants.js +67 -0
- package/dist/dag/topological-sort.d.ts +7 -0
- package/dist/dag/topological-sort.js +32 -0
- package/dist/domain/business-key.d.ts +15 -0
- package/dist/domain/business-key.js +249 -0
- package/dist/domain/canonical-lock-key.d.ts +17 -0
- package/dist/domain/canonical-lock-key.js +60 -0
- package/dist/domain/canonical-record-key.d.ts +18 -0
- package/dist/domain/canonical-record-key.js +70 -0
- package/dist/domain/load-network.d.ts +4 -0
- package/dist/domain/load-network.js +47 -0
- package/dist/domain/network-definition.d.ts +47 -0
- package/dist/domain/network-definition.js +7 -0
- package/dist/domain/persistence-model.d.ts +198 -0
- package/dist/domain/persistence-model.js +2 -0
- package/dist/domain/run-aggregate.d.ts +3 -0
- package/dist/domain/run-aggregate.js +15 -0
- package/dist/domain/validate-network-path.d.ts +11 -0
- package/dist/domain/validate-network-path.js +33 -0
- package/dist/domain/validate-network.d.ts +3 -0
- package/dist/domain/validate-network.js +180 -0
- package/dist/executor/attempt-executor.d.ts +43 -0
- package/dist/executor/attempt-executor.js +236 -0
- package/dist/executor/job-log-reader.d.ts +37 -0
- package/dist/executor/job-log-reader.js +91 -0
- package/dist/executor/ksql-flow-cli.d.ts +86 -0
- package/dist/executor/ksql-flow-cli.js +161 -0
- package/dist/executor/preflight.d.ts +41 -0
- package/dist/executor/preflight.js +150 -0
- package/dist/executor/result-classifier.d.ts +69 -0
- package/dist/executor/result-classifier.js +237 -0
- package/dist/executor/run-subprocess.d.ts +64 -0
- package/dist/executor/run-subprocess.js +175 -0
- package/dist/io/input-baseline.d.ts +19 -0
- package/dist/io/input-baseline.js +214 -0
- package/dist/io/io-config.d.ts +6 -0
- package/dist/io/io-config.js +33 -0
- package/dist/io/io-path.d.ts +37 -0
- package/dist/io/io-path.js +279 -0
- package/dist/orchestration/archive-run.d.ts +53 -0
- package/dist/orchestration/archive-run.js +323 -0
- package/dist/orchestration/cancel-request.d.ts +18 -0
- package/dist/orchestration/cancel-request.js +67 -0
- package/dist/orchestration/ensure-run.d.ts +83 -0
- package/dist/orchestration/ensure-run.js +632 -0
- package/dist/orchestration/job-lock-audit.d.ts +24 -0
- package/dist/orchestration/job-lock-audit.js +123 -0
- package/dist/orchestration/orphan-attempt-adjudication.d.ts +39 -0
- package/dist/orchestration/orphan-attempt-adjudication.js +167 -0
- package/dist/orchestration/reconciliation.d.ts +41 -0
- package/dist/orchestration/reconciliation.js +526 -0
- package/dist/orchestration/resolve-node.d.ts +29 -0
- package/dist/orchestration/resolve-node.js +105 -0
- package/dist/orchestration/run-activity.d.ts +13 -0
- package/dist/orchestration/run-activity.js +14 -0
- package/dist/orchestration/sequential-scheduler.d.ts +79 -0
- package/dist/orchestration/sequential-scheduler.js +706 -0
- package/dist/orchestration/status.d.ts +91 -0
- package/dist/orchestration/status.js +171 -0
- package/dist/persistence/in-memory-repository.d.ts +41 -0
- package/dist/persistence/in-memory-repository.js +333 -0
- package/dist/persistence/kintone/client.d.ts +36 -0
- package/dist/persistence/kintone/client.js +93 -0
- package/dist/persistence/kintone/design-notes.d.ts +38 -0
- package/dist/persistence/kintone/design-notes.js +38 -0
- package/dist/persistence/kintone/repository.d.ts +44 -0
- package/dist/persistence/kintone/repository.js +836 -0
- package/dist/persistence/kintone/schema.d.ts +116 -0
- package/dist/persistence/kintone/schema.js +115 -0
- package/dist/persistence/network-lock-reader.d.ts +24 -0
- package/dist/persistence/network-lock-reader.js +43 -0
- package/dist/persistence/network-lock-recovery.d.ts +66 -0
- package/dist/persistence/network-lock-recovery.js +392 -0
- package/dist/persistence/network-lock.d.ts +97 -0
- package/dist/persistence/network-lock.js +408 -0
- package/dist/persistence/repository.d.ts +101 -0
- package/dist/persistence/repository.js +11 -0
- package/dist/persistence/state-transition.d.ts +4 -0
- package/dist/persistence/state-transition.js +25 -0
- package/dist/requests/flownet-child-client.d.ts +78 -0
- package/dist/requests/flownet-child-client.js +243 -0
- package/dist/requests/kintone-request-store.d.ts +44 -0
- package/dist/requests/kintone-request-store.js +202 -0
- package/dist/requests/poll-requests-config.d.ts +26 -0
- package/dist/requests/poll-requests-config.js +105 -0
- package/dist/requests/request-model.d.ts +57 -0
- package/dist/requests/request-model.js +305 -0
- package/dist/requests/request-poller.d.ts +33 -0
- package/dist/requests/request-poller.js +353 -0
- package/dist/requests/request-result.d.ts +18 -0
- package/dist/requests/request-result.js +215 -0
- package/dist/requests/start-request.d.ts +26 -0
- package/dist/requests/start-request.js +83 -0
- package/package.json +63 -0
- package/schemas/.gitkeep +1 -0
- package/schemas/network-definition.schema.json +121 -0
|
@@ -0,0 +1,408 @@
|
|
|
1
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
2
|
+
import { MAX_KINTONE_UNIQUE_KEY_LENGTH, networkLockKey, } from "../domain/canonical-lock-key.js";
|
|
3
|
+
import { KintoneApiError, KintoneClient, KintoneTransportError, } from "./kintone/client.js";
|
|
4
|
+
export class NetworkLockError extends Error {
|
|
5
|
+
code;
|
|
6
|
+
causeDetail;
|
|
7
|
+
constructor(code, message, causeDetail) {
|
|
8
|
+
super(message);
|
|
9
|
+
this.name = "NetworkLockError";
|
|
10
|
+
this.code = code;
|
|
11
|
+
this.causeDetail = causeDetail;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
const field = (value) => ({ value });
|
|
15
|
+
const text = (record, code) => String(record[code]?.value ?? "");
|
|
16
|
+
const revisionOf = (record) => Number(record.$revision?.value);
|
|
17
|
+
const idOf = (record) => text(record, "$id");
|
|
18
|
+
function quote(value) {
|
|
19
|
+
return `"${value.replaceAll("\\", "\\\\").replaceAll('"', '\\"')}"`;
|
|
20
|
+
}
|
|
21
|
+
function recordKeyFor(lockKey) {
|
|
22
|
+
const value = `LOCK:${lockKey}`;
|
|
23
|
+
if (value.length > MAX_KINTONE_UNIQUE_KEY_LENGTH) {
|
|
24
|
+
throw new NetworkLockError("LOCK_UNAVAILABLE", `network lock record key exceeds ${MAX_KINTONE_UNIQUE_KEY_LENGTH} characters`);
|
|
25
|
+
}
|
|
26
|
+
return value;
|
|
27
|
+
}
|
|
28
|
+
export function releaseTombstoneRecordKey(uniqueValue) {
|
|
29
|
+
const direct = `LOCKDONE:${uniqueValue}`;
|
|
30
|
+
if (direct.length <= MAX_KINTONE_UNIQUE_KEY_LENGTH)
|
|
31
|
+
return direct;
|
|
32
|
+
const digest = createHash("sha256")
|
|
33
|
+
.update(uniqueValue, "utf8")
|
|
34
|
+
.digest("base64url");
|
|
35
|
+
return `LOCKDONE:${digest}`;
|
|
36
|
+
}
|
|
37
|
+
function decodeReference(record, lockKey, recordKey, leaseDurationSec) {
|
|
38
|
+
return {
|
|
39
|
+
lockKey,
|
|
40
|
+
recordId: idOf(record),
|
|
41
|
+
originalRecordKey: recordKey,
|
|
42
|
+
leaseToken: text(record, "lease_token"),
|
|
43
|
+
ownerInvocationId: text(record, "owner_invocation_id"),
|
|
44
|
+
leaseDurationSec,
|
|
45
|
+
revision: revisionOf(record),
|
|
46
|
+
businessRevision: Number(record.revision?.value),
|
|
47
|
+
heartbeatAt: text(record, "heartbeat_at"),
|
|
48
|
+
leaseExpiresAt: text(record, "lease_expires_at"),
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
export class NetworkLockManager {
|
|
52
|
+
client;
|
|
53
|
+
config;
|
|
54
|
+
now;
|
|
55
|
+
uuid;
|
|
56
|
+
lockKey;
|
|
57
|
+
recordKey;
|
|
58
|
+
constructor(config) {
|
|
59
|
+
if (!Number.isSafeInteger(config.leaseDurationSec) ||
|
|
60
|
+
config.leaseDurationSec <= 0) {
|
|
61
|
+
throw new TypeError("leaseDurationSec must be a positive integer");
|
|
62
|
+
}
|
|
63
|
+
this.config = config;
|
|
64
|
+
this.now = config.now ?? (() => new Date());
|
|
65
|
+
this.uuid = config.uuid ?? randomUUID;
|
|
66
|
+
this.lockKey = networkLockKey(config.profile, config.networkId);
|
|
67
|
+
this.recordKey = recordKeyFor(this.lockKey);
|
|
68
|
+
this.client = new KintoneClient({
|
|
69
|
+
baseUrl: config.baseUrl,
|
|
70
|
+
appId: config.appId,
|
|
71
|
+
apiToken: config.apiToken,
|
|
72
|
+
...(config.fetch ? { fetch: config.fetch } : {}),
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
async acquire() {
|
|
76
|
+
const acquiredAt = this.now();
|
|
77
|
+
const leaseToken = this.uuid();
|
|
78
|
+
const leaseExpiresAt = new Date(acquiredAt.getTime() + this.config.leaseDurationSec * 1000).toISOString();
|
|
79
|
+
const record = {
|
|
80
|
+
record_key: field(this.recordKey),
|
|
81
|
+
record_type: field("NETWORK_LOCK"),
|
|
82
|
+
lock_key: field(this.lockKey),
|
|
83
|
+
profile: field(this.config.profile),
|
|
84
|
+
owner_invocation_id: field(this.config.ownerInvocationId),
|
|
85
|
+
lease_token: field(leaseToken),
|
|
86
|
+
started_at: field(acquiredAt.toISOString()),
|
|
87
|
+
heartbeat_at: field(acquiredAt.toISOString()),
|
|
88
|
+
lease_expires_at: field(leaseExpiresAt),
|
|
89
|
+
status: field("RUNNING"),
|
|
90
|
+
revision: field(1),
|
|
91
|
+
status_reason: field(`owner_instance_id=${this.config.ownerInstanceId}`),
|
|
92
|
+
};
|
|
93
|
+
try {
|
|
94
|
+
const created = await this.client.postRecord(record);
|
|
95
|
+
return decodeReference({
|
|
96
|
+
...record,
|
|
97
|
+
$id: field(created.id),
|
|
98
|
+
$revision: field(created.revision),
|
|
99
|
+
}, this.lockKey, this.recordKey, this.config.leaseDurationSec);
|
|
100
|
+
}
|
|
101
|
+
catch (error) {
|
|
102
|
+
if (!(error instanceof KintoneTransportError ||
|
|
103
|
+
(error instanceof KintoneApiError &&
|
|
104
|
+
error.status === 400 &&
|
|
105
|
+
error.apiCode === "CB_VA01"))) {
|
|
106
|
+
throw new NetworkLockError("LOCK_UNAVAILABLE", "network lock insert failed", error);
|
|
107
|
+
}
|
|
108
|
+
const current = await this.getUniqueAfterAcquireFailure(error);
|
|
109
|
+
if (text(current, "status") === "RUNNING" &&
|
|
110
|
+
text(current, "lease_token") === leaseToken &&
|
|
111
|
+
text(current, "owner_invocation_id") === this.config.ownerInvocationId) {
|
|
112
|
+
return decodeReference(current, this.lockKey, this.recordKey, this.config.leaseDurationSec);
|
|
113
|
+
}
|
|
114
|
+
if (text(current, "status") === "RUNNING") {
|
|
115
|
+
throw new NetworkLockError("LOCK_CONFLICT", "network lock is held", error);
|
|
116
|
+
}
|
|
117
|
+
throw new NetworkLockError("LOCK_UNAVAILABLE", "network lock insert outcome could not be confirmed", error);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
async heartbeat(reference) {
|
|
121
|
+
const current = await this.getUnique(reference.originalRecordKey);
|
|
122
|
+
this.assertFence(current, reference);
|
|
123
|
+
const at = this.now();
|
|
124
|
+
const leaseExpiresAt = new Date(at.getTime() + reference.leaseDurationSec * 1000).toISOString();
|
|
125
|
+
let revision;
|
|
126
|
+
try {
|
|
127
|
+
revision = await this.client.putRecord(reference.originalRecordKey, reference.revision, {
|
|
128
|
+
heartbeat_at: field(at.toISOString()),
|
|
129
|
+
lease_expires_at: field(leaseExpiresAt),
|
|
130
|
+
revision: field(reference.businessRevision + 1),
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
catch (error) {
|
|
134
|
+
if (error instanceof KintoneApiError && error.status === 409) {
|
|
135
|
+
throw new NetworkLockError("LEASE_REVISION_CONFLICT", "network lease revision conflicted", error);
|
|
136
|
+
}
|
|
137
|
+
if (error instanceof KintoneTransportError) {
|
|
138
|
+
const adjudicated = await this.getUnique(reference.originalRecordKey);
|
|
139
|
+
if (text(adjudicated, "lease_token") === reference.leaseToken &&
|
|
140
|
+
revisionOf(adjudicated) === reference.revision + 1 &&
|
|
141
|
+
text(adjudicated, "heartbeat_at") === at.toISOString() &&
|
|
142
|
+
text(adjudicated, "lease_expires_at") === leaseExpiresAt) {
|
|
143
|
+
revision = revisionOf(adjudicated);
|
|
144
|
+
}
|
|
145
|
+
else {
|
|
146
|
+
throw new NetworkLockError("LOCK_UNAVAILABLE", "network heartbeat outcome could not be confirmed", error);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
else {
|
|
150
|
+
throw new NetworkLockError("LOCK_UNAVAILABLE", "network heartbeat failed", error);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
reference.revision = revision;
|
|
154
|
+
reference.businessRevision += 1;
|
|
155
|
+
reference.heartbeatAt = at.toISOString();
|
|
156
|
+
reference.leaseExpiresAt = leaseExpiresAt;
|
|
157
|
+
return reference;
|
|
158
|
+
}
|
|
159
|
+
async release(reference, status = "SUCCESS", resultCode = "NORMAL") {
|
|
160
|
+
const current = await this.getUnique(reference.originalRecordKey);
|
|
161
|
+
this.assertFence(current, reference);
|
|
162
|
+
const tombstone = releaseTombstoneRecordKey(`${reference.lockKey}:${reference.leaseToken}`);
|
|
163
|
+
const releaseRecord = () => this.client.putRecordById(reference.recordId, reference.revision, {
|
|
164
|
+
record_key: field(tombstone),
|
|
165
|
+
lock_key: field(""),
|
|
166
|
+
owner_invocation_id: field(""),
|
|
167
|
+
lease_token: field(""),
|
|
168
|
+
status: field(status),
|
|
169
|
+
status_reason: field(resultCode),
|
|
170
|
+
finished_at: field(this.now().toISOString()),
|
|
171
|
+
revision: field(reference.businessRevision + 1),
|
|
172
|
+
});
|
|
173
|
+
const adjudicateLostResponse = async (cause) => {
|
|
174
|
+
try {
|
|
175
|
+
const adjudicated = await this.getUnique(tombstone);
|
|
176
|
+
if (text(adjudicated, "lock_key") === "" &&
|
|
177
|
+
text(adjudicated, "lease_token") === "" &&
|
|
178
|
+
text(adjudicated, "status") === status) {
|
|
179
|
+
reference.revision = revisionOf(adjudicated);
|
|
180
|
+
reference.businessRevision = Number(adjudicated.revision?.value);
|
|
181
|
+
return { released: true, recordKey: tombstone };
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
catch {
|
|
185
|
+
// The stable error below intentionally hides ambiguous remote details.
|
|
186
|
+
}
|
|
187
|
+
throw new NetworkLockError("LOCK_UNAVAILABLE", "network lock release outcome could not be confirmed", cause);
|
|
188
|
+
};
|
|
189
|
+
try {
|
|
190
|
+
const revision = await releaseRecord();
|
|
191
|
+
reference.revision = revision;
|
|
192
|
+
reference.businessRevision += 1;
|
|
193
|
+
return { released: true, recordKey: tombstone };
|
|
194
|
+
}
|
|
195
|
+
catch (error) {
|
|
196
|
+
if (error instanceof KintoneApiError && error.status === 409) {
|
|
197
|
+
let latest;
|
|
198
|
+
try {
|
|
199
|
+
latest = await this.getUnique(reference.originalRecordKey);
|
|
200
|
+
}
|
|
201
|
+
catch (retryError) {
|
|
202
|
+
throw new NetworkLockError("LEASE_REVISION_CONFLICT", "network lock release revision conflicted", retryError);
|
|
203
|
+
}
|
|
204
|
+
if (text(latest, "record_key") !== reference.originalRecordKey ||
|
|
205
|
+
text(latest, "lease_token") !== reference.leaseToken) {
|
|
206
|
+
throw new NetworkLockError("LEASE_REVISION_CONFLICT", "network lock release revision conflicted", error);
|
|
207
|
+
}
|
|
208
|
+
// A revision advanced by our own heartbeat is safe to release once.
|
|
209
|
+
reference.revision = revisionOf(latest);
|
|
210
|
+
reference.businessRevision = Number(latest.revision?.value);
|
|
211
|
+
try {
|
|
212
|
+
const revision = await releaseRecord();
|
|
213
|
+
reference.revision = revision;
|
|
214
|
+
reference.businessRevision += 1;
|
|
215
|
+
return { released: true, recordKey: tombstone };
|
|
216
|
+
}
|
|
217
|
+
catch (retryError) {
|
|
218
|
+
if (retryError instanceof KintoneApiError &&
|
|
219
|
+
retryError.status === 409)
|
|
220
|
+
throw new NetworkLockError("LEASE_REVISION_CONFLICT", "network lock release revision conflicted", retryError);
|
|
221
|
+
if (retryError instanceof KintoneTransportError)
|
|
222
|
+
return adjudicateLostResponse(retryError);
|
|
223
|
+
throw new NetworkLockError("LOCK_UNAVAILABLE", "network lock release failed", retryError);
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
if (error instanceof KintoneTransportError) {
|
|
227
|
+
return adjudicateLostResponse(error);
|
|
228
|
+
}
|
|
229
|
+
throw new NetworkLockError("LOCK_UNAVAILABLE", "network lock release failed", error);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
async getUniqueAfterAcquireFailure(cause) {
|
|
233
|
+
try {
|
|
234
|
+
return await this.getUnique(this.recordKey);
|
|
235
|
+
}
|
|
236
|
+
catch (error) {
|
|
237
|
+
throw new NetworkLockError("LOCK_UNAVAILABLE", "network lock holder could not be confirmed", { cause, adjudication: error });
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
async getUnique(recordKey) {
|
|
241
|
+
let records;
|
|
242
|
+
try {
|
|
243
|
+
records = await this.client.getRecords(`record_key in (${quote(recordKey)})`);
|
|
244
|
+
}
|
|
245
|
+
catch (error) {
|
|
246
|
+
throw new NetworkLockError("LOCK_UNAVAILABLE", "network lock read failed", error);
|
|
247
|
+
}
|
|
248
|
+
if (records.length !== 1) {
|
|
249
|
+
throw new NetworkLockError("LOCK_UNAVAILABLE", `network lock read returned ${records.length} records`);
|
|
250
|
+
}
|
|
251
|
+
return records[0];
|
|
252
|
+
}
|
|
253
|
+
assertFence(current, reference) {
|
|
254
|
+
if (text(current, "lease_token") !== reference.leaseToken) {
|
|
255
|
+
throw new NetworkLockError("LEASE_TOKEN_MISMATCH", "network lease token no longer matches");
|
|
256
|
+
}
|
|
257
|
+
if (revisionOf(current) !== reference.revision) {
|
|
258
|
+
throw new NetworkLockError("LEASE_REVISION_MISMATCH", "network lease revision no longer matches");
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
export class LeaseMonitor {
|
|
263
|
+
manager;
|
|
264
|
+
reference;
|
|
265
|
+
config;
|
|
266
|
+
state = "HELD";
|
|
267
|
+
failures = 0;
|
|
268
|
+
timer = null;
|
|
269
|
+
tickInFlight = null;
|
|
270
|
+
listeners = new Set();
|
|
271
|
+
now;
|
|
272
|
+
failureThreshold;
|
|
273
|
+
safetyMs;
|
|
274
|
+
schedule;
|
|
275
|
+
cancelSchedule;
|
|
276
|
+
constructor(manager, reference, config) {
|
|
277
|
+
this.manager = manager;
|
|
278
|
+
this.reference = reference;
|
|
279
|
+
this.config = config;
|
|
280
|
+
validateMonitorConfig(config);
|
|
281
|
+
this.now = config.now ?? (() => new Date());
|
|
282
|
+
this.failureThreshold = config.consecutiveFailureThreshold ?? 3;
|
|
283
|
+
this.safetyMs =
|
|
284
|
+
(config.remainingLeaseSafetySec ?? config.heartbeatIntervalSec) * 1000;
|
|
285
|
+
this.schedule =
|
|
286
|
+
config.schedule ??
|
|
287
|
+
((callback, intervalMs) => setInterval(callback, intervalMs));
|
|
288
|
+
this.cancelSchedule =
|
|
289
|
+
config.cancelSchedule ??
|
|
290
|
+
((handle) => clearInterval(handle));
|
|
291
|
+
}
|
|
292
|
+
get currentState() {
|
|
293
|
+
return this.state;
|
|
294
|
+
}
|
|
295
|
+
canStartNewNode() {
|
|
296
|
+
return this.state === "HELD";
|
|
297
|
+
}
|
|
298
|
+
canPersistResults() {
|
|
299
|
+
return this.state === "HELD" || this.state === "FINAL_WRITE_CONFIRMED";
|
|
300
|
+
}
|
|
301
|
+
requiresNetworkLeaseInterruptedFinalization() {
|
|
302
|
+
return this.state === "FINAL_WRITE_CONFIRMED";
|
|
303
|
+
}
|
|
304
|
+
markControlPlaneUnreachable() {
|
|
305
|
+
if (this.state !== "LEASE_UNCERTAIN")
|
|
306
|
+
this.transition("LEASE_UNCERTAIN", "FAILURE_THRESHOLD");
|
|
307
|
+
}
|
|
308
|
+
subscribe(listener) {
|
|
309
|
+
this.listeners.add(listener);
|
|
310
|
+
return () => this.listeners.delete(listener);
|
|
311
|
+
}
|
|
312
|
+
async tick() {
|
|
313
|
+
if (this.tickInFlight !== null)
|
|
314
|
+
return this.tickInFlight;
|
|
315
|
+
const operation = this.performTick();
|
|
316
|
+
this.tickInFlight = operation;
|
|
317
|
+
try {
|
|
318
|
+
return await operation;
|
|
319
|
+
}
|
|
320
|
+
finally {
|
|
321
|
+
if (this.tickInFlight === operation)
|
|
322
|
+
this.tickInFlight = null;
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
async performTick() {
|
|
326
|
+
if (this.state !== "HELD")
|
|
327
|
+
return false;
|
|
328
|
+
try {
|
|
329
|
+
await this.manager.heartbeat(this.reference);
|
|
330
|
+
this.failures = 0;
|
|
331
|
+
return true;
|
|
332
|
+
}
|
|
333
|
+
catch (error) {
|
|
334
|
+
this.failures += 1;
|
|
335
|
+
const fenceLost = error instanceof NetworkLockError &&
|
|
336
|
+
(error.code === "LEASE_TOKEN_MISMATCH" ||
|
|
337
|
+
error.code === "LEASE_REVISION_MISMATCH" ||
|
|
338
|
+
error.code === "LEASE_REVISION_CONFLICT");
|
|
339
|
+
if (fenceLost)
|
|
340
|
+
this.transition("LEASE_UNCERTAIN", "FENCE_LOST");
|
|
341
|
+
else if (this.failures >= this.failureThreshold)
|
|
342
|
+
this.transition("LEASE_UNCERTAIN", "FAILURE_THRESHOLD");
|
|
343
|
+
else if (this.remainingLeaseMs() <= this.safetyMs)
|
|
344
|
+
this.transition("LEASE_UNCERTAIN", "LEASE_SAFETY_THRESHOLD");
|
|
345
|
+
return false;
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
async confirmLeaseForFinalWrite() {
|
|
349
|
+
if (this.tickInFlight !== null)
|
|
350
|
+
await this.tickInFlight;
|
|
351
|
+
if (this.state !== "LEASE_UNCERTAIN")
|
|
352
|
+
return false;
|
|
353
|
+
try {
|
|
354
|
+
await this.manager.heartbeat(this.reference);
|
|
355
|
+
this.transition("FINAL_WRITE_CONFIRMED", "FINAL_WRITE_CONFIRMED");
|
|
356
|
+
return true;
|
|
357
|
+
}
|
|
358
|
+
catch {
|
|
359
|
+
return false;
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
start() {
|
|
363
|
+
if (this.timer !== null)
|
|
364
|
+
return;
|
|
365
|
+
this.timer = this.schedule(() => void this.tick().catch(() => undefined), this.config.heartbeatIntervalSec * 1000);
|
|
366
|
+
}
|
|
367
|
+
stop() {
|
|
368
|
+
if (this.timer === null)
|
|
369
|
+
return;
|
|
370
|
+
this.cancelSchedule(this.timer);
|
|
371
|
+
this.timer = null;
|
|
372
|
+
}
|
|
373
|
+
remainingLeaseMs() {
|
|
374
|
+
return Date.parse(this.reference.leaseExpiresAt) - this.now().getTime();
|
|
375
|
+
}
|
|
376
|
+
transition(current, reason) {
|
|
377
|
+
if (this.state === current)
|
|
378
|
+
return;
|
|
379
|
+
const previous = this.state;
|
|
380
|
+
this.state = current;
|
|
381
|
+
const event = { previous, current, reason, at: this.now().toISOString() };
|
|
382
|
+
for (const listener of this.listeners)
|
|
383
|
+
listener(event);
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
function validateMonitorConfig(config) {
|
|
387
|
+
if ((config.schedule === undefined) !==
|
|
388
|
+
(config.cancelSchedule === undefined)) {
|
|
389
|
+
throw new TypeError("schedule and cancelSchedule must be provided together");
|
|
390
|
+
}
|
|
391
|
+
for (const [name, value] of [
|
|
392
|
+
["leaseDurationSec", config.leaseDurationSec],
|
|
393
|
+
["heartbeatIntervalSec", config.heartbeatIntervalSec],
|
|
394
|
+
["consecutiveFailureThreshold", config.consecutiveFailureThreshold ?? 3],
|
|
395
|
+
[
|
|
396
|
+
"remainingLeaseSafetySec",
|
|
397
|
+
config.remainingLeaseSafetySec ?? config.heartbeatIntervalSec,
|
|
398
|
+
],
|
|
399
|
+
]) {
|
|
400
|
+
if (!Number.isSafeInteger(value) || value <= 0) {
|
|
401
|
+
throw new TypeError(`${name} must be a positive integer`);
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
if (config.heartbeatIntervalSec >= config.leaseDurationSec) {
|
|
405
|
+
throw new TypeError("heartbeatIntervalSec must be shorter than leaseDurationSec");
|
|
406
|
+
}
|
|
407
|
+
}
|
|
408
|
+
//# sourceMappingURL=network-lock.js.map
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import type { AttemptResolution, CancelRequest, NetworkRun, NetworkRunStatus, NodeAttempt, NodeAttemptStatus, NodeState, OperationAudit, RunInvocation, RunInvocationStatus } from "../domain/persistence-model.js";
|
|
2
|
+
export type RepositoryErrorCode = "DUPLICATE_RECORD" | "REVISION_CONFLICT" | "INVALID_STATE_TRANSITION" | "ATTEMPT_NUMBER_CONFLICT" | "ATTEMPT_TERMINAL" | "ATTEMPT_LIFECYCLE_VIOLATION" | "AMBIGUOUS_WRITE" | "RECORD_NOT_FOUND" | "MULTIPLE_RECORDS" | "UNIQUE_KEY_TOO_LONG" | "REMOTE_ERROR" | "AUDIT_CONFLICT";
|
|
3
|
+
export declare class RepositoryError extends Error {
|
|
4
|
+
readonly code: RepositoryErrorCode;
|
|
5
|
+
readonly causeDetail: unknown;
|
|
6
|
+
constructor(code: RepositoryErrorCode, message: string, causeDetail?: unknown);
|
|
7
|
+
}
|
|
8
|
+
export interface Versioned<T> {
|
|
9
|
+
value: T;
|
|
10
|
+
revision: number;
|
|
11
|
+
}
|
|
12
|
+
export interface RunAggregateUpdate {
|
|
13
|
+
status: NetworkRunStatus;
|
|
14
|
+
started_at: string | null;
|
|
15
|
+
finished_at: string | null;
|
|
16
|
+
updated_at: string;
|
|
17
|
+
}
|
|
18
|
+
export interface InvocationFinalization {
|
|
19
|
+
status: Exclude<RunInvocationStatus, "RUNNING">;
|
|
20
|
+
result_code: string;
|
|
21
|
+
finished_at: string;
|
|
22
|
+
selected_node_ids?: readonly string[];
|
|
23
|
+
preserved_node_ids?: readonly string[];
|
|
24
|
+
blocked_node_ids?: readonly string[];
|
|
25
|
+
reason?: string;
|
|
26
|
+
}
|
|
27
|
+
export interface NodeStateWrite {
|
|
28
|
+
value: NodeState;
|
|
29
|
+
expected_revision: number | null;
|
|
30
|
+
resolution_event?: {
|
|
31
|
+
event_type: "ATTEMPT_RESOLVED";
|
|
32
|
+
attempt_id: string;
|
|
33
|
+
resolved_outcome: "SUCCESS" | "FAILED" | "CANCELLED";
|
|
34
|
+
resolved_at: string;
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
export interface CreateAttemptInput {
|
|
38
|
+
node_state: Versioned<NodeState>;
|
|
39
|
+
node_attempt_id: string;
|
|
40
|
+
invocation_id: string;
|
|
41
|
+
}
|
|
42
|
+
export interface AttemptExecutionStart {
|
|
43
|
+
execution_started_at: string;
|
|
44
|
+
}
|
|
45
|
+
export interface AttemptInputBaselineWrite {
|
|
46
|
+
error_message: string;
|
|
47
|
+
}
|
|
48
|
+
export interface AttemptFinalization {
|
|
49
|
+
status: Exclude<NodeAttemptStatus, "RUNNING">;
|
|
50
|
+
result_code: string;
|
|
51
|
+
runner_execution_started_at: string | null;
|
|
52
|
+
execution_id: string | null;
|
|
53
|
+
finished_at: string;
|
|
54
|
+
duration_sec: number | null;
|
|
55
|
+
error_message: string | null;
|
|
56
|
+
read_count: number;
|
|
57
|
+
written_count: number;
|
|
58
|
+
last_successful_chunk_no: number | null;
|
|
59
|
+
last_written_key: string | null;
|
|
60
|
+
}
|
|
61
|
+
export type Inconsistency = {
|
|
62
|
+
code: "ACTIVE_ATTEMPT_MISSING";
|
|
63
|
+
node_state: Versioned<NodeState>;
|
|
64
|
+
} | {
|
|
65
|
+
code: "STATE_ATTEMPT_STATUS_MISMATCH";
|
|
66
|
+
node_state: Versioned<NodeState>;
|
|
67
|
+
attempt: Versioned<NodeAttempt>;
|
|
68
|
+
} | {
|
|
69
|
+
code: "MULTIPLE_RUNNING_ATTEMPTS";
|
|
70
|
+
node_state: Versioned<NodeState>;
|
|
71
|
+
attempts: Versioned<NodeAttempt>[];
|
|
72
|
+
};
|
|
73
|
+
export interface PersistenceRepository {
|
|
74
|
+
createRun(run: NetworkRun): Promise<Versioned<NetworkRun>>;
|
|
75
|
+
getRun(runId: string): Promise<Versioned<NetworkRun>>;
|
|
76
|
+
getRunByBusinessKey(profile: string, networkId: string, businessKey: string): Promise<Versioned<NetworkRun> | null>;
|
|
77
|
+
listRuns(profile: string, networkId: string): Promise<Versioned<NetworkRun>[]>;
|
|
78
|
+
getCancelRequest(runId: string): Promise<Versioned<CancelRequest> | null>;
|
|
79
|
+
createCancelRequest(request: CancelRequest): Promise<Versioned<CancelRequest>>;
|
|
80
|
+
updateCancelRequest(runId: string, expectedRevision: number, request: CancelRequest): Promise<Versioned<CancelRequest>>;
|
|
81
|
+
updateRunAggregate(runId: string, expectedRevision: number, update: RunAggregateUpdate): Promise<Versioned<NetworkRun>>;
|
|
82
|
+
archiveRun(runId: string, expectedRevision: number, at: string): Promise<Versioned<NetworkRun>>;
|
|
83
|
+
createInvocation(invocation: RunInvocation): Promise<Versioned<RunInvocation>>;
|
|
84
|
+
getInvocations(runId: string): Promise<Versioned<RunInvocation>[]>;
|
|
85
|
+
finalizeInvocation(invocationId: string, expectedRevision: number, finalization: InvocationFinalization): Promise<Versioned<RunInvocation>>;
|
|
86
|
+
getNodeStates(runId: string): Promise<Versioned<NodeState>[]>;
|
|
87
|
+
getAttempts(runId: string): Promise<Versioned<NodeAttempt>[]>;
|
|
88
|
+
getResolutions(runId: string): Promise<Versioned<AttemptResolution>[]>;
|
|
89
|
+
upsertNodeState(write: NodeStateWrite): Promise<Versioned<NodeState>>;
|
|
90
|
+
createAttempt(input: CreateAttemptInput): Promise<Versioned<NodeAttempt>>;
|
|
91
|
+
setAttemptInputBaseline(attemptId: string, expectedRevision: number, write: AttemptInputBaselineWrite): Promise<Versioned<NodeAttempt>>;
|
|
92
|
+
setAttemptExecutionStarted(attemptId: string, expectedRevision: number, start: AttemptExecutionStart): Promise<Versioned<NodeAttempt>>;
|
|
93
|
+
finalizeAttempt(attemptId: string, expectedRevision: number, finalization: AttemptFinalization): Promise<Versioned<NodeAttempt>>;
|
|
94
|
+
appendResolution(resolution: AttemptResolution): Promise<Versioned<AttemptResolution>>;
|
|
95
|
+
appendOperationAudit(audit: OperationAudit): Promise<Versioned<OperationAudit>>;
|
|
96
|
+
getOperationAuditByEventId(eventId: string): Promise<Versioned<OperationAudit> | null>;
|
|
97
|
+
listInconsistencies(runId: string): Promise<Inconsistency[]>;
|
|
98
|
+
}
|
|
99
|
+
/** The complete persistence surface available to read-only status inspection. */
|
|
100
|
+
export type StatusReadRepository = Pick<PersistenceRepository, "getRun" | "getRunByBusinessKey" | "listRuns" | "getInvocations" | "getNodeStates" | "getAttempts" | "getResolutions" | "getCancelRequest">;
|
|
101
|
+
//# sourceMappingURL=repository.d.ts.map
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export class RepositoryError extends Error {
|
|
2
|
+
code;
|
|
3
|
+
causeDetail;
|
|
4
|
+
constructor(code, message, causeDetail) {
|
|
5
|
+
super(message);
|
|
6
|
+
this.name = "RepositoryError";
|
|
7
|
+
this.code = code;
|
|
8
|
+
this.causeDetail = causeDetail;
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=repository.js.map
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { NodeStateStatus } from "../domain/persistence-model.js";
|
|
2
|
+
export declare function isAllowedNodeStateTransition(from: NodeStateStatus, to: NodeStateStatus): boolean;
|
|
3
|
+
export declare function isAllowedResolutionTransition(from: NodeStateStatus, to: NodeStateStatus): boolean;
|
|
4
|
+
//# sourceMappingURL=state-transition.d.ts.map
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
const ALLOWED_TRANSITIONS = {
|
|
2
|
+
WAITING: new Set(["WAITING", "RUNNING", "BLOCKED"]),
|
|
3
|
+
RUNNING: new Set([
|
|
4
|
+
"RUNNING",
|
|
5
|
+
"WAITING",
|
|
6
|
+
"SUCCESS",
|
|
7
|
+
"FAILED",
|
|
8
|
+
"CANCELLED",
|
|
9
|
+
"UNKNOWN",
|
|
10
|
+
]),
|
|
11
|
+
SUCCESS: new Set(["SUCCESS", "WAITING"]),
|
|
12
|
+
FAILED: new Set(["FAILED", "WAITING"]),
|
|
13
|
+
BLOCKED: new Set(["BLOCKED", "WAITING"]),
|
|
14
|
+
SKIPPED: new Set(["SKIPPED"]),
|
|
15
|
+
CANCELLED: new Set(["CANCELLED", "WAITING"]),
|
|
16
|
+
UNKNOWN: new Set(["UNKNOWN", "SUCCESS", "FAILED", "CANCELLED"]),
|
|
17
|
+
};
|
|
18
|
+
export function isAllowedNodeStateTransition(from, to) {
|
|
19
|
+
return ALLOWED_TRANSITIONS[from].has(to);
|
|
20
|
+
}
|
|
21
|
+
export function isAllowedResolutionTransition(from, to) {
|
|
22
|
+
return (from === "FAILED" &&
|
|
23
|
+
(to === "SUCCESS" || to === "FAILED" || to === "CANCELLED"));
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=state-transition.js.map
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import type { ArchiveRunOutcome } from "../orchestration/archive-run.js";
|
|
2
|
+
import type { StatusOutput } from "../orchestration/status.js";
|
|
3
|
+
import type { PollRequestsNetwork } from "./poll-requests-config.js";
|
|
4
|
+
import type { RequestRecord } from "./request-model.js";
|
|
5
|
+
import type { StartNetworkInput } from "./start-request.js";
|
|
6
|
+
export declare const DEFAULT_CHILD_OUTPUT_LIMIT: number;
|
|
7
|
+
export declare const REQUESTED_BY_MAX_LENGTH = 1024;
|
|
8
|
+
export interface ChildProcessResult {
|
|
9
|
+
readonly exitCode: number | null;
|
|
10
|
+
readonly stdout: string;
|
|
11
|
+
readonly stderr: string;
|
|
12
|
+
readonly stdoutTruncated: boolean;
|
|
13
|
+
readonly stderrTruncated: boolean;
|
|
14
|
+
readonly spawnError?: string;
|
|
15
|
+
}
|
|
16
|
+
export interface RunNetworkJsonOutput {
|
|
17
|
+
readonly outcome: "NEW" | "RESUME" | "NOOP" | "REJECTED";
|
|
18
|
+
readonly run_id: string | null;
|
|
19
|
+
readonly invocation_id: string | null;
|
|
20
|
+
readonly aggregate_status: string | null;
|
|
21
|
+
readonly invocation_result_code: string;
|
|
22
|
+
readonly retry_brake_node_ids?: readonly string[];
|
|
23
|
+
readonly blocked_run_ids?: readonly string[];
|
|
24
|
+
}
|
|
25
|
+
export interface FlownetChildClientOptions {
|
|
26
|
+
readonly profile: string;
|
|
27
|
+
readonly environment?: NodeJS.ProcessEnv;
|
|
28
|
+
readonly outputLimit?: number;
|
|
29
|
+
readonly cliPath?: string;
|
|
30
|
+
readonly execute?: (command: string, args: readonly string[], options: {
|
|
31
|
+
readonly env: NodeJS.ProcessEnv;
|
|
32
|
+
readonly shell: false;
|
|
33
|
+
}) => Promise<ChildProcessResult>;
|
|
34
|
+
readonly makeTempDirectory?: () => Promise<string>;
|
|
35
|
+
readonly removeTempDirectory?: (path: string) => Promise<void>;
|
|
36
|
+
}
|
|
37
|
+
export declare class FlownetChildClient {
|
|
38
|
+
private readonly profile;
|
|
39
|
+
private readonly environment;
|
|
40
|
+
private readonly outputLimit;
|
|
41
|
+
private readonly cliPath;
|
|
42
|
+
private readonly executeProcess;
|
|
43
|
+
private readonly makeTempDirectory;
|
|
44
|
+
private readonly removeTempDirectory;
|
|
45
|
+
constructor(options: FlownetChildClientOptions);
|
|
46
|
+
status(network: PollRequestsNetwork, selector: string | {
|
|
47
|
+
readonly businessKey: string;
|
|
48
|
+
}): Promise<StatusOutput | null>;
|
|
49
|
+
runNetwork(network: PollRequestsNetwork, request: RequestRecord): Promise<{
|
|
50
|
+
readonly output: RunNetworkJsonOutput | null;
|
|
51
|
+
readonly process: ChildProcessResult;
|
|
52
|
+
}>;
|
|
53
|
+
startNetwork(network: PollRequestsNetwork, request: Pick<RequestRecord, "id" | "creatorCode">, input: StartNetworkInput): Promise<{
|
|
54
|
+
readonly output: RunNetworkJsonOutput | null;
|
|
55
|
+
readonly process: ChildProcessResult;
|
|
56
|
+
}>;
|
|
57
|
+
cancelRun(request: RequestRecord, release: boolean): Promise<ChildProcessResult>;
|
|
58
|
+
archiveRun(network: PollRequestsNetwork, request: RequestRecord): Promise<{
|
|
59
|
+
readonly output: ArchiveRunOutcome | null;
|
|
60
|
+
readonly process: ChildProcessResult;
|
|
61
|
+
}>;
|
|
62
|
+
private run;
|
|
63
|
+
}
|
|
64
|
+
export declare class ChildClientError extends Error {
|
|
65
|
+
readonly code: string;
|
|
66
|
+
readonly result: ChildProcessResult;
|
|
67
|
+
constructor(code: string, message: string, result: ChildProcessResult, cause?: unknown);
|
|
68
|
+
}
|
|
69
|
+
export declare function requestedBy(request: Pick<RequestRecord, "id" | "creatorCode">): string;
|
|
70
|
+
export declare class RequestCorrelationError extends Error {
|
|
71
|
+
readonly code = "REQUESTED_BY_INVALID";
|
|
72
|
+
constructor(message: string, cause?: unknown);
|
|
73
|
+
}
|
|
74
|
+
export declare function executeChild(command: string, args: readonly string[], options: {
|
|
75
|
+
readonly env: NodeJS.ProcessEnv;
|
|
76
|
+
readonly shell: false;
|
|
77
|
+
}, limit: number): Promise<ChildProcessResult>;
|
|
78
|
+
//# sourceMappingURL=flownet-child-client.d.ts.map
|