@rex0220/ksql-flownet 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.ja.md +81 -0
- package/README.md +117 -0
- package/dist/bundle/bundle-builder.d.ts +49 -0
- package/dist/bundle/bundle-builder.js +208 -0
- package/dist/bundle/index.d.ts +4 -0
- package/dist/bundle/index.js +4 -0
- package/dist/bundle/upload.d.ts +28 -0
- package/dist/bundle/upload.js +72 -0
- package/dist/bundle/zip-store.d.ts +17 -0
- package/dist/bundle/zip-store.js +175 -0
- package/dist/cli/archive-run-command.d.ts +12 -0
- package/dist/cli/archive-run-command.js +131 -0
- package/dist/cli/cancel-run-command.d.ts +9 -0
- package/dist/cli/cancel-run-command.js +77 -0
- package/dist/cli/force-unlock-network-command.d.ts +8 -0
- package/dist/cli/force-unlock-network-command.js +129 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.js +132 -0
- package/dist/cli/plan-command.d.ts +2 -0
- package/dist/cli/plan-command.js +100 -0
- package/dist/cli/poll-requests-command.d.ts +3 -0
- package/dist/cli/poll-requests-command.js +74 -0
- package/dist/cli/record-job-unlock-command.d.ts +11 -0
- package/dist/cli/record-job-unlock-command.js +78 -0
- package/dist/cli/resolve-node-command.d.ts +13 -0
- package/dist/cli/resolve-node-command.js +174 -0
- package/dist/cli/run-network-command.d.ts +14 -0
- package/dist/cli/run-network-command.js +418 -0
- package/dist/cli/status-command.d.ts +5 -0
- package/dist/cli/status-command.js +137 -0
- package/dist/cli/validate-command.d.ts +6 -0
- package/dist/cli/validate-command.js +17 -0
- package/dist/dag/descendants.d.ts +9 -0
- package/dist/dag/descendants.js +67 -0
- package/dist/dag/topological-sort.d.ts +7 -0
- package/dist/dag/topological-sort.js +32 -0
- package/dist/domain/business-key.d.ts +15 -0
- package/dist/domain/business-key.js +249 -0
- package/dist/domain/canonical-lock-key.d.ts +17 -0
- package/dist/domain/canonical-lock-key.js +60 -0
- package/dist/domain/canonical-record-key.d.ts +18 -0
- package/dist/domain/canonical-record-key.js +70 -0
- package/dist/domain/load-network.d.ts +4 -0
- package/dist/domain/load-network.js +47 -0
- package/dist/domain/network-definition.d.ts +47 -0
- package/dist/domain/network-definition.js +7 -0
- package/dist/domain/persistence-model.d.ts +198 -0
- package/dist/domain/persistence-model.js +2 -0
- package/dist/domain/run-aggregate.d.ts +3 -0
- package/dist/domain/run-aggregate.js +15 -0
- package/dist/domain/validate-network-path.d.ts +11 -0
- package/dist/domain/validate-network-path.js +33 -0
- package/dist/domain/validate-network.d.ts +3 -0
- package/dist/domain/validate-network.js +180 -0
- package/dist/executor/attempt-executor.d.ts +43 -0
- package/dist/executor/attempt-executor.js +236 -0
- package/dist/executor/job-log-reader.d.ts +37 -0
- package/dist/executor/job-log-reader.js +91 -0
- package/dist/executor/ksql-flow-cli.d.ts +86 -0
- package/dist/executor/ksql-flow-cli.js +161 -0
- package/dist/executor/preflight.d.ts +41 -0
- package/dist/executor/preflight.js +150 -0
- package/dist/executor/result-classifier.d.ts +69 -0
- package/dist/executor/result-classifier.js +237 -0
- package/dist/executor/run-subprocess.d.ts +64 -0
- package/dist/executor/run-subprocess.js +175 -0
- package/dist/io/input-baseline.d.ts +19 -0
- package/dist/io/input-baseline.js +214 -0
- package/dist/io/io-config.d.ts +6 -0
- package/dist/io/io-config.js +33 -0
- package/dist/io/io-path.d.ts +37 -0
- package/dist/io/io-path.js +279 -0
- package/dist/orchestration/archive-run.d.ts +53 -0
- package/dist/orchestration/archive-run.js +323 -0
- package/dist/orchestration/cancel-request.d.ts +18 -0
- package/dist/orchestration/cancel-request.js +67 -0
- package/dist/orchestration/ensure-run.d.ts +83 -0
- package/dist/orchestration/ensure-run.js +632 -0
- package/dist/orchestration/job-lock-audit.d.ts +24 -0
- package/dist/orchestration/job-lock-audit.js +123 -0
- package/dist/orchestration/orphan-attempt-adjudication.d.ts +39 -0
- package/dist/orchestration/orphan-attempt-adjudication.js +167 -0
- package/dist/orchestration/reconciliation.d.ts +41 -0
- package/dist/orchestration/reconciliation.js +526 -0
- package/dist/orchestration/resolve-node.d.ts +29 -0
- package/dist/orchestration/resolve-node.js +105 -0
- package/dist/orchestration/run-activity.d.ts +13 -0
- package/dist/orchestration/run-activity.js +14 -0
- package/dist/orchestration/sequential-scheduler.d.ts +79 -0
- package/dist/orchestration/sequential-scheduler.js +706 -0
- package/dist/orchestration/status.d.ts +91 -0
- package/dist/orchestration/status.js +171 -0
- package/dist/persistence/in-memory-repository.d.ts +41 -0
- package/dist/persistence/in-memory-repository.js +333 -0
- package/dist/persistence/kintone/client.d.ts +36 -0
- package/dist/persistence/kintone/client.js +93 -0
- package/dist/persistence/kintone/design-notes.d.ts +38 -0
- package/dist/persistence/kintone/design-notes.js +38 -0
- package/dist/persistence/kintone/repository.d.ts +44 -0
- package/dist/persistence/kintone/repository.js +836 -0
- package/dist/persistence/kintone/schema.d.ts +116 -0
- package/dist/persistence/kintone/schema.js +115 -0
- package/dist/persistence/network-lock-reader.d.ts +24 -0
- package/dist/persistence/network-lock-reader.js +43 -0
- package/dist/persistence/network-lock-recovery.d.ts +66 -0
- package/dist/persistence/network-lock-recovery.js +392 -0
- package/dist/persistence/network-lock.d.ts +97 -0
- package/dist/persistence/network-lock.js +408 -0
- package/dist/persistence/repository.d.ts +101 -0
- package/dist/persistence/repository.js +11 -0
- package/dist/persistence/state-transition.d.ts +4 -0
- package/dist/persistence/state-transition.js +25 -0
- package/dist/requests/flownet-child-client.d.ts +78 -0
- package/dist/requests/flownet-child-client.js +243 -0
- package/dist/requests/kintone-request-store.d.ts +44 -0
- package/dist/requests/kintone-request-store.js +202 -0
- package/dist/requests/poll-requests-config.d.ts +26 -0
- package/dist/requests/poll-requests-config.js +105 -0
- package/dist/requests/request-model.d.ts +57 -0
- package/dist/requests/request-model.js +305 -0
- package/dist/requests/request-poller.d.ts +33 -0
- package/dist/requests/request-poller.js +353 -0
- package/dist/requests/request-result.d.ts +18 -0
- package/dist/requests/request-result.js +215 -0
- package/dist/requests/start-request.d.ts +26 -0
- package/dist/requests/start-request.js +83 -0
- package/package.json +63 -0
- package/schemas/.gitkeep +1 -0
- package/schemas/network-definition.schema.json +121 -0
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
import { KintoneApiError, KintoneTransportError, } from "../persistence/kintone/client.js";
|
|
2
|
+
import { decideMissingResult, } from "./job-log-reader.js";
|
|
3
|
+
import { NO_EXECUTION_RESULT, readAndClassifyResult, } from "./result-classifier.js";
|
|
4
|
+
import { serializeIoAuditSummary, serializeInputAuditSummary, serializeInputBaseline, serializeOutputAuditSummary, } from "../io/input-baseline.js";
|
|
5
|
+
export class AttemptResultPersistenceDeferredError extends Error {
|
|
6
|
+
classification;
|
|
7
|
+
process;
|
|
8
|
+
code = "NETWORK_LEASE_INTERRUPTED";
|
|
9
|
+
constructor(classification, process) {
|
|
10
|
+
super("subprocess completed but the Network lease does not permit result persistence");
|
|
11
|
+
this.classification = classification;
|
|
12
|
+
this.process = process;
|
|
13
|
+
this.name = "AttemptResultPersistenceDeferredError";
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
/** FN-10 entry point for one already-created RUNNING attempt and node state. */
|
|
17
|
+
export class AttemptExecutor {
|
|
18
|
+
options;
|
|
19
|
+
now;
|
|
20
|
+
constructor(options) {
|
|
21
|
+
this.options = options;
|
|
22
|
+
this.now = options.now ?? (() => new Date().toISOString());
|
|
23
|
+
}
|
|
24
|
+
async execute(input) {
|
|
25
|
+
assertPrepared(input);
|
|
26
|
+
const startedAttempt = await this.options.repository.setAttemptExecutionStarted(input.attempt.value.node_attempt_id, input.attempt.revision, { execution_started_at: input.executionStartedAt });
|
|
27
|
+
const process = await this.options.runner.run(input);
|
|
28
|
+
let classification = await readAndClassifyResult(process.resultJsonPath, {
|
|
29
|
+
correlationId: input.correlationId,
|
|
30
|
+
attemptId: input.attemptId,
|
|
31
|
+
processExitCode: process.exitCode,
|
|
32
|
+
}, this.options.resultFileReader);
|
|
33
|
+
if (classification.kind === "VALID_RESULT" &&
|
|
34
|
+
(input.imports?.length ?? 0) > 0) {
|
|
35
|
+
const receipts = classification.result?.input_files;
|
|
36
|
+
const actual = [...(receipts ?? [])]
|
|
37
|
+
.sort((left, right) => left.name.localeCompare(right.name))
|
|
38
|
+
.map(({ name, sha256, bytes }) => ({ name, sha256, bytes }));
|
|
39
|
+
const expectedWithBytes = [...(input.imports ?? [])]
|
|
40
|
+
.sort((left, right) => left.name.localeCompare(right.name))
|
|
41
|
+
.map(({ name, sha256, bytes }) => ({ name, sha256, bytes }));
|
|
42
|
+
if (receipts === undefined ||
|
|
43
|
+
JSON.stringify(actual) !== JSON.stringify(expectedWithBytes)) {
|
|
44
|
+
classification = {
|
|
45
|
+
kind: "INVALID_RESULT",
|
|
46
|
+
attemptOutcome: "UNKNOWN",
|
|
47
|
+
resultCode: null,
|
|
48
|
+
details: ["input_files does not match the preflight input baseline"],
|
|
49
|
+
result: null,
|
|
50
|
+
invocationResultCode: "INVALID_EXECUTION_RESULT",
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
if (classification.kind === "VALID_RESULT" &&
|
|
55
|
+
(input.exports?.length ?? 0) > 0) {
|
|
56
|
+
const receipts = classification.result?.output_files;
|
|
57
|
+
const expectedNames = [...(input.exports ?? [])]
|
|
58
|
+
.sort((left, right) => left.name < right.name ? -1 : left.name > right.name ? 1 : 0)
|
|
59
|
+
.map(({ name }) => name);
|
|
60
|
+
const actualNames = (receipts ?? []).map(({ name }) => name);
|
|
61
|
+
let previousIndex = -1;
|
|
62
|
+
const matchesConfiguredOutputs = actualNames.every((name) => {
|
|
63
|
+
const index = expectedNames.indexOf(name);
|
|
64
|
+
if (index < 0 || index <= previousIndex)
|
|
65
|
+
return false;
|
|
66
|
+
previousIndex = index;
|
|
67
|
+
return true;
|
|
68
|
+
});
|
|
69
|
+
if (receipts === undefined || !matchesConfiguredOutputs) {
|
|
70
|
+
classification = {
|
|
71
|
+
kind: "INVALID_RESULT",
|
|
72
|
+
attemptOutcome: "UNKNOWN",
|
|
73
|
+
resultCode: null,
|
|
74
|
+
details: ["output_files does not match the configured export sinks"],
|
|
75
|
+
result: null,
|
|
76
|
+
invocationResultCode: "INVALID_EXECUTION_RESULT",
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
else if (classification.kind === "VALID_RESULT" &&
|
|
81
|
+
(classification.result?.output_files?.length ?? 0) > 0) {
|
|
82
|
+
classification = {
|
|
83
|
+
kind: "INVALID_RESULT",
|
|
84
|
+
attemptOutcome: "UNKNOWN",
|
|
85
|
+
resultCode: null,
|
|
86
|
+
details: ["output_files contains an unconfigured export sink"],
|
|
87
|
+
result: null,
|
|
88
|
+
invocationResultCode: "INVALID_EXECUTION_RESULT",
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
if (process.forced) {
|
|
92
|
+
classification = {
|
|
93
|
+
kind: "INVALID_RESULT",
|
|
94
|
+
attemptOutcome: "UNKNOWN",
|
|
95
|
+
resultCode: null,
|
|
96
|
+
details: [
|
|
97
|
+
"process exceeded its grace period and was forcibly terminated",
|
|
98
|
+
],
|
|
99
|
+
result: null,
|
|
100
|
+
invocationResultCode: "FORCED_TERMINATION",
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
const controlPlane = input.runControlPlaneOperation ??
|
|
104
|
+
((operation) => operation());
|
|
105
|
+
const marker = await controlPlane(() => this.readMarker(input.attemptId, classification.result?.executionId));
|
|
106
|
+
const runnerStartedAt = marker?.runnerExecutionStartedAt ?? null;
|
|
107
|
+
let terminalStatus;
|
|
108
|
+
let resultCode;
|
|
109
|
+
let stateStatus;
|
|
110
|
+
let invocationResultCode = classification.invocationResultCode;
|
|
111
|
+
if (classification.kind === "VALID_RESULT") {
|
|
112
|
+
if (classification.attemptOutcome === "LOCK_CONFLICT") {
|
|
113
|
+
terminalStatus = "CANCELLED";
|
|
114
|
+
resultCode = "PREPARE_FAILED";
|
|
115
|
+
stateStatus = "WAITING";
|
|
116
|
+
}
|
|
117
|
+
else {
|
|
118
|
+
terminalStatus = classification.attemptOutcome;
|
|
119
|
+
resultCode = classification.resultCode;
|
|
120
|
+
stateStatus = terminalStatus;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
else {
|
|
124
|
+
const missingDecision = decideMissingResult({
|
|
125
|
+
orchestratorExecutionStartedAt: startedAttempt.value.execution_started_at,
|
|
126
|
+
runnerExecutionStartedAt: runnerStartedAt,
|
|
127
|
+
durableLaunchFailureConfirmed: process.launchFailureConfirmed && marker !== undefined,
|
|
128
|
+
});
|
|
129
|
+
if (missingDecision === "NOT_EXECUTED") {
|
|
130
|
+
terminalStatus = "CANCELLED";
|
|
131
|
+
resultCode = "PREPARE_FAILED";
|
|
132
|
+
stateStatus = "WAITING";
|
|
133
|
+
invocationResultCode ??= "PREPARE_FAILED";
|
|
134
|
+
}
|
|
135
|
+
else {
|
|
136
|
+
terminalStatus = "UNKNOWN";
|
|
137
|
+
resultCode = process.forced
|
|
138
|
+
? "FORCED_TERMINATION"
|
|
139
|
+
: classification.kind === "NO_RESULT"
|
|
140
|
+
? NO_EXECUTION_RESULT
|
|
141
|
+
: "INVALID_EXECUTION_RESULT";
|
|
142
|
+
stateStatus = "UNKNOWN";
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
const result = classification.result;
|
|
146
|
+
const finishedAt = result?.finishedAt ?? this.now();
|
|
147
|
+
if (input.authorizeResultPersistence !== undefined &&
|
|
148
|
+
!(await input.authorizeResultPersistence())) {
|
|
149
|
+
throw new AttemptResultPersistenceDeferredError(classification, process);
|
|
150
|
+
}
|
|
151
|
+
const attempt = await controlPlane(() => this.options.repository.finalizeAttempt(startedAttempt.value.node_attempt_id, startedAttempt.revision, {
|
|
152
|
+
status: terminalStatus,
|
|
153
|
+
result_code: resultCode,
|
|
154
|
+
runner_execution_started_at: runnerStartedAt,
|
|
155
|
+
execution_id: result?.executionId ?? marker?.executionId ?? null,
|
|
156
|
+
finished_at: finishedAt,
|
|
157
|
+
duration_sec: result ? result.durationMs / 1000 : null,
|
|
158
|
+
error_message: auditSummary(input, result, classification),
|
|
159
|
+
read_count: result?.readCount ?? 0,
|
|
160
|
+
written_count: result?.writtenCount ?? 0,
|
|
161
|
+
last_successful_chunk_no: result?.lastSuccessfulChunkNo ?? null,
|
|
162
|
+
last_written_key: result?.lastWrittenKey ?? null,
|
|
163
|
+
}));
|
|
164
|
+
const waiting = stateStatus === "WAITING";
|
|
165
|
+
const nodeState = await controlPlane(() => this.options.repository.upsertNodeState({
|
|
166
|
+
expected_revision: input.nodeState.revision,
|
|
167
|
+
value: {
|
|
168
|
+
...input.nodeState.value,
|
|
169
|
+
status: stateStatus,
|
|
170
|
+
active_attempt_id: null,
|
|
171
|
+
status_reason: waiting ? "PREPARE_FAILED" : resultCode,
|
|
172
|
+
started_at: waiting ? null : input.nodeState.value.started_at,
|
|
173
|
+
finished_at: waiting ? null : finishedAt,
|
|
174
|
+
updated_at: finishedAt,
|
|
175
|
+
},
|
|
176
|
+
}));
|
|
177
|
+
return {
|
|
178
|
+
classification,
|
|
179
|
+
process,
|
|
180
|
+
attempt,
|
|
181
|
+
nodeState,
|
|
182
|
+
invocationResultCode,
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
async readMarker(attemptId, executionId) {
|
|
186
|
+
try {
|
|
187
|
+
return await this.options.jobLogReader.findExecutionStarted({
|
|
188
|
+
attemptId,
|
|
189
|
+
...(executionId === undefined ? {} : { executionId }),
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
catch (error) {
|
|
193
|
+
if (error instanceof KintoneTransportError ||
|
|
194
|
+
error instanceof KintoneApiError)
|
|
195
|
+
throw error;
|
|
196
|
+
return undefined;
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
function auditSummary(input, result, classification) {
|
|
201
|
+
const imports = input.imports ?? [];
|
|
202
|
+
const exports = input.exports ?? [];
|
|
203
|
+
if (result !== null && imports.length > 0 && exports.length > 0) {
|
|
204
|
+
return serializeIoAuditSummary(imports, (result.input_files ?? []).map((file) => ({
|
|
205
|
+
source: file.name,
|
|
206
|
+
sha256: file.sha256,
|
|
207
|
+
bytes: file.bytes,
|
|
208
|
+
rows: file.rows,
|
|
209
|
+
encoding: file.encoding,
|
|
210
|
+
})), result.output_files ?? []);
|
|
211
|
+
}
|
|
212
|
+
if (result !== null && exports.length > 0) {
|
|
213
|
+
return serializeOutputAuditSummary(result.output_files ?? []);
|
|
214
|
+
}
|
|
215
|
+
if (imports.length > 0) {
|
|
216
|
+
return result?.input_files === undefined
|
|
217
|
+
? serializeInputBaseline(imports)
|
|
218
|
+
: serializeInputAuditSummary(imports, result.input_files.map((file) => ({
|
|
219
|
+
source: file.name,
|
|
220
|
+
sha256: file.sha256,
|
|
221
|
+
bytes: file.bytes,
|
|
222
|
+
rows: file.rows,
|
|
223
|
+
encoding: file.encoding,
|
|
224
|
+
})));
|
|
225
|
+
}
|
|
226
|
+
return result?.error?.message ?? (classification.details.join("; ") || null);
|
|
227
|
+
}
|
|
228
|
+
function assertPrepared(input) {
|
|
229
|
+
if (input.attempt.value.status !== "RUNNING" ||
|
|
230
|
+
input.nodeState.value.status !== "RUNNING" ||
|
|
231
|
+
input.nodeState.value.active_attempt_id !==
|
|
232
|
+
input.attempt.value.node_attempt_id ||
|
|
233
|
+
input.attempt.value.node_attempt_id !== input.attemptId)
|
|
234
|
+
throw new Error("attempt and node state must be the same prepared RUNNING execution");
|
|
235
|
+
}
|
|
236
|
+
//# sourceMappingURL=attempt-executor.js.map
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
export interface JobLogLookup {
|
|
2
|
+
readonly attemptId: string;
|
|
3
|
+
readonly executionId?: string;
|
|
4
|
+
}
|
|
5
|
+
export interface JobLogMarker {
|
|
6
|
+
readonly runnerExecutionStartedAt: string | null;
|
|
7
|
+
readonly executionId: string | null;
|
|
8
|
+
}
|
|
9
|
+
export interface JobLogAttemptResult extends JobLogMarker {
|
|
10
|
+
readonly status: string;
|
|
11
|
+
readonly finishedAt: string | null;
|
|
12
|
+
}
|
|
13
|
+
export interface JobLogReader {
|
|
14
|
+
findExecutionStarted(lookup: JobLogLookup): Promise<JobLogMarker | null>;
|
|
15
|
+
findAttemptResult(attemptId: string): Promise<JobLogAttemptResult | null>;
|
|
16
|
+
}
|
|
17
|
+
export interface KintoneJobLogReaderOptions {
|
|
18
|
+
readonly baseUrl: string;
|
|
19
|
+
readonly appId: number;
|
|
20
|
+
readonly apiToken: string;
|
|
21
|
+
readonly fetch?: typeof fetch;
|
|
22
|
+
}
|
|
23
|
+
export declare class KintoneJobLogReader implements JobLogReader {
|
|
24
|
+
private readonly options;
|
|
25
|
+
private readonly fetchImpl;
|
|
26
|
+
constructor(options: KintoneJobLogReaderOptions);
|
|
27
|
+
findExecutionStarted(lookup: JobLogLookup): Promise<JobLogMarker | null>;
|
|
28
|
+
findAttemptResult(attemptId: string): Promise<JobLogAttemptResult | null>;
|
|
29
|
+
private findRecord;
|
|
30
|
+
}
|
|
31
|
+
export type MissingResultDecision = "NOT_EXECUTED" | "UNKNOWN";
|
|
32
|
+
export declare function decideMissingResult(input: {
|
|
33
|
+
readonly orchestratorExecutionStartedAt: string | null;
|
|
34
|
+
readonly runnerExecutionStartedAt: string | null;
|
|
35
|
+
readonly durableLaunchFailureConfirmed: boolean;
|
|
36
|
+
}): MissingResultDecision;
|
|
37
|
+
//# sourceMappingURL=job-log-reader.d.ts.map
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { KintoneApiError, KintoneTransportError, } from "../persistence/kintone/client.js";
|
|
2
|
+
export class KintoneJobLogReader {
|
|
3
|
+
options;
|
|
4
|
+
fetchImpl;
|
|
5
|
+
constructor(options) {
|
|
6
|
+
this.options = options;
|
|
7
|
+
this.fetchImpl = options.fetch ?? fetch;
|
|
8
|
+
}
|
|
9
|
+
async findExecutionStarted(lookup) {
|
|
10
|
+
const record = await this.findRecord(lookup);
|
|
11
|
+
return record === null
|
|
12
|
+
? null
|
|
13
|
+
: {
|
|
14
|
+
runnerExecutionStartedAt: fieldString(record.runner_execution_started_at),
|
|
15
|
+
executionId: fieldString(record.execution_id),
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
async findAttemptResult(attemptId) {
|
|
19
|
+
const record = await this.findRecord({ attemptId });
|
|
20
|
+
if (record === null)
|
|
21
|
+
return null;
|
|
22
|
+
const status = fieldString(record.status);
|
|
23
|
+
if (status === null)
|
|
24
|
+
throw new Error("job log status is missing");
|
|
25
|
+
return {
|
|
26
|
+
status,
|
|
27
|
+
runnerExecutionStartedAt: fieldString(record.runner_execution_started_at),
|
|
28
|
+
executionId: fieldString(record.execution_id),
|
|
29
|
+
finishedAt: fieldString(record.finished_at),
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
async findRecord(lookup) {
|
|
33
|
+
const clauses = [`attempt_id = "${escapeQuery(lookup.attemptId)}"`];
|
|
34
|
+
if (lookup.executionId)
|
|
35
|
+
clauses.push(`execution_id = "${escapeQuery(lookup.executionId)}"`);
|
|
36
|
+
const query = `${clauses.join(" and ")} order by runner_execution_started_at desc limit 2`;
|
|
37
|
+
const url = new URL(`${this.options.baseUrl.replace(/\/$/, "")}/k/v1/records.json`);
|
|
38
|
+
url.searchParams.set("app", String(this.options.appId));
|
|
39
|
+
url.searchParams.set("query", query);
|
|
40
|
+
let response;
|
|
41
|
+
try {
|
|
42
|
+
response = await this.fetchImpl(url, {
|
|
43
|
+
method: "GET",
|
|
44
|
+
headers: { "X-Cybozu-API-Token": this.options.apiToken },
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
catch (error) {
|
|
48
|
+
throw new KintoneTransportError(error);
|
|
49
|
+
}
|
|
50
|
+
const body = await response.json().catch(() => null);
|
|
51
|
+
if (!response.ok) {
|
|
52
|
+
const apiCode = isRecord(body) && typeof body.code === "string" ? body.code : null;
|
|
53
|
+
throw new KintoneApiError(response.status, apiCode, body);
|
|
54
|
+
}
|
|
55
|
+
if (!isRecord(body) || !Array.isArray(body.records))
|
|
56
|
+
throw new Error("job log GET returned an invalid response");
|
|
57
|
+
const records = body.records;
|
|
58
|
+
if (records.length === 0)
|
|
59
|
+
return null;
|
|
60
|
+
if (records.length > 1)
|
|
61
|
+
throw new Error("job log lookup is ambiguous");
|
|
62
|
+
const record = records[0];
|
|
63
|
+
if (!isRecord(record))
|
|
64
|
+
throw new Error("job log record is invalid");
|
|
65
|
+
return record;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
export function decideMissingResult(input) {
|
|
69
|
+
if (input.orchestratorExecutionStartedAt === null &&
|
|
70
|
+
input.runnerExecutionStartedAt === null)
|
|
71
|
+
return "NOT_EXECUTED";
|
|
72
|
+
if (input.orchestratorExecutionStartedAt !== null &&
|
|
73
|
+
input.runnerExecutionStartedAt === null &&
|
|
74
|
+
input.durableLaunchFailureConfirmed)
|
|
75
|
+
return "NOT_EXECUTED";
|
|
76
|
+
return "UNKNOWN";
|
|
77
|
+
}
|
|
78
|
+
function escapeQuery(value) {
|
|
79
|
+
return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
|
|
80
|
+
}
|
|
81
|
+
function isRecord(value) {
|
|
82
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
83
|
+
}
|
|
84
|
+
function fieldString(value) {
|
|
85
|
+
if (!isRecord(value))
|
|
86
|
+
return null;
|
|
87
|
+
return typeof value.value === "string" && value.value.length > 0
|
|
88
|
+
? value.value
|
|
89
|
+
: null;
|
|
90
|
+
}
|
|
91
|
+
//# sourceMappingURL=job-log-reader.js.map
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
export interface CapabilitiesResult {
|
|
2
|
+
readonly formatVersion: 1;
|
|
3
|
+
readonly kind: "CAPABILITIES";
|
|
4
|
+
readonly ksqlFlowVersion: string;
|
|
5
|
+
readonly engineVersion: string;
|
|
6
|
+
readonly executionContracts: readonly string[];
|
|
7
|
+
readonly resultSchema: {
|
|
8
|
+
readonly $id: string;
|
|
9
|
+
readonly contract: string;
|
|
10
|
+
};
|
|
11
|
+
readonly features: Readonly<Record<string, boolean | string>>;
|
|
12
|
+
}
|
|
13
|
+
export interface ProfileDescription {
|
|
14
|
+
readonly formatVersion: 1;
|
|
15
|
+
readonly kind: "PROFILE_DESCRIPTION";
|
|
16
|
+
readonly profile: string;
|
|
17
|
+
readonly baseUrl: string;
|
|
18
|
+
readonly guestSpaceId: number | null;
|
|
19
|
+
readonly timezone: string | null;
|
|
20
|
+
readonly apps: Readonly<Record<string, number>>;
|
|
21
|
+
readonly logApp: {
|
|
22
|
+
readonly name: string;
|
|
23
|
+
readonly appId: number;
|
|
24
|
+
} | null;
|
|
25
|
+
readonly limits: Readonly<Record<string, number | null>>;
|
|
26
|
+
readonly retry: {
|
|
27
|
+
readonly initialDelayMs: number;
|
|
28
|
+
readonly maxAttempts: number;
|
|
29
|
+
readonly maxDelayMs: number;
|
|
30
|
+
readonly respectRetryAfter: boolean;
|
|
31
|
+
readonly [key: string]: unknown;
|
|
32
|
+
};
|
|
33
|
+
readonly httpTimeoutMs: number;
|
|
34
|
+
}
|
|
35
|
+
export interface InspectionDiagnostic {
|
|
36
|
+
readonly code: string;
|
|
37
|
+
readonly severity: string;
|
|
38
|
+
readonly line: number;
|
|
39
|
+
readonly column: number;
|
|
40
|
+
readonly message: string;
|
|
41
|
+
}
|
|
42
|
+
export interface JobInspection {
|
|
43
|
+
readonly formatVersion: 1;
|
|
44
|
+
readonly kind: "JOB_INSPECTION";
|
|
45
|
+
readonly jobId: string;
|
|
46
|
+
readonly fileName: string;
|
|
47
|
+
readonly dialect: number;
|
|
48
|
+
readonly statementCount: number;
|
|
49
|
+
readonly dependsOn: readonly string[];
|
|
50
|
+
readonly timeoutSec: number | null;
|
|
51
|
+
readonly diagnostics: readonly InspectionDiagnostic[];
|
|
52
|
+
readonly nondeterministicElements: readonly InspectionDiagnostic[];
|
|
53
|
+
}
|
|
54
|
+
export type KsqlFlowCliErrorCode = "KSQL_FLOW_PROCESS_FAILED" | "KSQL_FLOW_INVALID_JSON" | "KSQL_FLOW_EXIT_MISMATCH" | "KSQL_FLOW_OUTPUT_INVALID";
|
|
55
|
+
export declare class KsqlFlowCliError extends Error {
|
|
56
|
+
readonly code: KsqlFlowCliErrorCode;
|
|
57
|
+
constructor(code: KsqlFlowCliErrorCode, message: string, options?: ErrorOptions);
|
|
58
|
+
}
|
|
59
|
+
export interface SpawnRequest {
|
|
60
|
+
readonly command: string;
|
|
61
|
+
readonly args: readonly string[];
|
|
62
|
+
}
|
|
63
|
+
export interface SpawnResult {
|
|
64
|
+
readonly exitCode: number | null;
|
|
65
|
+
readonly stdout: string;
|
|
66
|
+
readonly stderr: string;
|
|
67
|
+
}
|
|
68
|
+
export type SpawnInvoker = (request: SpawnRequest) => Promise<SpawnResult>;
|
|
69
|
+
export interface KsqlFlowCliOptions {
|
|
70
|
+
readonly command: string;
|
|
71
|
+
readonly binArgs?: readonly string[];
|
|
72
|
+
readonly profile: string;
|
|
73
|
+
readonly configPath: string;
|
|
74
|
+
readonly spawn?: SpawnInvoker;
|
|
75
|
+
}
|
|
76
|
+
export declare class KsqlFlowCli {
|
|
77
|
+
private readonly options;
|
|
78
|
+
private readonly invokeSpawn;
|
|
79
|
+
constructor(options: KsqlFlowCliOptions);
|
|
80
|
+
capabilities(): Promise<CapabilitiesResult>;
|
|
81
|
+
describeProfile(): Promise<ProfileDescription>;
|
|
82
|
+
inspectJob(sqlPath: string): Promise<JobInspection>;
|
|
83
|
+
private invokeJson;
|
|
84
|
+
}
|
|
85
|
+
export declare const spawnProcess: SpawnInvoker;
|
|
86
|
+
//# sourceMappingURL=ksql-flow-cli.d.ts.map
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
export class KsqlFlowCliError extends Error {
|
|
3
|
+
code;
|
|
4
|
+
constructor(code, message, options) {
|
|
5
|
+
super(message, options);
|
|
6
|
+
this.code = code;
|
|
7
|
+
this.name = "KsqlFlowCliError";
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
export class KsqlFlowCli {
|
|
11
|
+
options;
|
|
12
|
+
invokeSpawn;
|
|
13
|
+
constructor(options) {
|
|
14
|
+
this.options = options;
|
|
15
|
+
this.invokeSpawn = options.spawn ?? spawnProcess;
|
|
16
|
+
}
|
|
17
|
+
capabilities() {
|
|
18
|
+
return this.invokeJson(["capabilities", "--json"], isCapabilities, "CAPABILITIES");
|
|
19
|
+
}
|
|
20
|
+
describeProfile() {
|
|
21
|
+
return this.invokeJson([
|
|
22
|
+
"describe-profile",
|
|
23
|
+
"--profile",
|
|
24
|
+
this.options.profile,
|
|
25
|
+
"--config",
|
|
26
|
+
this.options.configPath,
|
|
27
|
+
"--json",
|
|
28
|
+
], isProfileDescription, "PROFILE_DESCRIPTION");
|
|
29
|
+
}
|
|
30
|
+
inspectJob(sqlPath) {
|
|
31
|
+
return this.invokeJson([
|
|
32
|
+
"inspect-job",
|
|
33
|
+
"-f",
|
|
34
|
+
sqlPath,
|
|
35
|
+
"--profile",
|
|
36
|
+
this.options.profile,
|
|
37
|
+
"--config",
|
|
38
|
+
this.options.configPath,
|
|
39
|
+
"--json",
|
|
40
|
+
], isJobInspection, "JOB_INSPECTION");
|
|
41
|
+
}
|
|
42
|
+
async invokeJson(args, validate, expectedKind) {
|
|
43
|
+
let result;
|
|
44
|
+
try {
|
|
45
|
+
result = await this.invokeSpawn({
|
|
46
|
+
command: this.options.command,
|
|
47
|
+
args: [...(this.options.binArgs ?? []), ...args],
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
catch (error) {
|
|
51
|
+
throw new KsqlFlowCliError("KSQL_FLOW_PROCESS_FAILED", `kSQL-Flow process could not be started for ${args[0]}`, { cause: error });
|
|
52
|
+
}
|
|
53
|
+
if (result.exitCode === null) {
|
|
54
|
+
throw new KsqlFlowCliError("KSQL_FLOW_PROCESS_FAILED", `kSQL-Flow process ended without an exit code for ${args[0]}`);
|
|
55
|
+
}
|
|
56
|
+
if (result.exitCode !== 0) {
|
|
57
|
+
throw new KsqlFlowCliError("KSQL_FLOW_EXIT_MISMATCH", `kSQL-Flow ${args[0]} exited with ${result.exitCode}; expected 0`);
|
|
58
|
+
}
|
|
59
|
+
let value;
|
|
60
|
+
try {
|
|
61
|
+
value = JSON.parse(result.stdout);
|
|
62
|
+
}
|
|
63
|
+
catch (error) {
|
|
64
|
+
throw new KsqlFlowCliError("KSQL_FLOW_INVALID_JSON", `kSQL-Flow ${args[0]} returned invalid JSON`, { cause: error });
|
|
65
|
+
}
|
|
66
|
+
if (!validate(value)) {
|
|
67
|
+
throw new KsqlFlowCliError("KSQL_FLOW_OUTPUT_INVALID", `kSQL-Flow ${args[0]} output does not match ${expectedKind} formatVersion 1`);
|
|
68
|
+
}
|
|
69
|
+
return value;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
export const spawnProcess = ({ command, args }) => new Promise((resolve, reject) => {
|
|
73
|
+
const child = spawn(command, args, {
|
|
74
|
+
shell: false,
|
|
75
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
76
|
+
windowsHide: true,
|
|
77
|
+
});
|
|
78
|
+
let stdout = "";
|
|
79
|
+
let stderr = "";
|
|
80
|
+
child.stdout.setEncoding("utf8");
|
|
81
|
+
child.stderr.setEncoding("utf8");
|
|
82
|
+
child.stdout.on("data", (chunk) => {
|
|
83
|
+
stdout += chunk;
|
|
84
|
+
});
|
|
85
|
+
child.stderr.on("data", (chunk) => {
|
|
86
|
+
stderr += chunk;
|
|
87
|
+
});
|
|
88
|
+
child.once("error", reject);
|
|
89
|
+
child.once("close", (exitCode) => resolve({ exitCode, stdout, stderr }));
|
|
90
|
+
});
|
|
91
|
+
function isRecord(value) {
|
|
92
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
93
|
+
}
|
|
94
|
+
function isStringArray(value) {
|
|
95
|
+
return (Array.isArray(value) && value.every((item) => typeof item === "string"));
|
|
96
|
+
}
|
|
97
|
+
function isCapabilities(value) {
|
|
98
|
+
if (!isRecord(value) ||
|
|
99
|
+
!isRecord(value.resultSchema) ||
|
|
100
|
+
!isRecord(value.features)) {
|
|
101
|
+
return false;
|
|
102
|
+
}
|
|
103
|
+
return (value.formatVersion === 1 &&
|
|
104
|
+
value.kind === "CAPABILITIES" &&
|
|
105
|
+
typeof value.ksqlFlowVersion === "string" &&
|
|
106
|
+
typeof value.engineVersion === "string" &&
|
|
107
|
+
isStringArray(value.executionContracts) &&
|
|
108
|
+
typeof value.resultSchema.$id === "string" &&
|
|
109
|
+
typeof value.resultSchema.contract === "string" &&
|
|
110
|
+
Object.values(value.features).every((feature) => typeof feature === "boolean" || typeof feature === "string"));
|
|
111
|
+
}
|
|
112
|
+
function isProfileDescription(value) {
|
|
113
|
+
if (!isRecord(value) ||
|
|
114
|
+
!isRecord(value.apps) ||
|
|
115
|
+
!isRecord(value.limits) ||
|
|
116
|
+
!isRecord(value.retry)) {
|
|
117
|
+
return false;
|
|
118
|
+
}
|
|
119
|
+
const logAppValid = value.logApp === null ||
|
|
120
|
+
(isRecord(value.logApp) &&
|
|
121
|
+
typeof value.logApp.name === "string" &&
|
|
122
|
+
Number.isSafeInteger(value.logApp.appId));
|
|
123
|
+
return (value.formatVersion === 1 &&
|
|
124
|
+
value.kind === "PROFILE_DESCRIPTION" &&
|
|
125
|
+
typeof value.profile === "string" &&
|
|
126
|
+
typeof value.baseUrl === "string" &&
|
|
127
|
+
(value.guestSpaceId === null || Number.isSafeInteger(value.guestSpaceId)) &&
|
|
128
|
+
(value.timezone === null || typeof value.timezone === "string") &&
|
|
129
|
+
Object.values(value.apps).every(Number.isSafeInteger) &&
|
|
130
|
+
logAppValid &&
|
|
131
|
+
Object.values(value.limits).every((item) => item === null || typeof item === "number") &&
|
|
132
|
+
typeof value.retry.initialDelayMs === "number" &&
|
|
133
|
+
typeof value.retry.maxAttempts === "number" &&
|
|
134
|
+
typeof value.retry.maxDelayMs === "number" &&
|
|
135
|
+
typeof value.retry.respectRetryAfter === "boolean" &&
|
|
136
|
+
typeof value.httpTimeoutMs === "number");
|
|
137
|
+
}
|
|
138
|
+
function isDiagnostic(value) {
|
|
139
|
+
return (isRecord(value) &&
|
|
140
|
+
typeof value.code === "string" &&
|
|
141
|
+
typeof value.severity === "string" &&
|
|
142
|
+
Number.isSafeInteger(value.line) &&
|
|
143
|
+
Number.isSafeInteger(value.column) &&
|
|
144
|
+
typeof value.message === "string");
|
|
145
|
+
}
|
|
146
|
+
function isJobInspection(value) {
|
|
147
|
+
return (isRecord(value) &&
|
|
148
|
+
value.formatVersion === 1 &&
|
|
149
|
+
value.kind === "JOB_INSPECTION" &&
|
|
150
|
+
typeof value.jobId === "string" &&
|
|
151
|
+
typeof value.fileName === "string" &&
|
|
152
|
+
Number.isSafeInteger(value.dialect) &&
|
|
153
|
+
Number.isSafeInteger(value.statementCount) &&
|
|
154
|
+
isStringArray(value.dependsOn) &&
|
|
155
|
+
(value.timeoutSec === null || Number.isSafeInteger(value.timeoutSec)) &&
|
|
156
|
+
Array.isArray(value.diagnostics) &&
|
|
157
|
+
value.diagnostics.every(isDiagnostic) &&
|
|
158
|
+
Array.isArray(value.nondeterministicElements) &&
|
|
159
|
+
value.nondeterministicElements.every(isDiagnostic));
|
|
160
|
+
}
|
|
161
|
+
//# sourceMappingURL=ksql-flow-cli.js.map
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import type { NetworkNodeDefinition } from "../domain/network-definition.js";
|
|
2
|
+
import type { CapabilitiesResult, JobInspection, ProfileDescription } from "./ksql-flow-cli.js";
|
|
3
|
+
export declare const EXECUTION_CONTRACT = "ksql-flow.execution/v1";
|
|
4
|
+
export declare const REQUIRED_CAPABILITY_FEATURES: readonly ["resultJson", "correlationIds", "describeProfile", "inspectJob", "durableExecutionStarted"];
|
|
5
|
+
export declare const NONDETERMINISTIC_CODES: readonly ["KSQL1306"];
|
|
6
|
+
export interface ApprovedInspectionException {
|
|
7
|
+
readonly node_id: string;
|
|
8
|
+
readonly code: (typeof NONDETERMINISTIC_CODES)[number];
|
|
9
|
+
readonly approved_by: string;
|
|
10
|
+
readonly reason: string;
|
|
11
|
+
readonly approved_at: string;
|
|
12
|
+
}
|
|
13
|
+
export interface ProfileSnapshot {
|
|
14
|
+
readonly canonicalJsonSha256: string;
|
|
15
|
+
readonly baseUrl: string;
|
|
16
|
+
readonly guestSpaceId: number | null;
|
|
17
|
+
readonly apps: Readonly<Record<string, number>>;
|
|
18
|
+
readonly timezone: string | null;
|
|
19
|
+
}
|
|
20
|
+
export interface InspectedNode {
|
|
21
|
+
readonly nodeId: string;
|
|
22
|
+
readonly jobId: string;
|
|
23
|
+
readonly inspection: JobInspection;
|
|
24
|
+
readonly nondeterministicCodes: readonly string[];
|
|
25
|
+
readonly approvedExceptions: readonly ApprovedInspectionException[];
|
|
26
|
+
}
|
|
27
|
+
export type PreflightErrorCode = "CAPABILITY_CONTRACT_MISSING" | "CAPABILITY_FEATURE_MISSING" | "PROFILE_SNAPSHOT_MISMATCH" | "JOB_ID_MISMATCH" | "NONDETERMINISTIC_IDEMPOTENT_JOB" | "INSPECTION_EXCEPTION_INVALID" | "INSPECTION_EXCEPTION_NOT_DETECTED";
|
|
28
|
+
export declare class PreflightError extends Error {
|
|
29
|
+
readonly code: PreflightErrorCode;
|
|
30
|
+
readonly details: readonly string[];
|
|
31
|
+
constructor(code: PreflightErrorCode, message: string, details?: readonly string[]);
|
|
32
|
+
}
|
|
33
|
+
export declare function validateCapabilities(capabilities: CapabilitiesResult, additionalFeatures?: readonly string[]): void;
|
|
34
|
+
/** All object keys are sorted recursively; array order remains significant. */
|
|
35
|
+
export declare function canonicalJson(value: unknown): string;
|
|
36
|
+
export declare function sha256Hex(value: Uint8Array | string): string;
|
|
37
|
+
export declare function canonicalJsonSha256(value: unknown): string;
|
|
38
|
+
export declare function profileSnapshot(description: ProfileDescription): ProfileSnapshot;
|
|
39
|
+
export declare function assertProfileSnapshot(description: ProfileDescription, expected: ProfileSnapshot): void;
|
|
40
|
+
export declare function validateJobInspections(nodes: readonly NetworkNodeDefinition[], inspections: ReadonlyMap<string, JobInspection>, exceptions?: readonly ApprovedInspectionException[]): readonly InspectedNode[];
|
|
41
|
+
//# sourceMappingURL=preflight.d.ts.map
|