@diia-inhouse/workflow 1.17.11 → 2.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/index.d.ts +1 -0
- package/dist/activities/index.js +2 -18
- package/dist/activities/proxy.d.ts +34 -0
- package/dist/activities/proxy.js +16 -24
- package/dist/activity.d.ts +2 -0
- package/dist/activity.js +2 -15
- package/dist/cli/checkWorkflowDeterminism.js +249 -275
- package/dist/cli/determinism/errorClassifier.js +56 -60
- package/dist/cli/determinism/historyFiles.js +68 -97
- package/dist/cli/determinism/index.js +7 -19
- package/dist/cli/determinism/replayExecutor.js +114 -133
- package/dist/cli/determinism/replayOptions.js +13 -22
- package/dist/cli/determinism/report.js +55 -45
- package/dist/cli/determinism/reportPrinter.js +101 -138
- package/dist/cli/index.d.ts +1 -0
- package/dist/cli/index.js +79 -119
- package/dist/cli/syncTemporalSchedules.js +74 -91
- package/dist/cli/updateTemporalSchedule.js +43 -53
- package/dist/client.d.ts +3 -0
- package/dist/client.js +3 -19
- package/dist/common.d.ts +2 -0
- package/dist/common.js +2 -13
- package/dist/encryption/crypto.d.ts +7 -0
- package/dist/encryption/crypto.js +20 -22
- package/dist/encryption/dataConverter.d.ts +7 -0
- package/dist/encryption/dataConverter.js +15 -22
- package/dist/encryption/encryptionCodec.d.ts +31 -0
- package/dist/encryption/encryptionCodec.js +108 -124
- package/dist/encryption/index.d.ts +3 -0
- package/dist/encryption/index.js +4 -20
- package/dist/index.d.ts +7 -0
- package/dist/index.js +6 -42
- package/dist/instrumentation.js +6 -10
- package/dist/interceptors/asyncLocalStorageBridge.js +29 -66
- package/dist/interceptors/traceLogAttributes.d.ts +6 -0
- package/dist/interceptors/traceLogAttributes.js +16 -54
- package/dist/interceptors.d.ts +6 -0
- package/dist/interceptors.js +6 -8
- package/dist/interfaces/config.d.ts +58 -0
- package/dist/interfaces/index.d.ts +1 -0
- package/dist/interfaces/services/schedulesExporter.d.ts +96 -0
- package/dist/interfaces/services/worker.d.ts +60 -0
- package/dist/operations.d.ts +9 -0
- package/dist/operations.js +11 -75
- package/dist/services/client.d.ts +24 -0
- package/dist/services/client.js +89 -96
- package/dist/services/schedulesExporter.d.ts +101 -0
- package/dist/services/schedulesExporter.js +456 -0
- package/dist/services/worker/identity.d.ts +4 -0
- package/dist/services/worker/identity.js +6 -9
- package/dist/services/worker.d.ts +124 -0
- package/dist/services/worker.js +324 -304
- package/dist/services/workerHealth.d.ts +15 -0
- package/dist/services/workerHealth.js +26 -35
- package/dist/testing.d.ts +42 -0
- package/dist/testing.js +43 -54
- package/dist/worker.d.ts +9 -0
- package/dist/worker.js +7 -25
- package/package.json +40 -37
- package/dist/activities/index.js.map +0 -1
- package/dist/activities/proxy.js.map +0 -1
- package/dist/activity.js.map +0 -1
- package/dist/cli/checkWorkflowDeterminism.js.map +0 -1
- package/dist/cli/determinism/errorClassifier.js.map +0 -1
- package/dist/cli/determinism/historyFiles.js.map +0 -1
- package/dist/cli/determinism/index.js.map +0 -1
- package/dist/cli/determinism/replayExecutor.js.map +0 -1
- package/dist/cli/determinism/replayOptions.js.map +0 -1
- package/dist/cli/determinism/report.js.map +0 -1
- package/dist/cli/determinism/reportPrinter.js.map +0 -1
- package/dist/cli/determinism/types.js +0 -3
- package/dist/cli/determinism/types.js.map +0 -1
- package/dist/cli/index.js.map +0 -1
- package/dist/cli/syncTemporalSchedules.js.map +0 -1
- package/dist/cli/updateTemporalSchedule.js.map +0 -1
- package/dist/client.js.map +0 -1
- package/dist/common.js.map +0 -1
- package/dist/encryption/crypto.js.map +0 -1
- package/dist/encryption/dataConverter.js.map +0 -1
- package/dist/encryption/encryptionCodec.js.map +0 -1
- package/dist/encryption/index.js.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/instrumentation.js.map +0 -1
- package/dist/interceptors/asyncLocalStorageBridge.js.map +0 -1
- package/dist/interceptors/index.js +0 -8
- package/dist/interceptors/index.js.map +0 -1
- package/dist/interceptors/traceLogAttributes.js.map +0 -1
- package/dist/interceptors.js.map +0 -1
- package/dist/interfaces/config.js +0 -3
- package/dist/interfaces/config.js.map +0 -1
- package/dist/interfaces/index.js +0 -18
- package/dist/interfaces/index.js.map +0 -1
- package/dist/interfaces/services/worker.js +0 -3
- package/dist/interfaces/services/worker.js.map +0 -1
- package/dist/operations.js.map +0 -1
- package/dist/services/client.js.map +0 -1
- package/dist/services/index.js +0 -19
- package/dist/services/index.js.map +0 -1
- package/dist/services/worker/identity.js.map +0 -1
- package/dist/services/worker/index.js +0 -18
- package/dist/services/worker/index.js.map +0 -1
- package/dist/services/worker.js.map +0 -1
- package/dist/services/workerHealth.js.map +0 -1
- package/dist/testing.js.map +0 -1
- package/dist/types/activities/index.d.ts +0 -1
- package/dist/types/activities/proxy.d.ts +0 -35
- package/dist/types/activity.d.ts +0 -1
- package/dist/types/cli/checkWorkflowDeterminism.d.ts +0 -19
- package/dist/types/cli/determinism/errorClassifier.d.ts +0 -15
- package/dist/types/cli/determinism/historyFiles.d.ts +0 -18
- package/dist/types/cli/determinism/index.d.ts +0 -10
- package/dist/types/cli/determinism/replayExecutor.d.ts +0 -9
- package/dist/types/cli/determinism/replayOptions.d.ts +0 -7
- package/dist/types/cli/determinism/report.d.ts +0 -16
- package/dist/types/cli/determinism/reportPrinter.d.ts +0 -5
- package/dist/types/cli/determinism/types.d.ts +0 -44
- package/dist/types/cli/index.d.ts +0 -2
- package/dist/types/cli/syncTemporalSchedules.d.ts +0 -12
- package/dist/types/cli/updateTemporalSchedule.d.ts +0 -9
- package/dist/types/client.d.ts +0 -2
- package/dist/types/common.d.ts +0 -1
- package/dist/types/encryption/crypto.d.ts +0 -3
- package/dist/types/encryption/dataConverter.d.ts +0 -3
- package/dist/types/encryption/encryptionCodec.d.ts +0 -27
- package/dist/types/encryption/index.d.ts +0 -3
- package/dist/types/index.d.ts +0 -3
- package/dist/types/instrumentation.d.ts +0 -2
- package/dist/types/interceptors/asyncLocalStorageBridge.d.ts +0 -21
- package/dist/types/interceptors/index.d.ts +0 -2
- package/dist/types/interceptors/traceLogAttributes.d.ts +0 -2
- package/dist/types/interceptors.d.ts +0 -2
- package/dist/types/interfaces/config.d.ts +0 -38
- package/dist/types/interfaces/index.d.ts +0 -1
- package/dist/types/interfaces/services/worker.d.ts +0 -37
- package/dist/types/operations.d.ts +0 -5
- package/dist/types/services/client.d.ts +0 -20
- package/dist/types/services/index.d.ts +0 -2
- package/dist/types/services/worker/identity.d.ts +0 -1
- package/dist/types/services/worker/index.d.ts +0 -1
- package/dist/types/services/worker.d.ts +0 -113
- package/dist/types/services/workerHealth.d.ts +0 -11
- package/dist/types/testing.d.ts +0 -42
- package/dist/types/worker.d.ts +0 -3
- package/dist/worker.js.map +0 -1
|
@@ -1,275 +1,249 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
});
|
|
251
|
-
let count = 0;
|
|
252
|
-
for await (const workflow of workflows) {
|
|
253
|
-
workflowIds.push(workflow.workflowId);
|
|
254
|
-
count++;
|
|
255
|
-
if (count >= this.maxWorkflowsPerType) {
|
|
256
|
-
break;
|
|
257
|
-
}
|
|
258
|
-
}
|
|
259
|
-
if (count > 0) {
|
|
260
|
-
typesWithWorkflows.push(workflowType);
|
|
261
|
-
}
|
|
262
|
-
}
|
|
263
|
-
this.logger.info(`Found workflows of ${typesWithWorkflows.length} relevant types: ${typesWithWorkflows.join(', ')}`);
|
|
264
|
-
return workflowIds;
|
|
265
|
-
}
|
|
266
|
-
getTaskQueue() {
|
|
267
|
-
const taskQueue = env_1.EnvService.getVar('TEMPORAL_TASK_QUEUE', 'string', '');
|
|
268
|
-
if (!taskQueue) {
|
|
269
|
-
return utils_1.utils.getServiceName();
|
|
270
|
-
}
|
|
271
|
-
return taskQueue;
|
|
272
|
-
}
|
|
273
|
-
}
|
|
274
|
-
exports.CheckWorkflowDeterminismCommand = CheckWorkflowDeterminismCommand;
|
|
275
|
-
//# sourceMappingURL=checkWorkflowDeterminism.js.map
|
|
1
|
+
import { TemporalClient } from "../services/client.js";
|
|
2
|
+
import { isNewStepsAdded } from "./determinism/errorClassifier.js";
|
|
3
|
+
import { loadHistoryEntries } from "./determinism/historyFiles.js";
|
|
4
|
+
import { DeterminismReportBuilder } from "./determinism/report.js";
|
|
5
|
+
import { buildReplayOptions, resolveWorkflowsPath } from "./determinism/replayOptions.js";
|
|
6
|
+
import { replayBatch, replaySingle } from "./determinism/replayExecutor.js";
|
|
7
|
+
import { printReport } from "./determinism/reportPrinter.js";
|
|
8
|
+
import "./determinism/index.js";
|
|
9
|
+
import { EnvService } from "@diia-inhouse/env";
|
|
10
|
+
import { Runtime } from "@temporalio/worker";
|
|
11
|
+
import { existsSync } from "node:fs";
|
|
12
|
+
import { setTimeout } from "node:timers/promises";
|
|
13
|
+
import { utils } from "@diia-inhouse/utils";
|
|
14
|
+
//#region src/cli/checkWorkflowDeterminism.ts
|
|
15
|
+
var CheckWorkflowDeterminismCommand = class {
|
|
16
|
+
logger;
|
|
17
|
+
envService;
|
|
18
|
+
maxWorkflowsPerType = 10;
|
|
19
|
+
maxRetries = 3;
|
|
20
|
+
retryDelayMs = 500;
|
|
21
|
+
replayTimeoutMs = 3e4;
|
|
22
|
+
delayBetweenWorkflows = 100;
|
|
23
|
+
constructor(logger, envService) {
|
|
24
|
+
this.logger = logger;
|
|
25
|
+
this.envService = envService;
|
|
26
|
+
}
|
|
27
|
+
async run(workflowsPath = "worker/workflows", taskQueueParam, workflowId) {
|
|
28
|
+
const startTime = Date.now();
|
|
29
|
+
Runtime.install({ logger: this.logger });
|
|
30
|
+
this.logger.info("Starting workflow determinism check", {
|
|
31
|
+
workflowsPath,
|
|
32
|
+
taskQueue: taskQueueParam,
|
|
33
|
+
workflowId
|
|
34
|
+
});
|
|
35
|
+
const workflows = await this.loadWorkflows(workflowsPath);
|
|
36
|
+
this.logger.info(`Found ${Object.keys(workflows).length} workflows`, { workflows: Object.keys(workflows) });
|
|
37
|
+
const taskQueue = taskQueueParam || this.getTaskQueue();
|
|
38
|
+
if (!taskQueue) throw new Error("Task queue is not provided");
|
|
39
|
+
const temporalConfig = {
|
|
40
|
+
address: EnvService.getVar("TEMPORAL_ADDRESS"),
|
|
41
|
+
namespace: EnvService.getVar("TEMPORAL_NAMESPACE", "string", "default"),
|
|
42
|
+
taskQueue,
|
|
43
|
+
encryptionEnabled: EnvService.getVar("TEMPORAL_ENCRYPTION_ENABLED", "boolean", false),
|
|
44
|
+
encryptionKeyId: EnvService.getVar("TEMPORAL_ENCRYPTION_KEY_ID", "string", "")
|
|
45
|
+
};
|
|
46
|
+
const client = new TemporalClient(temporalConfig, this.envService, this.logger);
|
|
47
|
+
await client.onInit();
|
|
48
|
+
try {
|
|
49
|
+
const report = await this.checkFromServer(client, workflowsPath, temporalConfig, workflowId);
|
|
50
|
+
this.logger.info(`Workflow determinism check finished! It took ${((Date.now() - startTime) / 1e3).toFixed(1)} seconds`);
|
|
51
|
+
printReport(report);
|
|
52
|
+
this.logger.info(`Workflow determinism check results: ${report.successCount} passed, ${report.failureCount} failed`);
|
|
53
|
+
if (report.failureCount > 0) {
|
|
54
|
+
this.logger.error(`Determinism check failed: ${report.failureCount} workflows have determinism issues`);
|
|
55
|
+
process.exit(1);
|
|
56
|
+
}
|
|
57
|
+
} finally {
|
|
58
|
+
await client.nativeClient.connection.close();
|
|
59
|
+
}
|
|
60
|
+
process.exit(0);
|
|
61
|
+
}
|
|
62
|
+
async runFromFiles(workflowsPath = "worker/workflows", historyDir, limit) {
|
|
63
|
+
const startTime = Date.now();
|
|
64
|
+
Runtime.install({ logger: this.logger });
|
|
65
|
+
this.logger.info("Starting workflow determinism check from local files", {
|
|
66
|
+
workflowsPath,
|
|
67
|
+
historyDir,
|
|
68
|
+
limit
|
|
69
|
+
});
|
|
70
|
+
const workflows = await this.loadWorkflows(workflowsPath);
|
|
71
|
+
this.logger.info(`Found ${Object.keys(workflows).length} workflows`, { workflows: Object.keys(workflows) });
|
|
72
|
+
const encryption = {
|
|
73
|
+
enabled: EnvService.getVar("TEMPORAL_ENCRYPTION_ENABLED", "boolean", false),
|
|
74
|
+
keyId: EnvService.getVar("TEMPORAL_ENCRYPTION_KEY_ID", "string", "")
|
|
75
|
+
};
|
|
76
|
+
const { entries, encryptedCount, runningCount } = loadHistoryEntries(historyDir, workflows, {
|
|
77
|
+
limit,
|
|
78
|
+
encryptionEnabled: encryption.enabled,
|
|
79
|
+
logger: this.logger
|
|
80
|
+
});
|
|
81
|
+
this.logger.info(`Loaded ${entries.length} valid histories${limit ? ` (limited to ${limit})` : ""}`);
|
|
82
|
+
if (runningCount > 0) this.logger.info(`Skipped ${runningCount} running workflow(s) — only completed/failed workflows are checked`);
|
|
83
|
+
if (encryptedCount > 0) this.logger.warn(`⚠️ Skipped ${encryptedCount} encrypted file(s) — set TEMPORAL_ENCRYPTION_ENABLED=true and provide encryption keys`);
|
|
84
|
+
if (entries.length === 0) {
|
|
85
|
+
this.logger.info("No history files found");
|
|
86
|
+
process.exit(0);
|
|
87
|
+
}
|
|
88
|
+
const options = await buildReplayOptions(workflowsPath, encryption, this.envService);
|
|
89
|
+
const reportBuilder = new DeterminismReportBuilder();
|
|
90
|
+
reportBuilder.setSkippedCount(encryptedCount);
|
|
91
|
+
this.logger.info(`Replaying ${entries.length} workflows...`);
|
|
92
|
+
let processed = 0;
|
|
93
|
+
try {
|
|
94
|
+
for await (const outcome of replayBatch(options, entries)) {
|
|
95
|
+
processed++;
|
|
96
|
+
switch (outcome.status) {
|
|
97
|
+
case "success":
|
|
98
|
+
reportBuilder.addSuccess(outcome.workflowId, outcome.workflowType);
|
|
99
|
+
this.logger.info(`✅ Workflow ${outcome.workflowId} (${outcome.workflowType}) is deterministic`);
|
|
100
|
+
break;
|
|
101
|
+
case "failure":
|
|
102
|
+
if (isNewStepsAdded(outcome.error)) {
|
|
103
|
+
reportBuilder.addSuccess(outcome.workflowId, outcome.workflowType);
|
|
104
|
+
reportBuilder.addWarning(outcome.error);
|
|
105
|
+
this.logger.warn(`⚠️ Workflow ${outcome.workflowId} (${outcome.workflowType}) has been modified to add new steps`);
|
|
106
|
+
} else {
|
|
107
|
+
reportBuilder.addFailure(outcome.workflowId, outcome.workflowType, outcome.error);
|
|
108
|
+
this.logger.error(`❌ Workflow ${outcome.workflowId} (${outcome.workflowType}): ${outcome.error.errorMessage}`);
|
|
109
|
+
}
|
|
110
|
+
break;
|
|
111
|
+
case "timeout":
|
|
112
|
+
reportBuilder.addTimeout(outcome.workflowId, outcome.workflowType, {
|
|
113
|
+
workflowId: outcome.workflowId,
|
|
114
|
+
errorType: "ReplayFailure",
|
|
115
|
+
errorMessage: `Replay timed out after ${outcome.timeoutMs / 1e3}s`
|
|
116
|
+
});
|
|
117
|
+
this.logger.warn(`⏰ Workflow ${outcome.workflowId} timed out`);
|
|
118
|
+
break;
|
|
119
|
+
}
|
|
120
|
+
if (processed % 50 === 0) {
|
|
121
|
+
const report = reportBuilder.build();
|
|
122
|
+
this.logger.info(`Progress: ${processed}/${entries.length} (${report.successCount} passed, ${report.failureCount} failed)`);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
} catch (err) {
|
|
126
|
+
this.logger.error(`Replay stream stopped at ${processed}/${entries.length}: ${err.message}`);
|
|
127
|
+
}
|
|
128
|
+
const report = reportBuilder.build();
|
|
129
|
+
printReport(report);
|
|
130
|
+
this.logger.info(`Determinism check from files finished in ${((Date.now() - startTime) / 1e3).toFixed(1)} seconds`);
|
|
131
|
+
this.logger.info(`Workflow determinism check results: ${report.successCount} passed, ${report.failureCount} failed`);
|
|
132
|
+
if (report.failureCount > 0) {
|
|
133
|
+
this.logger.error(`Determinism check failed: ${report.failureCount} workflows have determinism issues`);
|
|
134
|
+
process.exit(1);
|
|
135
|
+
}
|
|
136
|
+
process.exit(0);
|
|
137
|
+
}
|
|
138
|
+
async checkFromServer(client, workflowsPath, temporalConfig, specificWorkflowId) {
|
|
139
|
+
const reportBuilder = new DeterminismReportBuilder();
|
|
140
|
+
try {
|
|
141
|
+
const workflowIds = specificWorkflowId ? [specificWorkflowId] : await this.listCompletedOrFailedWorkflows(client, workflowsPath);
|
|
142
|
+
if (workflowIds.length === 0) {
|
|
143
|
+
this.logger.info("No completed or failed workflows found to check");
|
|
144
|
+
return reportBuilder.build();
|
|
145
|
+
}
|
|
146
|
+
if (!specificWorkflowId) this.logger.info(`Found ${workflowIds.length} completed or failed workflows to check (limited to max ${this.maxWorkflowsPerType} per workflow type)`);
|
|
147
|
+
const options = await buildReplayOptions(workflowsPath, {
|
|
148
|
+
enabled: temporalConfig.encryptionEnabled,
|
|
149
|
+
keyId: temporalConfig.encryptionKeyId
|
|
150
|
+
}, this.envService);
|
|
151
|
+
for (const [i, workflowId] of workflowIds.entries()) {
|
|
152
|
+
if (i > 0) await setTimeout(this.delayBetweenWorkflows);
|
|
153
|
+
await this.checkSingleWorkflow(client, options, workflowId, reportBuilder);
|
|
154
|
+
}
|
|
155
|
+
} catch (err) {
|
|
156
|
+
this.logger.error("❌ Failed to check workflow determinism", { err });
|
|
157
|
+
throw err;
|
|
158
|
+
}
|
|
159
|
+
return reportBuilder.build();
|
|
160
|
+
}
|
|
161
|
+
async checkSingleWorkflow(client, options, workflowId, reportBuilder) {
|
|
162
|
+
this.logger.info(`Checking workflow ${workflowId}`);
|
|
163
|
+
const handle = client.workflow.getHandle(workflowId);
|
|
164
|
+
const history = await handle.fetchHistory();
|
|
165
|
+
const workflowName = (await handle.describe()).type;
|
|
166
|
+
const outcome = await replaySingle(options, history, workflowId, workflowName, {
|
|
167
|
+
maxRetries: this.maxRetries,
|
|
168
|
+
retryDelayMs: this.retryDelayMs,
|
|
169
|
+
timeoutMs: this.replayTimeoutMs
|
|
170
|
+
});
|
|
171
|
+
switch (outcome.status) {
|
|
172
|
+
case "success":
|
|
173
|
+
this.logger.info(`✅ Workflow ${workflowId} is deterministic`);
|
|
174
|
+
if (outcome.recoveredOnRetry) {
|
|
175
|
+
reportBuilder.addWarning({
|
|
176
|
+
workflowId,
|
|
177
|
+
errorType: "ReplayFailure",
|
|
178
|
+
errorMessage: `Workflow failed ${outcome.failedAttempts} time(s) but recovered on retry`,
|
|
179
|
+
details: {
|
|
180
|
+
issue: "Workflow Recovered on Retry",
|
|
181
|
+
explanation: `This workflow initially failed replay but succeeded after ${outcome.failedAttempts} failed attempt(s). This may indicate transient issues or race conditions.`,
|
|
182
|
+
failedAttempts: outcome.failedAttempts,
|
|
183
|
+
originalErrors: outcome.originalErrors
|
|
184
|
+
}
|
|
185
|
+
});
|
|
186
|
+
this.logger.warn(`⚠️ Workflow ${workflowId} recovered after ${outcome.failedAttempts} failed attempt(s)`);
|
|
187
|
+
}
|
|
188
|
+
reportBuilder.addSuccess(workflowId, workflowName);
|
|
189
|
+
break;
|
|
190
|
+
case "failure":
|
|
191
|
+
if (isNewStepsAdded(outcome.error)) {
|
|
192
|
+
reportBuilder.addSuccess(workflowId, workflowName);
|
|
193
|
+
reportBuilder.addWarning(outcome.error);
|
|
194
|
+
this.logger.warn(`⚠️ Workflow ${workflowId} has been modified to add new steps`);
|
|
195
|
+
} else {
|
|
196
|
+
reportBuilder.addFailure(workflowId, workflowName, outcome.error);
|
|
197
|
+
this.logger.error(`❌ Workflow ${workflowId} has determinism issues`);
|
|
198
|
+
}
|
|
199
|
+
break;
|
|
200
|
+
case "timeout":
|
|
201
|
+
reportBuilder.addTimeout(workflowId, workflowName, {
|
|
202
|
+
workflowId,
|
|
203
|
+
errorType: "ReplayFailure",
|
|
204
|
+
errorMessage: `Replay timed out after ${outcome.timeoutMs / 1e3}s`,
|
|
205
|
+
details: {
|
|
206
|
+
issue: "Replay Timeout",
|
|
207
|
+
explanation: `Replay did not complete within ${outcome.timeoutMs / 1e3}s. This may indicate a stuck workflow.`
|
|
208
|
+
}
|
|
209
|
+
});
|
|
210
|
+
this.logger.warn(`⏰ Workflow ${workflowId} timed out after ${outcome.timeoutMs / 1e3}s — skipping`);
|
|
211
|
+
break;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
async loadWorkflows(workflowsPath) {
|
|
215
|
+
const fullWorkflowsPath = resolveWorkflowsPath(workflowsPath);
|
|
216
|
+
if (!existsSync(fullWorkflowsPath)) throw new Error(`Workflow files not found under provided path: ${fullWorkflowsPath}`);
|
|
217
|
+
const { default: _default, interceptors: _interceptors, ...workflows } = await import(fullWorkflowsPath);
|
|
218
|
+
return workflows;
|
|
219
|
+
}
|
|
220
|
+
async listCompletedOrFailedWorkflows(client, workflowsPath) {
|
|
221
|
+
const taskQueue = this.getTaskQueue() || "default";
|
|
222
|
+
const relevantWorkflows = await this.loadWorkflows(workflowsPath);
|
|
223
|
+
const relevantWorkflowTypes = Object.keys(relevantWorkflows);
|
|
224
|
+
const workflowIds = [];
|
|
225
|
+
const typesWithWorkflows = [];
|
|
226
|
+
for (const workflowType of relevantWorkflowTypes) {
|
|
227
|
+
const workflows = client.workflow.list({
|
|
228
|
+
query: `TaskQueue="${taskQueue}" AND WorkflowType="${workflowType}" AND (ExecutionStatus="Completed" OR ExecutionStatus="Failed")`,
|
|
229
|
+
pageSize: this.maxWorkflowsPerType
|
|
230
|
+
});
|
|
231
|
+
let count = 0;
|
|
232
|
+
for await (const workflow of workflows) {
|
|
233
|
+
workflowIds.push(workflow.workflowId);
|
|
234
|
+
count++;
|
|
235
|
+
if (count >= this.maxWorkflowsPerType) break;
|
|
236
|
+
}
|
|
237
|
+
if (count > 0) typesWithWorkflows.push(workflowType);
|
|
238
|
+
}
|
|
239
|
+
this.logger.info(`Found workflows of ${typesWithWorkflows.length} relevant types: ${typesWithWorkflows.join(", ")}`);
|
|
240
|
+
return workflowIds;
|
|
241
|
+
}
|
|
242
|
+
getTaskQueue() {
|
|
243
|
+
const taskQueue = EnvService.getVar("TEMPORAL_TASK_QUEUE", "string", "");
|
|
244
|
+
if (!taskQueue) return utils.getServiceName();
|
|
245
|
+
return taskQueue;
|
|
246
|
+
}
|
|
247
|
+
};
|
|
248
|
+
//#endregion
|
|
249
|
+
export { CheckWorkflowDeterminismCommand };
|