@diia-inhouse/workflow 1.17.11 → 2.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/index.d.ts +1 -0
- package/dist/activities/index.js +2 -18
- package/dist/activities/proxy.d.ts +34 -0
- package/dist/activities/proxy.js +16 -24
- package/dist/activity.d.ts +2 -0
- package/dist/activity.js +2 -15
- package/dist/cli/checkWorkflowDeterminism.js +249 -275
- package/dist/cli/determinism/errorClassifier.js +56 -60
- package/dist/cli/determinism/historyFiles.js +68 -97
- package/dist/cli/determinism/index.js +7 -19
- package/dist/cli/determinism/replayExecutor.js +114 -133
- package/dist/cli/determinism/replayOptions.js +13 -22
- package/dist/cli/determinism/report.js +55 -45
- package/dist/cli/determinism/reportPrinter.js +101 -138
- package/dist/cli/index.d.ts +1 -0
- package/dist/cli/index.js +79 -119
- package/dist/cli/syncTemporalSchedules.js +74 -91
- package/dist/cli/updateTemporalSchedule.js +43 -53
- package/dist/client.d.ts +3 -0
- package/dist/client.js +3 -19
- package/dist/common.d.ts +2 -0
- package/dist/common.js +2 -13
- package/dist/encryption/crypto.d.ts +7 -0
- package/dist/encryption/crypto.js +20 -22
- package/dist/encryption/dataConverter.d.ts +7 -0
- package/dist/encryption/dataConverter.js +15 -22
- package/dist/encryption/encryptionCodec.d.ts +31 -0
- package/dist/encryption/encryptionCodec.js +108 -124
- package/dist/encryption/index.d.ts +3 -0
- package/dist/encryption/index.js +4 -20
- package/dist/index.d.ts +7 -0
- package/dist/index.js +6 -42
- package/dist/instrumentation.js +6 -10
- package/dist/interceptors/asyncLocalStorageBridge.js +29 -66
- package/dist/interceptors/traceLogAttributes.d.ts +6 -0
- package/dist/interceptors/traceLogAttributes.js +16 -54
- package/dist/interceptors.d.ts +6 -0
- package/dist/interceptors.js +6 -8
- package/dist/interfaces/config.d.ts +58 -0
- package/dist/interfaces/index.d.ts +1 -0
- package/dist/interfaces/services/schedulesExporter.d.ts +96 -0
- package/dist/interfaces/services/worker.d.ts +60 -0
- package/dist/operations.d.ts +9 -0
- package/dist/operations.js +11 -75
- package/dist/services/client.d.ts +24 -0
- package/dist/services/client.js +89 -96
- package/dist/services/schedulesExporter.d.ts +101 -0
- package/dist/services/schedulesExporter.js +456 -0
- package/dist/services/worker/identity.d.ts +4 -0
- package/dist/services/worker/identity.js +6 -9
- package/dist/services/worker.d.ts +124 -0
- package/dist/services/worker.js +324 -304
- package/dist/services/workerHealth.d.ts +15 -0
- package/dist/services/workerHealth.js +26 -35
- package/dist/testing.d.ts +42 -0
- package/dist/testing.js +43 -54
- package/dist/worker.d.ts +9 -0
- package/dist/worker.js +7 -25
- package/package.json +40 -37
- package/dist/activities/index.js.map +0 -1
- package/dist/activities/proxy.js.map +0 -1
- package/dist/activity.js.map +0 -1
- package/dist/cli/checkWorkflowDeterminism.js.map +0 -1
- package/dist/cli/determinism/errorClassifier.js.map +0 -1
- package/dist/cli/determinism/historyFiles.js.map +0 -1
- package/dist/cli/determinism/index.js.map +0 -1
- package/dist/cli/determinism/replayExecutor.js.map +0 -1
- package/dist/cli/determinism/replayOptions.js.map +0 -1
- package/dist/cli/determinism/report.js.map +0 -1
- package/dist/cli/determinism/reportPrinter.js.map +0 -1
- package/dist/cli/determinism/types.js +0 -3
- package/dist/cli/determinism/types.js.map +0 -1
- package/dist/cli/index.js.map +0 -1
- package/dist/cli/syncTemporalSchedules.js.map +0 -1
- package/dist/cli/updateTemporalSchedule.js.map +0 -1
- package/dist/client.js.map +0 -1
- package/dist/common.js.map +0 -1
- package/dist/encryption/crypto.js.map +0 -1
- package/dist/encryption/dataConverter.js.map +0 -1
- package/dist/encryption/encryptionCodec.js.map +0 -1
- package/dist/encryption/index.js.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/instrumentation.js.map +0 -1
- package/dist/interceptors/asyncLocalStorageBridge.js.map +0 -1
- package/dist/interceptors/index.js +0 -8
- package/dist/interceptors/index.js.map +0 -1
- package/dist/interceptors/traceLogAttributes.js.map +0 -1
- package/dist/interceptors.js.map +0 -1
- package/dist/interfaces/config.js +0 -3
- package/dist/interfaces/config.js.map +0 -1
- package/dist/interfaces/index.js +0 -18
- package/dist/interfaces/index.js.map +0 -1
- package/dist/interfaces/services/worker.js +0 -3
- package/dist/interfaces/services/worker.js.map +0 -1
- package/dist/operations.js.map +0 -1
- package/dist/services/client.js.map +0 -1
- package/dist/services/index.js +0 -19
- package/dist/services/index.js.map +0 -1
- package/dist/services/worker/identity.js.map +0 -1
- package/dist/services/worker/index.js +0 -18
- package/dist/services/worker/index.js.map +0 -1
- package/dist/services/worker.js.map +0 -1
- package/dist/services/workerHealth.js.map +0 -1
- package/dist/testing.js.map +0 -1
- package/dist/types/activities/index.d.ts +0 -1
- package/dist/types/activities/proxy.d.ts +0 -35
- package/dist/types/activity.d.ts +0 -1
- package/dist/types/cli/checkWorkflowDeterminism.d.ts +0 -19
- package/dist/types/cli/determinism/errorClassifier.d.ts +0 -15
- package/dist/types/cli/determinism/historyFiles.d.ts +0 -18
- package/dist/types/cli/determinism/index.d.ts +0 -10
- package/dist/types/cli/determinism/replayExecutor.d.ts +0 -9
- package/dist/types/cli/determinism/replayOptions.d.ts +0 -7
- package/dist/types/cli/determinism/report.d.ts +0 -16
- package/dist/types/cli/determinism/reportPrinter.d.ts +0 -5
- package/dist/types/cli/determinism/types.d.ts +0 -44
- package/dist/types/cli/index.d.ts +0 -2
- package/dist/types/cli/syncTemporalSchedules.d.ts +0 -12
- package/dist/types/cli/updateTemporalSchedule.d.ts +0 -9
- package/dist/types/client.d.ts +0 -2
- package/dist/types/common.d.ts +0 -1
- package/dist/types/encryption/crypto.d.ts +0 -3
- package/dist/types/encryption/dataConverter.d.ts +0 -3
- package/dist/types/encryption/encryptionCodec.d.ts +0 -27
- package/dist/types/encryption/index.d.ts +0 -3
- package/dist/types/index.d.ts +0 -3
- package/dist/types/instrumentation.d.ts +0 -2
- package/dist/types/interceptors/asyncLocalStorageBridge.d.ts +0 -21
- package/dist/types/interceptors/index.d.ts +0 -2
- package/dist/types/interceptors/traceLogAttributes.d.ts +0 -2
- package/dist/types/interceptors.d.ts +0 -2
- package/dist/types/interfaces/config.d.ts +0 -38
- package/dist/types/interfaces/index.d.ts +0 -1
- package/dist/types/interfaces/services/worker.d.ts +0 -37
- package/dist/types/operations.d.ts +0 -5
- package/dist/types/services/client.d.ts +0 -20
- package/dist/types/services/index.d.ts +0 -2
- package/dist/types/services/worker/identity.d.ts +0 -1
- package/dist/types/services/worker/index.d.ts +0 -1
- package/dist/types/services/worker.d.ts +0 -113
- package/dist/types/services/workerHealth.d.ts +0 -11
- package/dist/types/testing.d.ts +0 -42
- package/dist/types/worker.d.ts +0 -3
- package/dist/worker.js.map +0 -1
|
@@ -0,0 +1,456 @@
|
|
|
1
|
+
import * as promClient from "prom-client";
|
|
2
|
+
//#region src/services/schedulesExporter.ts
|
|
3
|
+
const SCHEDULE_LABELS = [
|
|
4
|
+
"schedule_id",
|
|
5
|
+
"workflow_type",
|
|
6
|
+
"task_queue"
|
|
7
|
+
];
|
|
8
|
+
const SCHEDULE_SLOT_LABELS = [...SCHEDULE_LABELS, "slot"];
|
|
9
|
+
const RUNNING_LABELS = ["workflow_type", "task_queue"];
|
|
10
|
+
/**
|
|
11
|
+
* Periodically polls the Temporal Schedule and Visibility APIs, exposing per-schedule and
|
|
12
|
+
* per-workflow-type business metrics that the SDK does not provide out of the box:
|
|
13
|
+
*
|
|
14
|
+
* - `diia_schedule_paused` (0/1)
|
|
15
|
+
* - `diia_schedule_last_action_age_seconds`
|
|
16
|
+
* - `diia_schedule_last_action_succeeded` (0/1)
|
|
17
|
+
* - `diia_schedule_next_action_at_seconds{slot}` — Unix seconds of upcoming fire
|
|
18
|
+
* - `diia_schedule_next_action_eta_seconds{slot}` — seconds from now until upcoming fire
|
|
19
|
+
|
|
20
|
+
* - `diia_schedule_cadence_seconds` — approximate gap between fires (slot1 − slot0, falls back to spec)
|
|
21
|
+
* - `diia_schedule_fires_total{result="ok"|"failed"}` — counter of observed schedule fires; pair
|
|
22
|
+
* with cadence to compute expected/day and detect missed runs
|
|
23
|
+
* - `diia_workflows_running` — count of executions in `Running` state for this task queue
|
|
24
|
+
* - `diia_workflows_oldest_running_age_seconds` — age of the oldest running workflow
|
|
25
|
+
*
|
|
26
|
+
* The `service` label is supplied by the `diia-metrics` default labels at scrape time, so it
|
|
27
|
+
* is intentionally absent from the metric labelNames.
|
|
28
|
+
*
|
|
29
|
+
* Schedules with `action.taskQueue` other than `deps.taskQueue` are ignored, so a service
|
|
30
|
+
* only emits metrics for schedules it actually owns. This avoids fan-out when many services
|
|
31
|
+
* share a Temporal namespace.
|
|
32
|
+
*
|
|
33
|
+
* The exporter does NOT run its own HTTP server. Metrics flow through the existing
|
|
34
|
+
* `diia-metrics` `/metrics` endpoint. If a service needs to expose the calendar event list
|
|
35
|
+
* or recent action feed as JSON (e.g. for a Grafana Infinity datasource), it should call
|
|
36
|
+
* {@link SchedulesExporter.getCalendarEvents} and {@link SchedulesExporter.getRecentActions}
|
|
37
|
+
* from a route on its existing HTTP framework.
|
|
38
|
+
*/
|
|
39
|
+
var SchedulesExporter = class SchedulesExporter {
|
|
40
|
+
deps;
|
|
41
|
+
pollIntervalMs;
|
|
42
|
+
visibilityPollIntervalMs;
|
|
43
|
+
pollVisibility;
|
|
44
|
+
nextActionSlots;
|
|
45
|
+
historySize;
|
|
46
|
+
logger;
|
|
47
|
+
calendarEvents = [];
|
|
48
|
+
recentActionsLog = [];
|
|
49
|
+
paused;
|
|
50
|
+
lastActionAge;
|
|
51
|
+
lastActionSucceeded;
|
|
52
|
+
nextActionAt;
|
|
53
|
+
nextActionEta;
|
|
54
|
+
cadence;
|
|
55
|
+
running;
|
|
56
|
+
oldestRunningAge;
|
|
57
|
+
firesTotal;
|
|
58
|
+
workflowDuration;
|
|
59
|
+
seenCompletedRunIds = [];
|
|
60
|
+
seenCompletedRunIdSet = /* @__PURE__ */ new Set();
|
|
61
|
+
seenActionKeys = [];
|
|
62
|
+
seenActionKeySet = /* @__PURE__ */ new Set();
|
|
63
|
+
static ACTIONS_SEEN_CAPACITY = 2e3;
|
|
64
|
+
lastSeenScheduleLabels = /* @__PURE__ */ new Map();
|
|
65
|
+
pollCompletions;
|
|
66
|
+
completionsPollIntervalMs;
|
|
67
|
+
completionsLookbackMs;
|
|
68
|
+
completionsSeenCapacity;
|
|
69
|
+
completionsLastPollMs;
|
|
70
|
+
scheduleTimer;
|
|
71
|
+
visibilityTimer;
|
|
72
|
+
completionsTimer;
|
|
73
|
+
schedulesPolling = false;
|
|
74
|
+
runningPolling = false;
|
|
75
|
+
completionsPolling = false;
|
|
76
|
+
activeSchedulesPoll;
|
|
77
|
+
activeRunningPoll;
|
|
78
|
+
activeCompletionsPoll;
|
|
79
|
+
stopped = false;
|
|
80
|
+
constructor(deps, config = {}) {
|
|
81
|
+
this.deps = deps;
|
|
82
|
+
this.pollIntervalMs = config.pollIntervalMs ?? 3e4;
|
|
83
|
+
this.visibilityPollIntervalMs = config.visibilityPollIntervalMs ?? 1e4;
|
|
84
|
+
this.pollVisibility = config.pollVisibility ?? true;
|
|
85
|
+
this.nextActionSlots = config.nextActionSlots ?? 5;
|
|
86
|
+
this.historySize = config.recentActionsHistorySize ?? 200;
|
|
87
|
+
this.pollCompletions = config.pollCompletions ?? true;
|
|
88
|
+
this.completionsPollIntervalMs = config.completionsPollIntervalMs ?? 6e4;
|
|
89
|
+
this.completionsLookbackMs = config.completionsLookbackMs ?? 3600 * 1e3;
|
|
90
|
+
this.completionsSeenCapacity = config.completionsSeenCapacity ?? 2e3;
|
|
91
|
+
this.completionsLastPollMs = Date.now() - this.completionsLookbackMs;
|
|
92
|
+
this.logger = deps.logger;
|
|
93
|
+
const scheduleLabels = [...SCHEDULE_LABELS];
|
|
94
|
+
const scheduleSlotLabels = [...SCHEDULE_SLOT_LABELS];
|
|
95
|
+
const runningLabels = [...RUNNING_LABELS];
|
|
96
|
+
this.paused = getOrCreateGauge("diia_schedule_paused", "1 if the schedule is paused", scheduleLabels);
|
|
97
|
+
this.lastActionAge = getOrCreateGauge("diia_schedule_last_action_age_seconds", "Seconds since the schedule last fired (across all action results)", scheduleLabels);
|
|
98
|
+
this.lastActionSucceeded = getOrCreateGauge("diia_schedule_last_action_succeeded", "1 if the most recent schedule action started a workflow successfully", scheduleLabels);
|
|
99
|
+
this.nextActionAt = getOrCreateGauge("diia_schedule_next_action_at_seconds", "Unix seconds of an upcoming scheduled action (slot=0 is the next fire)", scheduleSlotLabels);
|
|
100
|
+
this.nextActionEta = getOrCreateGauge("diia_schedule_next_action_eta_seconds", "Seconds from now until an upcoming scheduled action (slot=0 is the next fire)", scheduleSlotLabels);
|
|
101
|
+
this.cadence = getOrCreateGauge("diia_schedule_cadence_seconds", "Approximate cadence between fires, derived from the gap between the next two upcoming actions (falls back to spec for single-slot schedules)", scheduleLabels);
|
|
102
|
+
this.running = getOrCreateGauge("diia_workflows_running", "Currently running workflow executions for this task queue, by workflow type", runningLabels);
|
|
103
|
+
this.oldestRunningAge = getOrCreateGauge("diia_workflows_oldest_running_age_seconds", "Age of the oldest running workflow per workflow type", runningLabels);
|
|
104
|
+
this.firesTotal = getOrCreateCounter("diia_schedule_fires_total", "Total schedule fires observed by the exporter, partitioned by result (ok/failed). Compare with expected fires from `diia_schedule_cadence_seconds` to detect missed runs.", [...scheduleLabels, "result"]);
|
|
105
|
+
this.workflowDuration = getOrCreateHistogram("diia_workflow_duration_seconds", "End-to-end duration of completed (status=Completed) workflow executions per workflow_type. Sample is taken from Temporal Visibility close_time − start_time. Pair with `diia_schedule_fires_total` to compute per-schedule cost (fires/day × p95 duration).", ["workflow_type", "task_queue"], [
|
|
106
|
+
1,
|
|
107
|
+
5,
|
|
108
|
+
15,
|
|
109
|
+
60,
|
|
110
|
+
180,
|
|
111
|
+
600,
|
|
112
|
+
1800,
|
|
113
|
+
3600,
|
|
114
|
+
7200,
|
|
115
|
+
21600,
|
|
116
|
+
43200,
|
|
117
|
+
86400
|
|
118
|
+
]);
|
|
119
|
+
}
|
|
120
|
+
async onInit() {
|
|
121
|
+
this.runSchedulesPoll("initial");
|
|
122
|
+
if (this.pollVisibility) this.runRunningPoll("initial");
|
|
123
|
+
if (this.pollCompletions) this.runCompletionsPoll("initial");
|
|
124
|
+
this.armSchedulesTimer();
|
|
125
|
+
if (this.pollVisibility) this.armVisibilityTimer();
|
|
126
|
+
if (this.pollCompletions) this.armCompletionsTimer();
|
|
127
|
+
}
|
|
128
|
+
async onDestroy() {
|
|
129
|
+
this.stopped = true;
|
|
130
|
+
if (this.scheduleTimer) clearTimeout(this.scheduleTimer);
|
|
131
|
+
if (this.visibilityTimer) clearTimeout(this.visibilityTimer);
|
|
132
|
+
if (this.completionsTimer) clearTimeout(this.completionsTimer);
|
|
133
|
+
await Promise.allSettled([
|
|
134
|
+
this.activeSchedulesPoll,
|
|
135
|
+
this.activeRunningPoll,
|
|
136
|
+
this.activeCompletionsPoll
|
|
137
|
+
]);
|
|
138
|
+
}
|
|
139
|
+
armSchedulesTimer() {
|
|
140
|
+
if (this.stopped) return;
|
|
141
|
+
this.scheduleTimer = setTimeout(() => {
|
|
142
|
+
this.runSchedulesPoll("interval").finally(() => this.armSchedulesTimer());
|
|
143
|
+
}, this.pollIntervalMs);
|
|
144
|
+
this.scheduleTimer.unref();
|
|
145
|
+
}
|
|
146
|
+
armVisibilityTimer() {
|
|
147
|
+
if (this.stopped) return;
|
|
148
|
+
this.visibilityTimer = setTimeout(() => {
|
|
149
|
+
this.runRunningPoll("interval").finally(() => this.armVisibilityTimer());
|
|
150
|
+
}, this.visibilityPollIntervalMs);
|
|
151
|
+
this.visibilityTimer.unref();
|
|
152
|
+
}
|
|
153
|
+
armCompletionsTimer() {
|
|
154
|
+
if (this.stopped) return;
|
|
155
|
+
this.completionsTimer = setTimeout(() => {
|
|
156
|
+
this.runCompletionsPoll("interval").finally(() => this.armCompletionsTimer());
|
|
157
|
+
}, this.completionsPollIntervalMs);
|
|
158
|
+
this.completionsTimer.unref();
|
|
159
|
+
}
|
|
160
|
+
runSchedulesPoll(source) {
|
|
161
|
+
if (this.schedulesPolling) return Promise.resolve();
|
|
162
|
+
this.schedulesPolling = true;
|
|
163
|
+
const promise = (async () => {
|
|
164
|
+
try {
|
|
165
|
+
await this.pollSchedules();
|
|
166
|
+
} catch (err) {
|
|
167
|
+
this.logger?.error(`SchedulesExporter ${source} schedule poll failed`, { err });
|
|
168
|
+
} finally {
|
|
169
|
+
this.schedulesPolling = false;
|
|
170
|
+
this.activeSchedulesPoll = void 0;
|
|
171
|
+
}
|
|
172
|
+
})();
|
|
173
|
+
this.activeSchedulesPoll = promise;
|
|
174
|
+
return promise;
|
|
175
|
+
}
|
|
176
|
+
runRunningPoll(source) {
|
|
177
|
+
if (this.runningPolling) return Promise.resolve();
|
|
178
|
+
this.runningPolling = true;
|
|
179
|
+
const promise = (async () => {
|
|
180
|
+
try {
|
|
181
|
+
await this.pollRunning();
|
|
182
|
+
} catch (err) {
|
|
183
|
+
this.logger?.error(`SchedulesExporter ${source} visibility poll failed`, { err });
|
|
184
|
+
} finally {
|
|
185
|
+
this.runningPolling = false;
|
|
186
|
+
this.activeRunningPoll = void 0;
|
|
187
|
+
}
|
|
188
|
+
})();
|
|
189
|
+
this.activeRunningPoll = promise;
|
|
190
|
+
return promise;
|
|
191
|
+
}
|
|
192
|
+
runCompletionsPoll(source) {
|
|
193
|
+
if (this.completionsPolling) return Promise.resolve();
|
|
194
|
+
this.completionsPolling = true;
|
|
195
|
+
const promise = (async () => {
|
|
196
|
+
try {
|
|
197
|
+
await this.pollCompleted();
|
|
198
|
+
} catch (err) {
|
|
199
|
+
this.logger?.error(`SchedulesExporter ${source} completions poll failed`, { err });
|
|
200
|
+
} finally {
|
|
201
|
+
this.completionsPolling = false;
|
|
202
|
+
this.activeCompletionsPoll = void 0;
|
|
203
|
+
}
|
|
204
|
+
})();
|
|
205
|
+
this.activeCompletionsPoll = promise;
|
|
206
|
+
return promise;
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* Snapshot of the next-N upcoming fires across all tracked schedules.
|
|
210
|
+
* Each schedule contributes up to `nextActionSlots` events, ordered by `slot`.
|
|
211
|
+
* Mount on a service HTTP route to feed the Grafana Business Calendar panel via the
|
|
212
|
+
* Infinity datasource — the data shape is already calendar-event-shaped.
|
|
213
|
+
*/
|
|
214
|
+
getCalendarEvents() {
|
|
215
|
+
return this.calendarEvents;
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* Snapshot of the most recent schedule actions across all tracked schedules,
|
|
219
|
+
* ordered newest first. Capped at `recentActionsHistorySize` entries.
|
|
220
|
+
*/
|
|
221
|
+
getRecentActions(limit = 50) {
|
|
222
|
+
return this.recentActionsLog.slice(0, limit).map(({ key: _key, ...rest }) => rest);
|
|
223
|
+
}
|
|
224
|
+
async pollSchedules() {
|
|
225
|
+
const events = [];
|
|
226
|
+
const currentSeen = /* @__PURE__ */ new Map();
|
|
227
|
+
for await (const summary of this.deps.client.schedule.list()) {
|
|
228
|
+
const description = await this.deps.client.schedule.getHandle(summary.scheduleId).describe();
|
|
229
|
+
const action = description.action;
|
|
230
|
+
if (action.type !== "startWorkflow") continue;
|
|
231
|
+
if (action.taskQueue !== this.deps.taskQueue) continue;
|
|
232
|
+
const workflowType = action.workflowType;
|
|
233
|
+
const baseLabels = {
|
|
234
|
+
schedule_id: summary.scheduleId,
|
|
235
|
+
workflow_type: workflowType,
|
|
236
|
+
task_queue: this.deps.taskQueue
|
|
237
|
+
};
|
|
238
|
+
currentSeen.set(summary.scheduleId, baseLabels);
|
|
239
|
+
const info = description.info;
|
|
240
|
+
const state = description.state;
|
|
241
|
+
const recentActions = info.recentActions ?? [];
|
|
242
|
+
const upcoming = info.nextActionTimes ?? info.futureActionTimes ?? [];
|
|
243
|
+
const pausedFlag = state.paused ? 1 : 0;
|
|
244
|
+
const cadenceLabel = describeCadence(description.spec);
|
|
245
|
+
this.paused.set(baseLabels, pausedFlag);
|
|
246
|
+
const lastAction = recentActions[recentActions.length - 1];
|
|
247
|
+
const lastSucceeded = lastAction ? lastAction.action ? 1 : 0 : null;
|
|
248
|
+
if (lastAction) {
|
|
249
|
+
const at = lastAction.takenAt ?? lastAction.scheduledAt;
|
|
250
|
+
if (at) this.lastActionAge.set(baseLabels, (Date.now() - new Date(at).getTime()) / 1e3);
|
|
251
|
+
this.lastActionSucceeded.set(baseLabels, lastAction.action ? 1 : 0);
|
|
252
|
+
}
|
|
253
|
+
for (const recent of recentActions) {
|
|
254
|
+
const at = recent.takenAt ?? recent.scheduledAt;
|
|
255
|
+
if (!at) continue;
|
|
256
|
+
const key = `${summary.scheduleId}|${new Date(at).toISOString()}`;
|
|
257
|
+
if (this.seenActionKeySet.has(key)) continue;
|
|
258
|
+
this.seenActionKeySet.add(key);
|
|
259
|
+
this.seenActionKeys.push(key);
|
|
260
|
+
const succeeded = recent.action ? 1 : 0;
|
|
261
|
+
this.recentActionsLog.push({
|
|
262
|
+
key,
|
|
263
|
+
scheduleId: summary.scheduleId,
|
|
264
|
+
workflowType,
|
|
265
|
+
taskQueue: this.deps.taskQueue,
|
|
266
|
+
firedAt: new Date(at).toISOString(),
|
|
267
|
+
firedAtMs: new Date(at).getTime(),
|
|
268
|
+
succeeded
|
|
269
|
+
});
|
|
270
|
+
this.firesTotal.inc({
|
|
271
|
+
...baseLabels,
|
|
272
|
+
result: succeeded ? "ok" : "failed"
|
|
273
|
+
});
|
|
274
|
+
}
|
|
275
|
+
for (let slot = 0; slot < this.nextActionSlots; slot++) {
|
|
276
|
+
const at = upcoming[slot];
|
|
277
|
+
const slotLabels = {
|
|
278
|
+
...baseLabels,
|
|
279
|
+
slot: String(slot)
|
|
280
|
+
};
|
|
281
|
+
if (!at) {
|
|
282
|
+
this.nextActionAt.remove(slotLabels);
|
|
283
|
+
this.nextActionEta.remove(slotLabels);
|
|
284
|
+
continue;
|
|
285
|
+
}
|
|
286
|
+
const ms = new Date(at).getTime();
|
|
287
|
+
this.nextActionAt.set(slotLabels, ms / 1e3);
|
|
288
|
+
this.nextActionEta.set(slotLabels, Math.max(0, (ms - Date.now()) / 1e3));
|
|
289
|
+
events.push({
|
|
290
|
+
scheduleId: summary.scheduleId,
|
|
291
|
+
workflowType,
|
|
292
|
+
taskQueue: this.deps.taskQueue,
|
|
293
|
+
cadence: cadenceLabel,
|
|
294
|
+
slot,
|
|
295
|
+
fireAt: new Date(ms).toISOString(),
|
|
296
|
+
fireAtMs: ms,
|
|
297
|
+
paused: pausedFlag,
|
|
298
|
+
lastSucceeded
|
|
299
|
+
});
|
|
300
|
+
}
|
|
301
|
+
const cadenceSeconds = computeCadenceSeconds(upcoming, description.spec);
|
|
302
|
+
if (cadenceSeconds !== void 0) this.cadence.set(baseLabels, cadenceSeconds);
|
|
303
|
+
else this.cadence.remove(baseLabels);
|
|
304
|
+
}
|
|
305
|
+
this.calendarEvents.length = 0;
|
|
306
|
+
this.calendarEvents.push(...events);
|
|
307
|
+
this.recentActionsLog.sort((a, b) => b.firedAtMs - a.firedAtMs);
|
|
308
|
+
if (this.recentActionsLog.length > this.historySize) this.recentActionsLog.length = this.historySize;
|
|
309
|
+
while (this.seenActionKeys.length > SchedulesExporter.ACTIONS_SEEN_CAPACITY) {
|
|
310
|
+
const evicted = this.seenActionKeys.shift();
|
|
311
|
+
if (evicted) this.seenActionKeySet.delete(evicted);
|
|
312
|
+
}
|
|
313
|
+
for (const [scheduleId, labels] of this.lastSeenScheduleLabels) {
|
|
314
|
+
if (currentSeen.has(scheduleId)) continue;
|
|
315
|
+
this.paused.remove(labels);
|
|
316
|
+
this.lastActionAge.remove(labels);
|
|
317
|
+
this.lastActionSucceeded.remove(labels);
|
|
318
|
+
this.cadence.remove(labels);
|
|
319
|
+
this.firesTotal.remove({
|
|
320
|
+
...labels,
|
|
321
|
+
result: "ok"
|
|
322
|
+
});
|
|
323
|
+
this.firesTotal.remove({
|
|
324
|
+
...labels,
|
|
325
|
+
result: "failed"
|
|
326
|
+
});
|
|
327
|
+
for (let slot = 0; slot < this.nextActionSlots; slot++) {
|
|
328
|
+
const slotLabels = {
|
|
329
|
+
...labels,
|
|
330
|
+
slot: String(slot)
|
|
331
|
+
};
|
|
332
|
+
this.nextActionAt.remove(slotLabels);
|
|
333
|
+
this.nextActionEta.remove(slotLabels);
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
this.lastSeenScheduleLabels.clear();
|
|
337
|
+
for (const [scheduleId, labels] of currentSeen) this.lastSeenScheduleLabels.set(scheduleId, labels);
|
|
338
|
+
}
|
|
339
|
+
async pollRunning() {
|
|
340
|
+
const buckets = /* @__PURE__ */ new Map();
|
|
341
|
+
const query = `TaskQueue="${this.deps.taskQueue}" AND ExecutionStatus="Running"`;
|
|
342
|
+
for await (const wf of this.deps.client.workflow.list({ query })) {
|
|
343
|
+
const startMs = wf.startTime ? new Date(wf.startTime).getTime() : Date.now();
|
|
344
|
+
const existing = buckets.get(wf.type) ?? {
|
|
345
|
+
count: 0,
|
|
346
|
+
oldestStartMs: Date.now()
|
|
347
|
+
};
|
|
348
|
+
existing.count += 1;
|
|
349
|
+
if (startMs < existing.oldestStartMs) existing.oldestStartMs = startMs;
|
|
350
|
+
buckets.set(wf.type, existing);
|
|
351
|
+
}
|
|
352
|
+
this.running.reset();
|
|
353
|
+
this.oldestRunningAge.reset();
|
|
354
|
+
for (const [workflowType, { count, oldestStartMs }] of buckets) {
|
|
355
|
+
const labels = {
|
|
356
|
+
workflow_type: workflowType,
|
|
357
|
+
task_queue: this.deps.taskQueue
|
|
358
|
+
};
|
|
359
|
+
this.running.set(labels, count);
|
|
360
|
+
this.oldestRunningAge.set(labels, (Date.now() - oldestStartMs) / 1e3);
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
async pollCompleted() {
|
|
364
|
+
const sinceMs = this.completionsLastPollMs;
|
|
365
|
+
const nowMs = Date.now();
|
|
366
|
+
const sinceIso = new Date(sinceMs).toISOString();
|
|
367
|
+
const query = `TaskQueue="${this.deps.taskQueue}" AND ExecutionStatus="Completed" AND CloseTime>="${sinceIso}"`;
|
|
368
|
+
let observed = 0;
|
|
369
|
+
for await (const wf of this.deps.client.workflow.list({ query })) {
|
|
370
|
+
const raw = wf.raw;
|
|
371
|
+
const rawJson = raw && typeof raw.toJSON === "function" ? raw.toJSON() : raw;
|
|
372
|
+
const runId = rawJson?.execution?.runId;
|
|
373
|
+
const workflowType = rawJson?.type?.name;
|
|
374
|
+
const startTimeRaw = rawJson?.startTime ?? (wf.startTime ? new Date(wf.startTime).toISOString() : void 0);
|
|
375
|
+
const closeTimeRaw = rawJson?.closeTime ?? (wf.closeTime ? new Date(wf.closeTime).toISOString() : void 0);
|
|
376
|
+
if (!runId || !workflowType || !startTimeRaw || !closeTimeRaw) continue;
|
|
377
|
+
if (this.seenCompletedRunIdSet.has(runId)) continue;
|
|
378
|
+
const durationSec = (new Date(closeTimeRaw).getTime() - new Date(startTimeRaw).getTime()) / 1e3;
|
|
379
|
+
if (durationSec < 0) continue;
|
|
380
|
+
this.workflowDuration.observe({
|
|
381
|
+
workflow_type: workflowType,
|
|
382
|
+
task_queue: this.deps.taskQueue
|
|
383
|
+
}, durationSec);
|
|
384
|
+
this.seenCompletedRunIdSet.add(runId);
|
|
385
|
+
this.seenCompletedRunIds.push(runId);
|
|
386
|
+
observed++;
|
|
387
|
+
}
|
|
388
|
+
while (this.seenCompletedRunIds.length > this.completionsSeenCapacity) {
|
|
389
|
+
const evicted = this.seenCompletedRunIds.shift();
|
|
390
|
+
if (evicted) this.seenCompletedRunIdSet.delete(evicted);
|
|
391
|
+
}
|
|
392
|
+
this.completionsLastPollMs = nowMs - this.completionsPollIntervalMs;
|
|
393
|
+
if (observed > 0) this.logger?.debug("SchedulesExporter pollCompleted observed completions", {
|
|
394
|
+
taskQueue: this.deps.taskQueue,
|
|
395
|
+
observed
|
|
396
|
+
});
|
|
397
|
+
}
|
|
398
|
+
};
|
|
399
|
+
function getOrCreateGauge(name, help, labelNames) {
|
|
400
|
+
const existing = promClient.register.getSingleMetric(name);
|
|
401
|
+
if (existing instanceof promClient.Gauge) return existing;
|
|
402
|
+
return new promClient.Gauge({
|
|
403
|
+
name,
|
|
404
|
+
help,
|
|
405
|
+
labelNames
|
|
406
|
+
});
|
|
407
|
+
}
|
|
408
|
+
function getOrCreateCounter(name, help, labelNames) {
|
|
409
|
+
const existing = promClient.register.getSingleMetric(name);
|
|
410
|
+
if (existing instanceof promClient.Counter) return existing;
|
|
411
|
+
return new promClient.Counter({
|
|
412
|
+
name,
|
|
413
|
+
help,
|
|
414
|
+
labelNames
|
|
415
|
+
});
|
|
416
|
+
}
|
|
417
|
+
function getOrCreateHistogram(name, help, labelNames, buckets) {
|
|
418
|
+
const existing = promClient.register.getSingleMetric(name);
|
|
419
|
+
if (existing instanceof promClient.Histogram) return existing;
|
|
420
|
+
return new promClient.Histogram({
|
|
421
|
+
name,
|
|
422
|
+
help,
|
|
423
|
+
labelNames,
|
|
424
|
+
buckets
|
|
425
|
+
});
|
|
426
|
+
}
|
|
427
|
+
function describeCadence(spec) {
|
|
428
|
+
const s = spec;
|
|
429
|
+
if (!s) return "unknown";
|
|
430
|
+
if (s.intervals?.length) {
|
|
431
|
+
const every = s.intervals[0].every;
|
|
432
|
+
if (typeof every === "number") return `every ${Math.round(every / 1e3)}s`;
|
|
433
|
+
return `every ${every.toString()}`;
|
|
434
|
+
}
|
|
435
|
+
if (s.cronExpressions?.length) return s.cronExpressions[0];
|
|
436
|
+
if (s.calendars?.length) {
|
|
437
|
+
const calendar = s.calendars[0];
|
|
438
|
+
return `cal ${calendar.hour ?? "*"}:${String(calendar.minute ?? 0).padStart(2, "0")}`;
|
|
439
|
+
}
|
|
440
|
+
return "unknown";
|
|
441
|
+
}
|
|
442
|
+
function computeCadenceSeconds(upcoming, spec) {
|
|
443
|
+
if (upcoming.length >= 2) {
|
|
444
|
+
const gap = new Date(upcoming[1]).getTime() - new Date(upcoming[0]).getTime();
|
|
445
|
+
if (gap > 0) return gap / 1e3;
|
|
446
|
+
}
|
|
447
|
+
const s = spec;
|
|
448
|
+
if (!s) return;
|
|
449
|
+
if (s.intervals?.length) {
|
|
450
|
+
const every = s.intervals[0].every;
|
|
451
|
+
if (typeof every === "number") return every / 1e3;
|
|
452
|
+
}
|
|
453
|
+
if (s.calendars?.length) return 86400;
|
|
454
|
+
}
|
|
455
|
+
//#endregion
|
|
456
|
+
export { SchedulesExporter };
|
|
@@ -1,11 +1,8 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.buildWorkerIdentity = buildWorkerIdentity;
|
|
4
|
-
const node_os_1 = require("node:os");
|
|
1
|
+
import { hostname } from "node:os";
|
|
2
|
+
//#region src/services/worker/identity.ts
|
|
5
3
|
function buildWorkerIdentity(taskQueue, identityOverride) {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
}
|
|
9
|
-
return `${(0, node_os_1.hostname)()}-${process.pid}-${taskQueue}`;
|
|
4
|
+
if (identityOverride) return identityOverride;
|
|
5
|
+
return `${hostname()}-${process.pid}-${taskQueue}`;
|
|
10
6
|
}
|
|
11
|
-
//#
|
|
7
|
+
//#endregion
|
|
8
|
+
export { buildWorkerIdentity };
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { AppConfig } from "../interfaces/config.js";
|
|
2
|
+
import { ActivityClass, App, NodeTracerProviderLike, WorkerBootstrapOptions } from "../interfaces/services/worker.js";
|
|
3
|
+
import { WorkerHealthDetails, WorkerHealthService } from "./workerHealth.js";
|
|
4
|
+
import { buildWorkerIdentity } from "./worker/identity.js";
|
|
5
|
+
import { EnvService } from "@diia-inhouse/env";
|
|
6
|
+
import { AlsData, Logger } from "@diia-inhouse/types";
|
|
7
|
+
import { Worker, WorkerOptions } from "@temporalio/worker";
|
|
8
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
9
|
+
|
|
10
|
+
//#region src/services/worker.d.ts
|
|
11
|
+
/**
|
|
12
|
+
* Applies service process configuration overrides when the worker runs separately.
|
|
13
|
+
*
|
|
14
|
+
* When `temporal.workerInProcess` is `false`, disables `temporal` and `temporal-worker` scrapers
|
|
15
|
+
* so the main service does not scrape metrics that the worker process handles.
|
|
16
|
+
*
|
|
17
|
+
* Mutates the config object in place. Safe to call when scrapers are absent.
|
|
18
|
+
*/
|
|
19
|
+
declare function applyServiceProcessConfig(config: AppConfig): void;
|
|
20
|
+
/**
|
|
21
|
+
* Applies worker process configuration overrides.
|
|
22
|
+
*
|
|
23
|
+
* - Disables queue consumers on all rabbit connections (unless `temporal.disableQueueConsumers` is `false`)
|
|
24
|
+
* - Overrides `metrics.custom.port` with the `'temporal-worker'` scraper port and disables that scraper to prevent self-scraping
|
|
25
|
+
*
|
|
26
|
+
* Mutates the config object in place. Safe to call when queue config is absent.
|
|
27
|
+
*/
|
|
28
|
+
declare function applyWorkerProcessConfig(config: AppConfig): void;
|
|
29
|
+
/**
|
|
30
|
+
* Accepts either a filesystem path or a `file://` URL (e.g. from `import.meta.resolve`)
|
|
31
|
+
* and returns a filesystem path suitable for Temporal's worker.
|
|
32
|
+
*/
|
|
33
|
+
declare function toWorkflowsPath(input: string): string;
|
|
34
|
+
declare function instantiateActivities(app: App, workerActivities: Record<string, ActivityClass>): Record<string, (...args: unknown[]) => Promise<unknown>>;
|
|
35
|
+
/**
|
|
36
|
+
* Initializes and starts Temporal worker with full dependency injection support.
|
|
37
|
+
*
|
|
38
|
+
* This is the recommended way to initialize Temporal workers. It handles:
|
|
39
|
+
* - Automatic dependency injection for activities
|
|
40
|
+
* - AsyncLocalStorage setup for distributed tracing
|
|
41
|
+
* - OpenTelemetry integration
|
|
42
|
+
* - Activity instantiation and binding
|
|
43
|
+
*
|
|
44
|
+
* @param app - App instance for DI container and config access
|
|
45
|
+
* @param options - Worker configuration options
|
|
46
|
+
* @param options.nodeTracerProvider - OpenTelemetry tracer provider
|
|
47
|
+
* @param options.workflowsPath - Path to workflows module
|
|
48
|
+
* @param options.activities - Activity classes to instantiate
|
|
49
|
+
*
|
|
50
|
+
* @example
|
|
51
|
+
* ```typescript
|
|
52
|
+
* // Define your activities
|
|
53
|
+
* const workerActivities = {
|
|
54
|
+
* userActivity: UserActivity,
|
|
55
|
+
* notificationActivity: NotificationActivity,
|
|
56
|
+
* }
|
|
57
|
+
*
|
|
58
|
+
* // Initialize and start the worker
|
|
59
|
+
* await initTemporalWorker(app, {
|
|
60
|
+
* nodeTracerProvider,
|
|
61
|
+
* workflowsPath: import.meta.resolve('./worker/workflows/index.js'),
|
|
62
|
+
* activities: workerActivities,
|
|
63
|
+
* })
|
|
64
|
+
* ```
|
|
65
|
+
*/
|
|
66
|
+
declare function initTemporalWorker(app: App, options: {
|
|
67
|
+
nodeTracerProvider: NodeTracerProviderLike;
|
|
68
|
+
workflowsPath: string;
|
|
69
|
+
activities: Record<string, ActivityClass>;
|
|
70
|
+
} & Omit<WorkerOptions, "taskQueue" | "activities" | "workflowsPath">): Promise<void>;
|
|
71
|
+
/**
|
|
72
|
+
* Bootstraps and runs Temporal worker with graceful shutdown.
|
|
73
|
+
*
|
|
74
|
+
* Handles both in-process and separate-process worker topologies:
|
|
75
|
+
*
|
|
76
|
+
* - **In-process** (`workerInProcess` is `true` or unset): initializes and runs the worker.
|
|
77
|
+
* - **Separate process** (called from a dedicated worker entry with `configFactory`/`deps`):
|
|
78
|
+
* manages the full application lifecycle: setConfig → apply worker overrides → setDeps →
|
|
79
|
+
* initialize → start → run worker.
|
|
80
|
+
* - **Service-only** (`workerInProcess` is `false`, no `configFactory`): disables temporal
|
|
81
|
+
* scrapers on the main service (worker handles them separately) and returns immediately.
|
|
82
|
+
*
|
|
83
|
+
* Automatically integrates worker health with the app's centralized health check
|
|
84
|
+
* system via `HealthCheck.addHealthCheckable()`.
|
|
85
|
+
*
|
|
86
|
+
* @param app - App instance for DI container and config access
|
|
87
|
+
* @param options - Worker bootstrap options
|
|
88
|
+
*
|
|
89
|
+
* @example
|
|
90
|
+
* ```typescript
|
|
91
|
+
* // Separate worker process with full lifecycle management
|
|
92
|
+
* const app = new Application(serviceName, nodeTracerProvider, loggerConfig)
|
|
93
|
+
*
|
|
94
|
+
* await bootstrapWorker(app, {
|
|
95
|
+
* configFactory,
|
|
96
|
+
* deps,
|
|
97
|
+
* workflowsPath: import.meta.resolve('./worker/workflows/index.js'),
|
|
98
|
+
* activities: workerActivities,
|
|
99
|
+
* nodeTracerProvider,
|
|
100
|
+
* })
|
|
101
|
+
* ```
|
|
102
|
+
*/
|
|
103
|
+
declare function bootstrapWorker(app: App, options: WorkerBootstrapOptions): Promise<void>;
|
|
104
|
+
/**
|
|
105
|
+
* Initializes Temporal worker.
|
|
106
|
+
*
|
|
107
|
+
* @param config - Application configuration
|
|
108
|
+
* @param options - Worker options including workflows path
|
|
109
|
+
* @param envService - Environment service instance
|
|
110
|
+
* @param logger - Logger instance (optional)
|
|
111
|
+
* @param nodeTracerProvider - OpenTelemetry tracer provider (optional)
|
|
112
|
+
* @param asyncLocalStorage - AsyncLocalStorage instance for tracing context (optional)
|
|
113
|
+
* @returns Configured Temporal worker
|
|
114
|
+
*/
|
|
115
|
+
declare function initWorker({
|
|
116
|
+
temporal: temporalConfig,
|
|
117
|
+
metrics: {
|
|
118
|
+
custom: metricsConfig
|
|
119
|
+
}
|
|
120
|
+
}: AppConfig, options: Omit<WorkerOptions, "taskQueue"> & {
|
|
121
|
+
taskQueue?: string;
|
|
122
|
+
}, envService: EnvService, logger?: Logger, nodeTracerProvider?: NodeTracerProviderLike, asyncLocalStorage?: AsyncLocalStorage<AlsData>): Promise<Worker>;
|
|
123
|
+
//#endregion
|
|
124
|
+
export { applyServiceProcessConfig, applyWorkerProcessConfig, bootstrapWorker, initTemporalWorker, initWorker, instantiateActivities, toWorkflowsPath };
|