fyflow-scheduler 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +577 -0
- package/LICENSE +21 -0
- package/README.md +415 -0
- package/dist/browser/index.js +2378 -0
- package/dist/node/index.js +2380 -0
- package/dist/types/core/FyflowScheduler.d.ts +279 -0
- package/dist/types/core/FyflowScheduler.d.ts.map +1 -0
- package/dist/types/core/inlineWrapper.d.ts +45 -0
- package/dist/types/core/inlineWrapper.d.ts.map +1 -0
- package/dist/types/core/threadWrapper.d.ts +59 -0
- package/dist/types/core/threadWrapper.d.ts.map +1 -0
- package/dist/types/core/workerInterface.d.ts +166 -0
- package/dist/types/core/workerInterface.d.ts.map +1 -0
- package/dist/types/core/workerManager.d.ts +141 -0
- package/dist/types/core/workerManager.d.ts.map +1 -0
- package/dist/types/core/workerWrapper.d.ts +2 -0
- package/dist/types/core/workerWrapper.d.ts.map +1 -0
- package/dist/types/core/workerWrapperUrl.bundled.d.ts +2 -0
- package/dist/types/core/workerWrapperUrl.bundled.d.ts.map +1 -0
- package/dist/types/core/workerWrapperUrl.d.ts +2 -0
- package/dist/types/core/workerWrapperUrl.d.ts.map +1 -0
- package/dist/types/groups/concurrentLimitGroup.d.ts +33 -0
- package/dist/types/groups/concurrentLimitGroup.d.ts.map +1 -0
- package/dist/types/groups/keyedRateLimitGroup.d.ts +98 -0
- package/dist/types/groups/keyedRateLimitGroup.d.ts.map +1 -0
- package/dist/types/groups/rateLimitGroup.d.ts +79 -0
- package/dist/types/groups/rateLimitGroup.d.ts.map +1 -0
- package/dist/types/groups/resourceGroup.d.ts +41 -0
- package/dist/types/groups/resourceGroup.d.ts.map +1 -0
- package/dist/types/index.d.ts +15 -0
- package/dist/types/index.d.ts.map +1 -0
- package/package.json +83 -0
|
@@ -0,0 +1,2380 @@
|
|
|
1
|
+
var __defProp = Object.defineProperty;
|
|
2
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
3
|
+
var __esm = (fn, res) => function __init() {
|
|
4
|
+
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
5
|
+
};
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
// worker:C:\Users\matij\Documents\dev\fyflow-scheduler-public\core\workerWrapper.node.ts
|
|
12
|
+
var workerWrapper_node_exports = {};
|
|
13
|
+
__export(workerWrapper_node_exports, {
|
|
14
|
+
default: () => workerWrapper_node_default
|
|
15
|
+
});
|
|
16
|
+
var url, workerWrapper_node_default;
|
|
17
|
+
var init_workerWrapper_node = __esm({
|
|
18
|
+
"worker:C:\\Users\\matij\\Documents\\dev\\fyflow-scheduler-public\\core\\workerWrapper.node.ts"() {
|
|
19
|
+
url = "data:text/javascript;base64,Ly8gY29yZS93b3JrZXJXcmFwcGVyLm5vZGUudHMKaW1wb3J0IHsgcGFyZW50UG9ydCwgaXNNYWluVGhyZWFkIH0gZnJvbSAibm9kZTp3b3JrZXJfdGhyZWFkcyI7CmlmIChpc01haW5UaHJlYWQpIHsKICB0aHJvdyBuZXcgRXJyb3IoIndvcmtlcldyYXBwZXIubm9kZS50cyBzaG91bGQgb25seSBiZSB1c2VkIGluIHdvcmtlciB0aHJlYWRzIik7Cn0KdmFyIFdvcmtlckNsYXNzID0gbnVsbDsKdmFyIHdvcmtlckluc3RhbmNlID0gbnVsbDsKdmFyIHdvcmtlcklkID0gbnVsbDsKZnVuY3Rpb24gc2VuZE1lc3NhZ2UobWVzc2FnZSkgewogIG1lc3NhZ2UudGltZXN0YW1wID0gRGF0ZS5ub3coKTsKICBwYXJlbnRQb3J0Py5wb3N0TWVzc2FnZShtZXNzYWdlKTsKfQpmdW5jdGlvbiBjcmVhdGVCYXNlV29ya2VyQ29udGV4dCgpIHsKICByZXR1cm4gewogICAgd29ya2VySWQsCiAgICB0ZXJtaW5hdGVXaXRoRXJyb3I6IChlcnJvciwgbWV0YWRhdGEpID0+IHsKICAgICAgc2VuZE1lc3NhZ2UoewogICAgICAgIHR5cGU6ICJlcnJvciIsCiAgICAgICAgdGFza0lkOiAid29ya2VyX3Rlcm1pbmF0aW9uX3JlcXVlc3QiLAogICAgICAgIGRhdGE6IHsKICAgICAgICAgIG1lc3NhZ2U6IGVycm9yLm1lc3NhZ2UsCiAgICAgICAgICBpc1dvcmtlclRlcm1pbmF0aW9uUmVxdWVzdDogdHJ1ZSwKICAgICAgICAgIG1ldGFkYXRhCiAgICAgICAgfQogICAgICB9KTsKICAgIH0KICB9Owp9CmZ1bmN0aW9uIGNyZWF0ZVdvcmtlckNvbnRleHQodGFza0lkKSB7CiAgY29uc3QgYmFzZUNvbnRleHQgPSBjcmVhdGVCYXNlV29ya2VyQ29udGV4dCgpOwogIHJldHVybiB7CiAgICAvLyBXb3JrZXItbGV2ZWwgY29udGV4dAogICAgd29ya2VySWQ6IGJhc2VDb250ZXh0LndvcmtlcklkLAogICAgdGVybWluYXRlV2l0aEVycm9yOiBiYXNlQ29udGV4dC50ZXJtaW5hdGVXaXRoRXJyb3IsCiAgICAvLyBUYXNrLWxldmVsIGNvbnRleHQKICAgIHRhc2tJZCwKICAgIHNlbmRQcm9ncmVzczogKHByb2dyZXNzLCBtZXNzYWdlLCBkZXRhaWxzKSA9PiB7CiAgICAgIGNvbnN0IHByb2dyZXNzRGF0YSA9IHsgcHJvZ3Jlc3MsIG1lc3NhZ2UsIGRldGFpbHMgfTsKICAgICAgc2VuZE1lc3NhZ2UoeyB0eXBlOiAicHJvZ3Jlc3MiLCB0YXNrSWQsIGRhdGE6IHByb2dyZXNzRGF0YSB9KTsKICAgIH0sCiAgICBzcGF3blRhc2s6IChjb25maWcpID0+IHsKICAgICAgc2VuZE1lc3NhZ2UoeyB0eXBlOiAic3Bhd25fdGFzayIsIHRhc2tJZCwgZGF0YTogY29uZmlnIH0pOwogICAgfQogIH07Cn0KcGFyZW50UG9ydD8ub24oIm1lc3NhZ2UiLCBhc3luYyAoZGF0YSkgPT4gewogIGNvbnN0IHsgdGFza0lkLCBwYXlsb2FkLCBhY3Rpb24sIGNvbmZpZywgd29ya2VyVXJsLCB3b3JrZXJJZDogcmVjZWl2ZWRXb3JrZXJJZCB9ID0gZGF0YTsKICB0cnkgewogICAgaWYgKGFjdGlvbiA9PT0gImluaXQiKSB7CiAgICAgIHdvcmtlcklkID0gcmVjZWl2ZWRXb3JrZXJJZDsKICAgICAgY29uc3Qgd29ya2VyTW9kdWxlID0gYXdhaXQgKHdvcmtlclVybC5zdGFydHNXaXRoKCJkYXRhIikgPyBpbXBvcnQobmV3IFVSTCh3b3JrZXJVcmwpKSA6IGltcG9ydCh3b3JrZXJVcmwpKTsKICAgICAgV29ya2VyQ2xhc3MgPSB3b3JrZXJNb2R1bGUuZGVmYXVsdDsKICAgICAgaWYgKCFXb3JrZXJDbGFzcyB8fCB0eXBlb2YgV29ya2VyQ2xhc3MgIT09ICJmdW5jdGlvbiIpIHsKICAgICAgICB0aHJvdyBuZXcgRXJyb3IoYFdvcmtlciBzY3JpcHQgJHt3b3JrZXJVcmx9IG11c3QgZXhwb3J0IGEgZGVmYXVsdCBjbGFzc2ApOwogICAgICB9CiAgICAgIGNvbnN0IGJhc2VXb3JrZXJDb250ZXh0ID0gY3JlYXRlQmFzZVdvcmtlckNvbnRleHQoKTsKICAgICAgd29ya2VySW5zdGFuY2UgPSBuZXcgV29ya2VyQ2xhc3MoY29uZmlnLCBiYXNlV29ya2VyQ29udGV4dCk7CiAgICAgIHNlbmRNZXNzYWdlKHsgdHlwZTogInNldHVwX3N0YXJ0ZWQiIH0pOwogICAgICBjb25zdCBzZXR1cFN0YXJ0ID0gcGVyZm9ybWFuY2Uubm93KCk7CiAgICAgIGF3YWl0IHdvcmtlckluc3RhbmNlLnNldHVwPy4oKTsKICAgICAgc2VuZE1lc3NhZ2UoeyB0eXBlOiAic2V0dXBfY29tcGxldGVkIiwgZGF0YTogeyBkdXJhdGlvbjogcGVyZm9ybWFuY2Uubm93KCkgLSBzZXR1cFN0YXJ0IH0gfSk7CiAgICAgIHNlbmRNZXNzYWdlKHsgdHlwZTogImluaXQiIH0pOwogICAgfSBlbHNlIGlmIChhY3Rpb24gPT09ICJydW4iICYmIHdvcmtlckluc3RhbmNlICYmIHdvcmtlcklkKSB7CiAgICAgIGNvbnN0IGNvbnRleHQgPSBjcmVhdGVXb3JrZXJDb250ZXh0KHRhc2tJZCk7CiAgICAgIGNvbnN0IHJlc3VsdCA9IGF3YWl0IHdvcmtlckluc3RhbmNlLnJ1bihwYXlsb2FkLCBjb250ZXh0KTsKICAgICAgc2VuZE1lc3NhZ2UoeyB0eXBlOiAicmVzdWx0IiwgdGFza0lkLCBkYXRhOiByZXN1bHQgfSk7CiAgICB9IGVsc2UgaWYgKGFjdGlvbiA9PT0gInRlYXJkb3duIiAmJiB3b3JrZXJJbnN0YW5jZSkgewogICAgICBhd2FpdCB3b3JrZXJJbnN0YW5jZS50ZWFyZG93bj8uKCk7CiAgICAgIHdvcmtlckluc3RhbmNlID0gbnVsbDsKICAgICAgV29ya2VyQ2xhc3MgPSBudWxsOwogICAgICB3b3JrZXJJZCA9IG51bGw7CiAgICAgIHNlbmRNZXNzYWdlKHsgdHlwZTogInRlYXJkb3duIiB9KTsKICAgIH0gZWxzZSB7CiAgICAgIHRocm93IG5ldyBFcnJvcihgSW52YWxpZCBhY3Rpb246ICIke2FjdGlvbn0iIG9yIHdvcmtlciBub3QgaW5pdGlhbGl6ZWRgKTsKICAgIH0KICB9IGNhdGNoIChlcnJvcikgewogICAgaWYgKGVycm9yLm5hbWUgPT09ICJXb3JrZXJUZXJtaW5hdGlvbkVycm9yIiAmJiBlcnJvci5tZXRhZGF0YSkgewogICAgICBzZW5kTWVzc2FnZSh7CiAgICAgICAgdHlwZTogImVycm9yIiwKICAgICAgICB0YXNrSWQsCiAgICAgICAgZGF0YTogewogICAgICAgICAgbWVzc2FnZTogZXJyb3IubWVzc2FnZSwKICAgICAgICAgIGlzV29ya2VyVGVybWluYXRpb25SZXF1ZXN0OiB0cnVlLAogICAgICAgICAgbWV0YWRhdGE6IGVycm9yLm1ldGFkYXRhCiAgICAgICAgfQogICAgICB9KTsKICAgICAgYXdhaXQgbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHNldFRpbWVvdXQocmVzb2x2ZSwgMTApKTsKICAgICAgcHJvY2Vzcz8uZXhpdD8uKDApOwogICAgfSBlbHNlIHsKICAgICAgc2VuZE1lc3NhZ2UoeyB0eXBlOiAiZXJyb3IiLCB0YXNrSWQsIGRhdGE6IGVycm9yLm1lc3NhZ2UgfSk7CiAgICB9CiAgfQp9KTsK";
|
|
20
|
+
workerWrapper_node_default = new URL(url);
|
|
21
|
+
}
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
// core/FyflowScheduler.ts
|
|
25
|
+
var FyflowTask = class {
|
|
26
|
+
id;
|
|
27
|
+
workerType;
|
|
28
|
+
payload;
|
|
29
|
+
optional;
|
|
30
|
+
retryPolicy;
|
|
31
|
+
/** Retry attempts used so far, counted against `retryPolicy.maxRetries`. */
|
|
32
|
+
attempts = 0;
|
|
33
|
+
/**
|
|
34
|
+
* `pending` -> `running` -> `done` | `failed`.
|
|
35
|
+
*
|
|
36
|
+
* A non-optional task that fails with no retries left ends in `user_action`,
|
|
37
|
+
* signalling that something outside the scheduler has to intervene.
|
|
38
|
+
*/
|
|
39
|
+
state = "pending";
|
|
40
|
+
result;
|
|
41
|
+
error;
|
|
42
|
+
// Error message if task failed
|
|
43
|
+
resolve;
|
|
44
|
+
reject;
|
|
45
|
+
workerGroups;
|
|
46
|
+
handleRejection;
|
|
47
|
+
// If true (default), silently handle rejections for fire-and-forget
|
|
48
|
+
startTime;
|
|
49
|
+
// Timestamp when task started execution
|
|
50
|
+
endTime;
|
|
51
|
+
// Timestamp when task completed
|
|
52
|
+
/**
|
|
53
|
+
* Worker-measured execution time in ms (high resolution), set on completion.
|
|
54
|
+
* Unlike `endTime - startTime` this excludes time spent waiting for a worker
|
|
55
|
+
* slot. Cleared when a task is requeued or retried.
|
|
56
|
+
*/
|
|
57
|
+
executionTime;
|
|
58
|
+
/**
|
|
59
|
+
* Bucket this task belongs to for keyed resource groups. Read by the default
|
|
60
|
+
* `keyFrom` of {@link KeyedRateLimitGroup}; a group given its own `keyFrom`
|
|
61
|
+
* can derive the key from the payload instead.
|
|
62
|
+
*/
|
|
63
|
+
limitKey;
|
|
64
|
+
_resourcesPreAllocated;
|
|
65
|
+
// Flag to track if resources are pre-allocated
|
|
66
|
+
/**
|
|
67
|
+
* Keys resolved at dispatch, per keyed group id. Held so release uses the
|
|
68
|
+
* same bucket acquisition used, even if the payload changed meanwhile.
|
|
69
|
+
*/
|
|
70
|
+
_resourceKeys;
|
|
71
|
+
/**
|
|
72
|
+
* Set while this attempt is being settled, so the same outcome arriving from
|
|
73
|
+
* both the pool's `task.failed` event and the task promise settles once.
|
|
74
|
+
* Cleared on every dispatch, so each attempt gets its own settle.
|
|
75
|
+
*/
|
|
76
|
+
_settling;
|
|
77
|
+
// Private fields for resource management
|
|
78
|
+
_scheduler;
|
|
79
|
+
// Reference to scheduler for descendant tracking
|
|
80
|
+
/**
|
|
81
|
+
* @param config.id Unique task id. Reusing an id overwrites the earlier entry.
|
|
82
|
+
* @param config.workerType Key into the scheduler's `workerPools`. An unknown
|
|
83
|
+
* value makes `addTask` throw.
|
|
84
|
+
* @param config.payload Passed verbatim to the worker's `run()`.
|
|
85
|
+
* @param config.optional Default `false`. An optional task that fails
|
|
86
|
+
* resolves `null` instead of rejecting.
|
|
87
|
+
* @param config.retryPolicy `{ maxRetries, backoffMs }`. Omitted means no retries.
|
|
88
|
+
* @param config.workerGroups Resource groups for this task, *in addition* to
|
|
89
|
+
* the groups its WorkerManager declares.
|
|
90
|
+
* @param config.handleRejection Default `true`. Attaches a silent catch so a
|
|
91
|
+
* fire-and-forget failure does not surface as an unhandled rejection;
|
|
92
|
+
* `task.failed` is still emitted.
|
|
93
|
+
* @param config.limitKey Bucket for keyed resource groups, e.g. the endpoint
|
|
94
|
+
* or tenant this task belongs to. Required when the task belongs to a
|
|
95
|
+
* keyed group that has no custom `keyFrom`.
|
|
96
|
+
*/
|
|
97
|
+
constructor({ id, workerType, payload, optional = false, retryPolicy, workerGroups = [], handleRejection = true, limitKey }) {
|
|
98
|
+
this.id = id;
|
|
99
|
+
this.limitKey = limitKey;
|
|
100
|
+
this.workerType = workerType;
|
|
101
|
+
this.payload = payload;
|
|
102
|
+
this.optional = optional;
|
|
103
|
+
this.retryPolicy = retryPolicy;
|
|
104
|
+
this.workerGroups = workerGroups;
|
|
105
|
+
this.handleRejection = handleRejection;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Promise for this task alone, resolving with its result and rejecting if it
|
|
109
|
+
* fails. Prefer `addTask(task, { createPromise: true })`, which wires this up
|
|
110
|
+
* for you. Use `onCompleteDescendants()` to also wait for spawned tasks.
|
|
111
|
+
*/
|
|
112
|
+
onCompletePromise() {
|
|
113
|
+
return new Promise((resolve, reject) => {
|
|
114
|
+
this.resolve = resolve;
|
|
115
|
+
this.reject = reject;
|
|
116
|
+
});
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Wait for this task AND every task spawned from it (children, grandchildren, ...)
|
|
120
|
+
* to reach a terminal state.
|
|
121
|
+
*
|
|
122
|
+
* Descendants come from runtime spawning via `context.spawnTask()` - this is
|
|
123
|
+
* lineage, not a scheduling dependency, and does not affect dispatch order.
|
|
124
|
+
*
|
|
125
|
+
* Resolves with this task's result once the whole workflow has settled. A failed
|
|
126
|
+
* descendant does not reject - descendant failures surface via `task.failed`
|
|
127
|
+
* events - but this task failing rejects, matching `onCompletePromise()`.
|
|
128
|
+
*
|
|
129
|
+
* Must be called after the task has been added to a scheduler.
|
|
130
|
+
*/
|
|
131
|
+
onCompleteDescendants() {
|
|
132
|
+
return new Promise((resolve, reject) => {
|
|
133
|
+
if (!this._scheduler) {
|
|
134
|
+
reject(new Error("Task must be added to a scheduler before tracking descendants"));
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
this._scheduler._trackDescendants(this, resolve, reject);
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
};
|
|
141
|
+
var FyflowScheduler = class _FyflowScheduler extends EventTarget {
|
|
142
|
+
tasks = /* @__PURE__ */ new Map();
|
|
143
|
+
readyQueuesByWorker = /* @__PURE__ */ new Map();
|
|
144
|
+
// Per-worker-type queues for O(1) dispatch
|
|
145
|
+
blockedQueues = /* @__PURE__ */ new Map();
|
|
146
|
+
// Grouped by blocking group IDs for faster lookup
|
|
147
|
+
workerPools;
|
|
148
|
+
groups;
|
|
149
|
+
stats = { queued: 0, running: 0, done: 0, failed: 0 };
|
|
150
|
+
retryTimer = null;
|
|
151
|
+
// Active onCompleteDescendants() waits, keyed by tracker id so one task can be
|
|
152
|
+
// awaited more than once. Empty unless onCompleteDescendants() is used.
|
|
153
|
+
descendantTrackers = /* @__PURE__ */ new Map();
|
|
154
|
+
nextTrackerId = 0;
|
|
155
|
+
// Spawn lineage: parent task id -> ids of tasks spawned from it. Populated only
|
|
156
|
+
// when workers spawn tasks; carries no scheduling meaning.
|
|
157
|
+
spawnedChildren = /* @__PURE__ */ new Map();
|
|
158
|
+
// Terminal task ids in completion order, used only when maxCompletedTasks is
|
|
159
|
+
// set. A Set preserves insertion order, so the oldest entry is its first.
|
|
160
|
+
completedTaskIds = /* @__PURE__ */ new Set();
|
|
161
|
+
options;
|
|
162
|
+
// Track ALL event listeners for cleanup
|
|
163
|
+
allListeners = /* @__PURE__ */ new Map();
|
|
164
|
+
constructor(workerPools, groups = {}, options = {}) {
|
|
165
|
+
super();
|
|
166
|
+
this.workerPools = workerPools;
|
|
167
|
+
this.groups = groups;
|
|
168
|
+
this.options = {
|
|
169
|
+
periodicRetryIntervalMs: 50,
|
|
170
|
+
// Default to 50ms - optimal balance of speed and efficiency
|
|
171
|
+
...options
|
|
172
|
+
};
|
|
173
|
+
for (const workerType of Object.keys(workerPools)) {
|
|
174
|
+
this.readyQueuesByWorker.set(workerType, []);
|
|
175
|
+
}
|
|
176
|
+
this._setupWorkerPoolListeners();
|
|
177
|
+
this._setupGroupEventListeners();
|
|
178
|
+
}
|
|
179
|
+
/** Every resource group a task must acquire: its own plus its pool's. */
|
|
180
|
+
_groupsForTask(task, pool) {
|
|
181
|
+
const ids = [.../* @__PURE__ */ new Set([...task.workerGroups || [], ...pool?.groups || []])];
|
|
182
|
+
return ids.map((id) => ({ id, group: this.groups[id] })).filter((entry) => entry.group);
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Blocked queues are per group, and per (group, key) for keyed groups, so a
|
|
186
|
+
* saturated key never stalls tasks belonging to a different one.
|
|
187
|
+
*/
|
|
188
|
+
_blockedQueueId(groupId, key) {
|
|
189
|
+
return key === void 0 ? groupId : `${groupId}\0${key}`;
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* Resolve the bucket for every keyed group this task belongs to.
|
|
193
|
+
*
|
|
194
|
+
* @throws if a keyed group cannot derive a key. A keyless task would
|
|
195
|
+
* otherwise silently share a bucket with every other keyless task, or skip
|
|
196
|
+
* the limit entirely - both discovered only once something is throttled.
|
|
197
|
+
*/
|
|
198
|
+
_resolveResourceKeys(task, pool) {
|
|
199
|
+
const keys = {};
|
|
200
|
+
for (const { id, group } of this._groupsForTask(task, pool)) {
|
|
201
|
+
if (!group.keyed) continue;
|
|
202
|
+
const key = group.keyFor?.(task);
|
|
203
|
+
if (key === void 0) {
|
|
204
|
+
throw new Error(
|
|
205
|
+
`Missing limit key for group "${id}" on task "${task.id}". Set task.limitKey, or give the group a keyFrom that derives one.`
|
|
206
|
+
);
|
|
207
|
+
}
|
|
208
|
+
keys[id] = key;
|
|
209
|
+
}
|
|
210
|
+
return keys;
|
|
211
|
+
}
|
|
212
|
+
// Any task waiting on a resource group. Blocked tasks are subtracted from
|
|
213
|
+
// stats.queued when they leave the ready queue, so they are invisible to the
|
|
214
|
+
// stats alone.
|
|
215
|
+
_hasBlockedTasks() {
|
|
216
|
+
for (const queue of this.blockedQueues.values()) {
|
|
217
|
+
if (queue.length > 0) return true;
|
|
218
|
+
}
|
|
219
|
+
return false;
|
|
220
|
+
}
|
|
221
|
+
_checkCompletion() {
|
|
222
|
+
const hasBlockedTasks = this._hasBlockedTasks();
|
|
223
|
+
if (this.stats.queued === 0 && this.stats.running === 0 && !hasBlockedTasks && this.stats.done > 0) {
|
|
224
|
+
this._clearPeriodicRetry();
|
|
225
|
+
this.dispatchEvent(new CustomEvent("scheduler.completed", { detail: this.stats }));
|
|
226
|
+
} else if (hasBlockedTasks && this.stats.queued === 0 && this.stats.running === 0) {
|
|
227
|
+
this._schedulePeriodicRetry();
|
|
228
|
+
} else if (this.stats.queued > 0 && this.stats.running === 0) {
|
|
229
|
+
this._retryBlockedTasks();
|
|
230
|
+
this._schedulePeriodicRetry();
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* Queue a task and start dispatching.
|
|
235
|
+
*
|
|
236
|
+
* Fire-and-forget by default: returns `undefined` unless
|
|
237
|
+
* `{ createPromise: true }` is passed, in which case it returns a promise
|
|
238
|
+
* that resolves with the task's result or rejects if it fails.
|
|
239
|
+
*
|
|
240
|
+
* @throws if `task.workerType` is not a key in the scheduler's `workerPools`.
|
|
241
|
+
*/
|
|
242
|
+
addTask(task, options) {
|
|
243
|
+
const workerQueue = this.readyQueuesByWorker.get(task.workerType);
|
|
244
|
+
if (!workerQueue) {
|
|
245
|
+
throw new Error(`Unknown worker type: ${task.workerType}`);
|
|
246
|
+
}
|
|
247
|
+
this._resolveResourceKeys(task, this.workerPools[task.workerType]);
|
|
248
|
+
task._scheduler = this;
|
|
249
|
+
this.tasks.set(task.id, task);
|
|
250
|
+
workerQueue.push(task);
|
|
251
|
+
this.stats.queued++;
|
|
252
|
+
this._dispatchLoop();
|
|
253
|
+
if (options?.createPromise) {
|
|
254
|
+
const promise = task.onCompletePromise();
|
|
255
|
+
if (task.handleRejection) {
|
|
256
|
+
promise.catch(() => {
|
|
257
|
+
});
|
|
258
|
+
}
|
|
259
|
+
return promise;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* Queue many tasks with a single dispatch pass - use this for bulk additions
|
|
264
|
+
* (>1000 tasks) instead of calling `addTask` in a loop, which would run the
|
|
265
|
+
* dispatch loop once per task.
|
|
266
|
+
*
|
|
267
|
+
* Like `addTask`, returns nothing unless `{ createPromise: true }` is passed;
|
|
268
|
+
* with it, returns one promise per task in the same order:
|
|
269
|
+
*
|
|
270
|
+
* ```typescript
|
|
271
|
+
* const results = await Promise.all(
|
|
272
|
+
* scheduler.addTasks(tasks, { createPromise: true }) as Promise<any>[]
|
|
273
|
+
* );
|
|
274
|
+
* ```
|
|
275
|
+
*
|
|
276
|
+
* @throws if any task's `workerType` is not a key in the scheduler's
|
|
277
|
+
* `workerPools`. The batch is validated up front, so a rejected call
|
|
278
|
+
* queues nothing at all.
|
|
279
|
+
*/
|
|
280
|
+
addTasks(tasks, options) {
|
|
281
|
+
for (const task of tasks) {
|
|
282
|
+
if (!this.readyQueuesByWorker.has(task.workerType)) {
|
|
283
|
+
throw new Error(`Unknown worker type: ${task.workerType}`);
|
|
284
|
+
}
|
|
285
|
+
this._resolveResourceKeys(task, this.workerPools[task.workerType]);
|
|
286
|
+
}
|
|
287
|
+
const promises = [];
|
|
288
|
+
for (const task of tasks) {
|
|
289
|
+
task._scheduler = this;
|
|
290
|
+
this.tasks.set(task.id, task);
|
|
291
|
+
this.readyQueuesByWorker.get(task.workerType).push(task);
|
|
292
|
+
this.stats.queued++;
|
|
293
|
+
if (options?.createPromise) {
|
|
294
|
+
const promise = task.onCompletePromise();
|
|
295
|
+
if (task.handleRejection) {
|
|
296
|
+
promise.catch(() => {
|
|
297
|
+
});
|
|
298
|
+
}
|
|
299
|
+
promises.push(promise);
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
this._dispatchLoop();
|
|
303
|
+
if (options?.createPromise) {
|
|
304
|
+
return promises;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
// _onParentComplete(parent: FyflowTask) {
|
|
308
|
+
// // No dependency tracking - just trigger dispatch loop
|
|
309
|
+
// this._dispatchLoop();
|
|
310
|
+
// }
|
|
311
|
+
_dispatchLoop() {
|
|
312
|
+
let tasksProcessed = 0;
|
|
313
|
+
let keyedSkips = 0;
|
|
314
|
+
const maxTasksPerLoop = 1e3;
|
|
315
|
+
for (const [workerType, queue] of this.readyQueuesByWorker) {
|
|
316
|
+
if (queue.length === 0) continue;
|
|
317
|
+
const pool = this.workerPools[workerType];
|
|
318
|
+
if (!pool) continue;
|
|
319
|
+
while (queue.length > 0 && tasksProcessed < maxTasksPerLoop) {
|
|
320
|
+
const task = queue[0];
|
|
321
|
+
const canDispatchResult = this._canDispatchTask(task, pool);
|
|
322
|
+
if (canDispatchResult === true) {
|
|
323
|
+
queue.shift();
|
|
324
|
+
this._dispatchTask(task, pool);
|
|
325
|
+
tasksProcessed++;
|
|
326
|
+
} else if (canDispatchResult === "blocked" || canDispatchResult === "blocked-keyed") {
|
|
327
|
+
queue.shift();
|
|
328
|
+
this.stats.queued = Math.max(0, this.stats.queued - 1);
|
|
329
|
+
if (canDispatchResult === "blocked-keyed" && keyedSkips < maxTasksPerLoop) {
|
|
330
|
+
keyedSkips++;
|
|
331
|
+
continue;
|
|
332
|
+
}
|
|
333
|
+
break;
|
|
334
|
+
} else {
|
|
335
|
+
this._schedulePeriodicRetry();
|
|
336
|
+
break;
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
const hasReadyTasks = Array.from(this.readyQueuesByWorker.values()).some((q) => q.length > 0);
|
|
341
|
+
const hasBlockedTasks = Array.from(this.blockedQueues.values()).some((q) => q.length > 0);
|
|
342
|
+
const hasQueuedInStats = this.stats.queued > 0;
|
|
343
|
+
if (!hasReadyTasks && !hasBlockedTasks && !hasQueuedInStats) {
|
|
344
|
+
this._clearPeriodicRetry();
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
// Helper: Check resource availability and block if needed
|
|
348
|
+
_checkAndBlockResources(task, pool) {
|
|
349
|
+
const taskGroupIds = task.workerGroups || [];
|
|
350
|
+
const workerGroupIds = pool.groups || [];
|
|
351
|
+
if (taskGroupIds.length === 0 && workerGroupIds.length === 0) {
|
|
352
|
+
return true;
|
|
353
|
+
}
|
|
354
|
+
const entries = this._groupsForTask(task, pool);
|
|
355
|
+
const keys = task._resourceKeys ?? this._resolveResourceKeys(task, pool);
|
|
356
|
+
task._resourceKeys = keys;
|
|
357
|
+
for (const { id, group } of entries) {
|
|
358
|
+
const key = group.keyed ? keys[id] : void 0;
|
|
359
|
+
if (!group.canRun(key)) {
|
|
360
|
+
const blockedOnKey = Boolean(group.keyed);
|
|
361
|
+
const queueId = this._blockedQueueId(id, key);
|
|
362
|
+
if (!this.blockedQueues.has(queueId)) {
|
|
363
|
+
this.blockedQueues.set(queueId, []);
|
|
364
|
+
}
|
|
365
|
+
const blockedQueue = this.blockedQueues.get(queueId);
|
|
366
|
+
if (!blockedQueue.includes(task)) {
|
|
367
|
+
blockedQueue.push(task);
|
|
368
|
+
}
|
|
369
|
+
this._schedulePeriodicRetry();
|
|
370
|
+
return blockedOnKey ? "blocked-keyed" : "blocked";
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
return true;
|
|
374
|
+
}
|
|
375
|
+
// NEW: Non-optimistic mode helper methods
|
|
376
|
+
_canDispatchTask(task, pool) {
|
|
377
|
+
const availableThreads = pool.threads.filter(
|
|
378
|
+
(t) => t.canAcceptTask && t.canAcceptTask()
|
|
379
|
+
).length;
|
|
380
|
+
if (availableThreads > 0) {
|
|
381
|
+
const resourceResult = this._checkAndBlockResources(task, pool);
|
|
382
|
+
if (resourceResult !== true) {
|
|
383
|
+
return resourceResult;
|
|
384
|
+
}
|
|
385
|
+
return true;
|
|
386
|
+
}
|
|
387
|
+
const initializingCount = pool.initializingThreads ? pool.initializingThreads.size : 0;
|
|
388
|
+
const canCreateThread = pool.threads.length + initializingCount < pool.maxThreads;
|
|
389
|
+
if (canCreateThread) {
|
|
390
|
+
const resourceResult = this._checkAndBlockResources(task, pool);
|
|
391
|
+
if (resourceResult !== true) {
|
|
392
|
+
return resourceResult;
|
|
393
|
+
}
|
|
394
|
+
return true;
|
|
395
|
+
}
|
|
396
|
+
return false;
|
|
397
|
+
}
|
|
398
|
+
_dispatchTask(task, pool) {
|
|
399
|
+
const entries = this._groupsForTask(task, pool);
|
|
400
|
+
const keys = task._resourceKeys ?? this._resolveResourceKeys(task, pool);
|
|
401
|
+
task._resourceKeys = keys;
|
|
402
|
+
entries.forEach(({ id, group }) => group.onStart(group.keyed ? keys[id] : void 0));
|
|
403
|
+
task._resourcesPreAllocated = true;
|
|
404
|
+
task._settling = false;
|
|
405
|
+
task.state = "running";
|
|
406
|
+
task.startTime = Date.now();
|
|
407
|
+
this.stats.queued = Math.max(0, this.stats.queued - 1);
|
|
408
|
+
this.stats.running++;
|
|
409
|
+
pool.enqueueNoQueue(task.id, task.payload).then((result) => {
|
|
410
|
+
this._onTaskComplete(task, result, pool);
|
|
411
|
+
}).catch((error) => {
|
|
412
|
+
if (error.message === "Worker at maximum concurrent task capacity") {
|
|
413
|
+
task.state = "pending";
|
|
414
|
+
task.startTime = void 0;
|
|
415
|
+
task.executionTime = void 0;
|
|
416
|
+
this.stats.running = Math.max(0, this.stats.running - 1);
|
|
417
|
+
this.stats.queued++;
|
|
418
|
+
this._releaseResourcesForTask(task, pool);
|
|
419
|
+
const workerQueue = this.readyQueuesByWorker.get(task.workerType);
|
|
420
|
+
if (workerQueue) {
|
|
421
|
+
workerQueue.push(task);
|
|
422
|
+
}
|
|
423
|
+
setTimeout(() => this._dispatchLoop(), 10);
|
|
424
|
+
} else {
|
|
425
|
+
this._onTaskFailed(task, error, pool);
|
|
426
|
+
}
|
|
427
|
+
});
|
|
428
|
+
}
|
|
429
|
+
_releaseResourcesForTask(task, pool) {
|
|
430
|
+
if (!task._resourcesPreAllocated) return;
|
|
431
|
+
const entries = this._groupsForTask(task, pool);
|
|
432
|
+
const keys = task._resourceKeys ?? {};
|
|
433
|
+
for (const { id, group } of entries) {
|
|
434
|
+
group.onFinish(group.keyed ? keys[id] : void 0);
|
|
435
|
+
}
|
|
436
|
+
task._resourcesPreAllocated = void 0;
|
|
437
|
+
task._resourceKeys = void 0;
|
|
438
|
+
for (const { id, group } of entries) {
|
|
439
|
+
this._retryBlockedTasksForGroup(id, group.keyed ? keys[id] : void 0);
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
/**
|
|
443
|
+
* The one place a task settles.
|
|
444
|
+
*
|
|
445
|
+
* An outcome can reach the scheduler twice - as the pool's `task.failed`
|
|
446
|
+
* event and as the settling of the pool's task promise - and each of those
|
|
447
|
+
* is the ONLY path in some situation, so neither can be dropped:
|
|
448
|
+
*
|
|
449
|
+
* - an ordinary task throw reaches both
|
|
450
|
+
* - a worker that dies with `requeueFailedTasks: false` reaches only the event
|
|
451
|
+
* - a worker that cannot be constructed reaches only the promise
|
|
452
|
+
*
|
|
453
|
+
* They used to be two near-duplicate implementations, so a single failure
|
|
454
|
+
* emitted `task.failed` twice, counted `stats.failed` twice, raced the
|
|
455
|
+
* terminal state between `user_action` and `failed`, and - because only the
|
|
456
|
+
* promise path released resources - leaked a resource group slot whenever
|
|
457
|
+
* only the event path ran.
|
|
458
|
+
*/
|
|
459
|
+
_settleTask(task, pool, outcome) {
|
|
460
|
+
if (task._settling) return;
|
|
461
|
+
task._settling = true;
|
|
462
|
+
this._releaseResourcesForTask(task, pool);
|
|
463
|
+
if ("requeue" in outcome) {
|
|
464
|
+
task.state = "pending";
|
|
465
|
+
task.startTime = void 0;
|
|
466
|
+
task.executionTime = void 0;
|
|
467
|
+
task._resourcesPreAllocated = void 0;
|
|
468
|
+
task._resourceKeys = void 0;
|
|
469
|
+
this.stats.running = Math.max(0, this.stats.running - 1);
|
|
470
|
+
this.stats.queued++;
|
|
471
|
+
const requeueTarget = this.readyQueuesByWorker.get(task.workerType);
|
|
472
|
+
if (requeueTarget) requeueTarget.push(task);
|
|
473
|
+
this._dispatchLoop();
|
|
474
|
+
return;
|
|
475
|
+
}
|
|
476
|
+
if (outcome.ok) {
|
|
477
|
+
task.state = "done";
|
|
478
|
+
task.endTime = Date.now();
|
|
479
|
+
task.result = outcome.result;
|
|
480
|
+
this.stats.running = Math.max(0, this.stats.running - 1);
|
|
481
|
+
this.stats.done++;
|
|
482
|
+
task.resolve?.(outcome.result);
|
|
483
|
+
this.dispatchEvent(new CustomEvent("task.completed", { detail: task }));
|
|
484
|
+
this._settleDescendantTrackers(task.id);
|
|
485
|
+
this._recordTerminalTask(task.id);
|
|
486
|
+
this._checkCompletion();
|
|
487
|
+
this._dispatchLoop();
|
|
488
|
+
return;
|
|
489
|
+
}
|
|
490
|
+
const error = outcome.error;
|
|
491
|
+
if (task.attempts < (task.retryPolicy?.maxRetries || 0)) {
|
|
492
|
+
task.attempts++;
|
|
493
|
+
task.state = "pending";
|
|
494
|
+
task.startTime = void 0;
|
|
495
|
+
task.executionTime = void 0;
|
|
496
|
+
task._resourcesPreAllocated = void 0;
|
|
497
|
+
task._resourceKeys = void 0;
|
|
498
|
+
this.stats.running = Math.max(0, this.stats.running - 1);
|
|
499
|
+
this.stats.queued++;
|
|
500
|
+
const workerQueue = this.readyQueuesByWorker.get(task.workerType);
|
|
501
|
+
if (workerQueue) workerQueue.push(task);
|
|
502
|
+
setTimeout(() => this._dispatchLoop(), task.retryPolicy?.backoffMs || 0);
|
|
503
|
+
return;
|
|
504
|
+
}
|
|
505
|
+
task.error = error instanceof Error ? error.message : String(error ?? "Task failed");
|
|
506
|
+
task.endTime = Date.now();
|
|
507
|
+
this.stats.running = Math.max(0, this.stats.running - 1);
|
|
508
|
+
this.stats.failed++;
|
|
509
|
+
if (task.optional) {
|
|
510
|
+
task.state = "failed";
|
|
511
|
+
task.resolve?.(null);
|
|
512
|
+
} else {
|
|
513
|
+
task.state = "user_action";
|
|
514
|
+
task.reject?.(error instanceof Error ? error : new Error(task.error));
|
|
515
|
+
}
|
|
516
|
+
this.dispatchEvent(new CustomEvent("task.failed", { detail: task }));
|
|
517
|
+
if (!task.optional) {
|
|
518
|
+
this.dispatchEvent(new CustomEvent("task.user_action", { detail: task }));
|
|
519
|
+
}
|
|
520
|
+
this._settleDescendantTrackers(task.id);
|
|
521
|
+
this._recordTerminalTask(task.id);
|
|
522
|
+
this._checkCompletion();
|
|
523
|
+
this._dispatchLoop();
|
|
524
|
+
}
|
|
525
|
+
_onTaskComplete(task, result, pool) {
|
|
526
|
+
this._settleTask(task, pool, { ok: true, result });
|
|
527
|
+
}
|
|
528
|
+
_onTaskFailed(task, error, pool) {
|
|
529
|
+
this._settleTask(task, pool, { ok: false, error });
|
|
530
|
+
}
|
|
531
|
+
_retryBlockedTasks() {
|
|
532
|
+
const hasReadyTasks = Array.from(this.readyQueuesByWorker.values()).some((q) => q.length > 0);
|
|
533
|
+
if (hasReadyTasks) {
|
|
534
|
+
this._dispatchLoop();
|
|
535
|
+
}
|
|
536
|
+
for (const [groupId, group] of Object.entries(this.groups)) {
|
|
537
|
+
if (!group || !group.canRun()) continue;
|
|
538
|
+
if (group.keyed) {
|
|
539
|
+
for (const key of this._blockedKeysFor(groupId)) {
|
|
540
|
+
this._retryBlockedTasksForGroup(groupId, key);
|
|
541
|
+
}
|
|
542
|
+
} else {
|
|
543
|
+
this._retryBlockedTasksForGroup(groupId);
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
/** Keys that currently have tasks blocked on this group. */
|
|
548
|
+
_blockedKeysFor(groupId) {
|
|
549
|
+
const prefix = `${groupId}\0`;
|
|
550
|
+
const keys = [];
|
|
551
|
+
for (const [queueId, queue] of this.blockedQueues) {
|
|
552
|
+
if (queue.length > 0 && queueId.startsWith(prefix)) {
|
|
553
|
+
keys.push(queueId.slice(prefix.length));
|
|
554
|
+
}
|
|
555
|
+
}
|
|
556
|
+
return keys;
|
|
557
|
+
}
|
|
558
|
+
_retryBlockedTasksForGroup(groupId, key) {
|
|
559
|
+
const queueId = this._blockedQueueId(groupId, key);
|
|
560
|
+
const blockedTasks = this.blockedQueues.get(queueId);
|
|
561
|
+
if (!blockedTasks || blockedTasks.length === 0) return;
|
|
562
|
+
const group = this.groups[groupId];
|
|
563
|
+
if (!group || !group.canRun(key)) return;
|
|
564
|
+
const tasksToRetry = [];
|
|
565
|
+
const stillBlocked = [];
|
|
566
|
+
while (blockedTasks.length > 0 && group.canRun(key)) {
|
|
567
|
+
const task = blockedTasks.shift();
|
|
568
|
+
const pool = this.workerPools[task.workerType];
|
|
569
|
+
if (!pool) continue;
|
|
570
|
+
const entries = this._groupsForTask(task, pool);
|
|
571
|
+
const keys = task._resourceKeys ?? {};
|
|
572
|
+
const runnable = entries.every(
|
|
573
|
+
({ id, group: g }) => g.canRun(g.keyed ? keys[id] : void 0)
|
|
574
|
+
);
|
|
575
|
+
if (!runnable) {
|
|
576
|
+
stillBlocked.push(task);
|
|
577
|
+
continue;
|
|
578
|
+
}
|
|
579
|
+
for (const { id, group: g } of entries) {
|
|
580
|
+
const otherQueueId = this._blockedQueueId(id, g.keyed ? keys[id] : void 0);
|
|
581
|
+
if (otherQueueId === queueId) continue;
|
|
582
|
+
const otherQueue = this.blockedQueues.get(otherQueueId);
|
|
583
|
+
if (otherQueue) {
|
|
584
|
+
const index = otherQueue.indexOf(task);
|
|
585
|
+
if (index !== -1) otherQueue.splice(index, 1);
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
tasksToRetry.push(task);
|
|
589
|
+
}
|
|
590
|
+
if (stillBlocked.length > 0) blockedTasks.unshift(...stillBlocked);
|
|
591
|
+
if (tasksToRetry.length > 0) {
|
|
592
|
+
for (const task of tasksToRetry) {
|
|
593
|
+
const workerQueue = this.readyQueuesByWorker.get(task.workerType);
|
|
594
|
+
if (workerQueue) {
|
|
595
|
+
workerQueue.unshift(task);
|
|
596
|
+
this.stats.queued++;
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
this._dispatchLoop();
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
_schedulePeriodicRetry() {
|
|
603
|
+
if (this.retryTimer) {
|
|
604
|
+
clearTimeout(this.retryTimer);
|
|
605
|
+
}
|
|
606
|
+
const hasReadyTasks = Array.from(this.readyQueuesByWorker.values()).some((q) => q.length > 0);
|
|
607
|
+
const hasBlockedTasks = Array.from(this.blockedQueues.values()).some((queue) => queue.length > 0);
|
|
608
|
+
if (hasReadyTasks || hasBlockedTasks) {
|
|
609
|
+
this.retryTimer = setTimeout(() => {
|
|
610
|
+
this._retryBlockedTasks();
|
|
611
|
+
this._schedulePeriodicRetry();
|
|
612
|
+
}, this.options.periodicRetryIntervalMs);
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
_clearPeriodicRetry() {
|
|
616
|
+
if (this.retryTimer) {
|
|
617
|
+
clearTimeout(this.retryTimer);
|
|
618
|
+
this.retryTimer = null;
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
// Helper to track internal listeners for cleanup
|
|
622
|
+
// Override addEventListener to track ALL listeners for cleanup
|
|
623
|
+
// TODO: likely unneccesary, verify
|
|
624
|
+
// override addEventListener(event: string, listener: EventListenerOrEventListenerObject, options?: boolean | AddEventListenerOptions): void {
|
|
625
|
+
// super.addEventListener(event, listener, options);
|
|
626
|
+
// this._trackListener(this, event, listener as Function);
|
|
627
|
+
// }
|
|
628
|
+
_addInternalListener(target, event, listener) {
|
|
629
|
+
this._trackListener(target, event, listener);
|
|
630
|
+
target.addEventListener(event, listener);
|
|
631
|
+
}
|
|
632
|
+
_trackListener(target, event, listener) {
|
|
633
|
+
if (!this.allListeners.has(target)) {
|
|
634
|
+
this.allListeners.set(target, []);
|
|
635
|
+
}
|
|
636
|
+
this.allListeners.get(target).push({ event, listener });
|
|
637
|
+
}
|
|
638
|
+
// Remove all event listeners (both internal and external)
|
|
639
|
+
_removeAllListeners() {
|
|
640
|
+
for (const [target, listeners] of this.allListeners) {
|
|
641
|
+
for (const { event, listener } of listeners) {
|
|
642
|
+
try {
|
|
643
|
+
target.removeEventListener(event, listener);
|
|
644
|
+
} catch {
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
}
|
|
648
|
+
this.allListeners.clear();
|
|
649
|
+
}
|
|
650
|
+
_setupGroupEventListeners() {
|
|
651
|
+
}
|
|
652
|
+
_setupWorkerPoolListeners() {
|
|
653
|
+
Object.values(this.workerPools).forEach((pool) => {
|
|
654
|
+
const taskStartedListener = (e) => {
|
|
655
|
+
const task = this.tasks.get(e.detail.taskId);
|
|
656
|
+
if (task) {
|
|
657
|
+
this.dispatchEvent(new CustomEvent("task.running", { detail: task }));
|
|
658
|
+
}
|
|
659
|
+
};
|
|
660
|
+
this._addInternalListener(pool, "task.started", taskStartedListener);
|
|
661
|
+
const taskProgressListener = (e) => {
|
|
662
|
+
const task = this.tasks.get(e.detail.taskId);
|
|
663
|
+
if (task) {
|
|
664
|
+
const eventDetail = {
|
|
665
|
+
...task,
|
|
666
|
+
progress: e.detail.progress,
|
|
667
|
+
message: e.detail.message,
|
|
668
|
+
details: e.detail.details,
|
|
669
|
+
workerId: e.detail.workerId,
|
|
670
|
+
workerType: e.detail.workerType,
|
|
671
|
+
timestamp: e.detail.timestamp
|
|
672
|
+
};
|
|
673
|
+
this.dispatchEvent(new CustomEvent("task.progress", { detail: eventDetail }));
|
|
674
|
+
}
|
|
675
|
+
};
|
|
676
|
+
this._addInternalListener(pool, "task.progress", taskProgressListener);
|
|
677
|
+
const taskSpawnListener = (e) => {
|
|
678
|
+
const task = this.tasks.get(e.detail.taskId);
|
|
679
|
+
if (task) {
|
|
680
|
+
const eventDetail = {
|
|
681
|
+
parentTask: task,
|
|
682
|
+
spawnConfig: e.detail.spawnConfig,
|
|
683
|
+
workerId: e.detail.workerId,
|
|
684
|
+
workerType: e.detail.workerType,
|
|
685
|
+
timestamp: e.detail.timestamp
|
|
686
|
+
};
|
|
687
|
+
this.dispatchEvent(new CustomEvent("task.spawn_request", { detail: eventDetail }));
|
|
688
|
+
const spawnConfig = e.detail.spawnConfig;
|
|
689
|
+
const childTask = new FyflowTask({
|
|
690
|
+
id: spawnConfig.id,
|
|
691
|
+
workerType: spawnConfig.workerType,
|
|
692
|
+
payload: spawnConfig.payload,
|
|
693
|
+
optional: spawnConfig.optional || false,
|
|
694
|
+
retryPolicy: spawnConfig.retryPolicy,
|
|
695
|
+
workerGroups: spawnConfig.workerGroups || []
|
|
696
|
+
});
|
|
697
|
+
try {
|
|
698
|
+
this.addTask(childTask);
|
|
699
|
+
} catch (error) {
|
|
700
|
+
this.dispatchEvent(new CustomEvent("task.spawn_failed", {
|
|
701
|
+
detail: {
|
|
702
|
+
parentTask: task,
|
|
703
|
+
spawnConfig,
|
|
704
|
+
error,
|
|
705
|
+
workerId: e.detail.workerId,
|
|
706
|
+
workerType: e.detail.workerType,
|
|
707
|
+
timestamp: Date.now()
|
|
708
|
+
}
|
|
709
|
+
}));
|
|
710
|
+
return;
|
|
711
|
+
}
|
|
712
|
+
this._recordSpawn(task.id, childTask.id);
|
|
713
|
+
this._addSpawnedTaskToTrackers(task.id, childTask);
|
|
714
|
+
}
|
|
715
|
+
};
|
|
716
|
+
this._addInternalListener(pool, "task.spawn_request", taskSpawnListener);
|
|
717
|
+
const taskRequeueListener = (e) => {
|
|
718
|
+
const task = this.tasks.get(e.detail.taskId);
|
|
719
|
+
if (!task || task.state !== "running") return;
|
|
720
|
+
this._settleTask(task, this.workerPools[task.workerType], { requeue: true });
|
|
721
|
+
};
|
|
722
|
+
this._addInternalListener(pool, "task.requeue_required", taskRequeueListener);
|
|
723
|
+
const taskCompletedListener = (e) => {
|
|
724
|
+
const task = this.tasks.get(e.detail.taskId);
|
|
725
|
+
if (task) {
|
|
726
|
+
if (e.detail.executionTime !== void 0) {
|
|
727
|
+
task.executionTime = e.detail.executionTime;
|
|
728
|
+
}
|
|
729
|
+
}
|
|
730
|
+
};
|
|
731
|
+
this._addInternalListener(pool, "task.completed", taskCompletedListener);
|
|
732
|
+
const taskFailedListener = (e) => {
|
|
733
|
+
const task = this.tasks.get(e.detail.taskId);
|
|
734
|
+
if (!task) return;
|
|
735
|
+
this._settleTask(task, pool, { ok: false, error: e.detail.error });
|
|
736
|
+
};
|
|
737
|
+
this._addInternalListener(pool, "task.failed", taskFailedListener);
|
|
738
|
+
});
|
|
739
|
+
}
|
|
740
|
+
// --- Descendant tracking -------------------------------------------------
|
|
741
|
+
// Descendants are tasks created at runtime via context.spawnTask(). This is
|
|
742
|
+
// lineage only: it never influences dispatch order or readiness.
|
|
743
|
+
static TERMINAL_STATES = /* @__PURE__ */ new Set(["done", "failed", "user_action"]);
|
|
744
|
+
_isTerminal(state) {
|
|
745
|
+
return _FyflowScheduler.TERMINAL_STATES.has(state);
|
|
746
|
+
}
|
|
747
|
+
// Record that childId was spawned from parentId
|
|
748
|
+
_recordSpawn(parentId, childId) {
|
|
749
|
+
let children = this.spawnedChildren.get(parentId);
|
|
750
|
+
if (!children) {
|
|
751
|
+
children = /* @__PURE__ */ new Set();
|
|
752
|
+
this.spawnedChildren.set(parentId, children);
|
|
753
|
+
}
|
|
754
|
+
children.add(childId);
|
|
755
|
+
}
|
|
756
|
+
_trackDescendants(task, resolve, reject) {
|
|
757
|
+
const pendingDescendants = /* @__PURE__ */ new Set();
|
|
758
|
+
this._collectPendingDescendants(task.id, pendingDescendants, /* @__PURE__ */ new Set());
|
|
759
|
+
if (pendingDescendants.size === 0) {
|
|
760
|
+
this._settleTracker({ rootId: task.id, rootTask: task, resolve, reject, pendingDescendants });
|
|
761
|
+
return;
|
|
762
|
+
}
|
|
763
|
+
this.descendantTrackers.set(this.nextTrackerId++, {
|
|
764
|
+
rootId: task.id,
|
|
765
|
+
rootTask: task,
|
|
766
|
+
resolve,
|
|
767
|
+
reject,
|
|
768
|
+
pendingDescendants
|
|
769
|
+
});
|
|
770
|
+
}
|
|
771
|
+
_collectPendingDescendants(taskId, pending, visited) {
|
|
772
|
+
if (visited.has(taskId)) return;
|
|
773
|
+
visited.add(taskId);
|
|
774
|
+
const task = this.tasks.get(taskId);
|
|
775
|
+
if (task && !this._isTerminal(task.state)) {
|
|
776
|
+
pending.add(taskId);
|
|
777
|
+
}
|
|
778
|
+
const children = this.spawnedChildren.get(taskId);
|
|
779
|
+
if (children) {
|
|
780
|
+
for (const childId of children) {
|
|
781
|
+
this._collectPendingDescendants(childId, pending, visited);
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
// Called when a task reaches a terminal state (done, failed or user_action).
|
|
786
|
+
// Does nothing unless maxCompletedTasks is configured.
|
|
787
|
+
_recordTerminalTask(taskId) {
|
|
788
|
+
const limit = this.options.maxCompletedTasks;
|
|
789
|
+
if (limit === void 0) return;
|
|
790
|
+
this.completedTaskIds.add(taskId);
|
|
791
|
+
while (this.completedTaskIds.size > limit) {
|
|
792
|
+
const oldest = this.completedTaskIds.values().next().value;
|
|
793
|
+
if (oldest === void 0) break;
|
|
794
|
+
this.completedTaskIds.delete(oldest);
|
|
795
|
+
this.tasks.delete(oldest);
|
|
796
|
+
this.spawnedChildren.delete(oldest);
|
|
797
|
+
}
|
|
798
|
+
}
|
|
799
|
+
// Called when a task reaches a terminal state (done, failed or user_action)
|
|
800
|
+
_settleDescendantTrackers(taskId) {
|
|
801
|
+
if (this.descendantTrackers.size === 0) return;
|
|
802
|
+
for (const [trackerId, tracker] of this.descendantTrackers) {
|
|
803
|
+
if (!tracker.pendingDescendants.delete(taskId)) continue;
|
|
804
|
+
if (tracker.pendingDescendants.size > 0) continue;
|
|
805
|
+
this.descendantTrackers.delete(trackerId);
|
|
806
|
+
this._settleTracker(tracker);
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
_settleTracker(tracker) {
|
|
810
|
+
const rootTask = tracker.rootTask;
|
|
811
|
+
if (rootTask.state === "failed" || rootTask.state === "user_action") {
|
|
812
|
+
tracker.reject(new Error(rootTask.error || `Task ${tracker.rootId} failed`));
|
|
813
|
+
return;
|
|
814
|
+
}
|
|
815
|
+
tracker.resolve(rootTask.result ?? null);
|
|
816
|
+
}
|
|
817
|
+
_addSpawnedTaskToTrackers(parentTaskId, spawnedTask) {
|
|
818
|
+
if (this.descendantTrackers.size === 0) return;
|
|
819
|
+
for (const tracker of this.descendantTrackers.values()) {
|
|
820
|
+
if (tracker.rootId === parentTaskId || tracker.pendingDescendants.has(parentTaskId)) {
|
|
821
|
+
if (!this._isTerminal(spawnedTask.state)) {
|
|
822
|
+
tracker.pendingDescendants.add(spawnedTask.id);
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
}
|
|
827
|
+
// Shutdown the scheduler and cleanup all resources
|
|
828
|
+
/**
|
|
829
|
+
* Get current metrics for all resource groups
|
|
830
|
+
*
|
|
831
|
+
* Returns real-time utilization and availability for monitoring
|
|
832
|
+
*
|
|
833
|
+
* @returns Record of group ID to metrics
|
|
834
|
+
*/
|
|
835
|
+
getResourceMetrics() {
|
|
836
|
+
const metrics = {};
|
|
837
|
+
for (const [id, group] of Object.entries(this.groups)) {
|
|
838
|
+
if (group && typeof group.getMetrics === "function") {
|
|
839
|
+
metrics[id] = group.getMetrics();
|
|
840
|
+
}
|
|
841
|
+
}
|
|
842
|
+
return metrics;
|
|
843
|
+
}
|
|
844
|
+
/**
|
|
845
|
+
* Get lifetime stats for resource groups
|
|
846
|
+
*
|
|
847
|
+
* Returns aggregated statistics for strict groups (acquisition times, rejections, etc.)
|
|
848
|
+
*
|
|
849
|
+
* @returns Record of group ID to stats (only strict groups provide stats)
|
|
850
|
+
*/
|
|
851
|
+
getResourceStats() {
|
|
852
|
+
const stats = {};
|
|
853
|
+
for (const [id, group] of Object.entries(this.groups)) {
|
|
854
|
+
if (group && typeof group.getStats === "function") {
|
|
855
|
+
stats[id] = group.getStats();
|
|
856
|
+
}
|
|
857
|
+
}
|
|
858
|
+
return stats;
|
|
859
|
+
}
|
|
860
|
+
/**
|
|
861
|
+
* Wait for running tasks, then terminate every worker pool, drop all
|
|
862
|
+
* listeners and clear internal state.
|
|
863
|
+
*
|
|
864
|
+
* Always call this when finished - live workers and the periodic retry timer
|
|
865
|
+
* will otherwise keep the process alive. Pending `onCompleteDescendants()`
|
|
866
|
+
* waits reject.
|
|
867
|
+
*/
|
|
868
|
+
async shutdown() {
|
|
869
|
+
this._clearPeriodicRetry();
|
|
870
|
+
this._removeAllListeners();
|
|
871
|
+
const drainDeadline = Date.now() + 5e3;
|
|
872
|
+
while (this.stats.running > 0 && Date.now() < drainDeadline) {
|
|
873
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
874
|
+
}
|
|
875
|
+
if (this.stats.running > 0) {
|
|
876
|
+
console.warn(
|
|
877
|
+
`\u26A0\uFE0F Shutdown proceeding with ${this.stats.running} task(s) still marked running - they were dispatched to a pool that could not complete them`
|
|
878
|
+
);
|
|
879
|
+
}
|
|
880
|
+
this.readyQueuesByWorker.forEach((q) => q.length = 0);
|
|
881
|
+
this.blockedQueues.clear();
|
|
882
|
+
const workerManagerPromises = Object.values(this.workerPools).map(async (pool) => {
|
|
883
|
+
if (pool && typeof pool.shutdown === "function") {
|
|
884
|
+
try {
|
|
885
|
+
await pool.shutdown();
|
|
886
|
+
} catch (error) {
|
|
887
|
+
console.warn(`\u26A0\uFE0F Error shutting down worker pool:`, error);
|
|
888
|
+
}
|
|
889
|
+
}
|
|
890
|
+
});
|
|
891
|
+
let i = 0;
|
|
892
|
+
for (const workerManagerPromise of workerManagerPromises) {
|
|
893
|
+
await workerManagerPromise;
|
|
894
|
+
i++;
|
|
895
|
+
}
|
|
896
|
+
for (const tracker of this.descendantTrackers.values()) {
|
|
897
|
+
tracker.reject(new Error("Scheduler shut down before all descendants completed"));
|
|
898
|
+
}
|
|
899
|
+
this.tasks.clear();
|
|
900
|
+
this.descendantTrackers.clear();
|
|
901
|
+
this.spawnedChildren.clear();
|
|
902
|
+
this.completedTaskIds.clear();
|
|
903
|
+
this.stats = { queued: 0, running: 0, done: 0, failed: 0 };
|
|
904
|
+
}
|
|
905
|
+
};
|
|
906
|
+
|
|
907
|
+
// groups/concurrentLimitGroup.ts
|
|
908
|
+
var ConcurrentLimitGroup = class extends EventTarget {
|
|
909
|
+
id;
|
|
910
|
+
type = "concurrent";
|
|
911
|
+
limit;
|
|
912
|
+
running = 0;
|
|
913
|
+
stats = {
|
|
914
|
+
totalAcquired: 0,
|
|
915
|
+
totalReleased: 0
|
|
916
|
+
};
|
|
917
|
+
constructor(limit, id) {
|
|
918
|
+
super();
|
|
919
|
+
this.limit = limit;
|
|
920
|
+
this.id = id || `concurrent-${Math.random().toString(36).substr(2, 9)}`;
|
|
921
|
+
}
|
|
922
|
+
canRun() {
|
|
923
|
+
return this.running < this.limit;
|
|
924
|
+
}
|
|
925
|
+
getMetrics() {
|
|
926
|
+
return {
|
|
927
|
+
limit: this.limit,
|
|
928
|
+
running: this.running,
|
|
929
|
+
available: Math.max(0, this.limit - this.running),
|
|
930
|
+
utilization: this.running / this.limit
|
|
931
|
+
};
|
|
932
|
+
}
|
|
933
|
+
getStats() {
|
|
934
|
+
return {
|
|
935
|
+
totalAcquired: this.stats.totalAcquired,
|
|
936
|
+
totalReleased: this.stats.totalReleased
|
|
937
|
+
};
|
|
938
|
+
}
|
|
939
|
+
onStart() {
|
|
940
|
+
this.running++;
|
|
941
|
+
this.stats.totalAcquired++;
|
|
942
|
+
if (this.running >= this.limit) {
|
|
943
|
+
this.dispatchEvent(new CustomEvent("capacity-exhausted", {
|
|
944
|
+
detail: { running: this.running, limit: this.limit, groupId: this.id }
|
|
945
|
+
}));
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
onFinish() {
|
|
949
|
+
const wasAtLimit = this.running >= this.limit;
|
|
950
|
+
this.running = Math.max(0, this.running - 1);
|
|
951
|
+
this.stats.totalReleased++;
|
|
952
|
+
if (wasAtLimit && this.canRun()) {
|
|
953
|
+
this.dispatchEvent(new CustomEvent("slot-released", {
|
|
954
|
+
detail: {
|
|
955
|
+
groupType: "concurrent-limit",
|
|
956
|
+
groupId: this.id,
|
|
957
|
+
running: this.running,
|
|
958
|
+
limit: this.limit,
|
|
959
|
+
available: this.limit - this.running
|
|
960
|
+
}
|
|
961
|
+
}));
|
|
962
|
+
}
|
|
963
|
+
}
|
|
964
|
+
};
|
|
965
|
+
|
|
966
|
+
// groups/rateLimitGroup.ts
|
|
967
|
+
var RateLimitGroup = class extends EventTarget {
|
|
968
|
+
id;
|
|
969
|
+
type = "rate-limit";
|
|
970
|
+
windows;
|
|
971
|
+
requestCounts = /* @__PURE__ */ new Map();
|
|
972
|
+
// windowIndex -> [timestamp, count][]
|
|
973
|
+
runningStartTimes = [];
|
|
974
|
+
// Track start times of currently running tasks
|
|
975
|
+
running = 0;
|
|
976
|
+
stats = {
|
|
977
|
+
totalAcquired: 0,
|
|
978
|
+
totalReleased: 0
|
|
979
|
+
};
|
|
980
|
+
constructor(windows, id) {
|
|
981
|
+
super();
|
|
982
|
+
this.windows = windows;
|
|
983
|
+
this.id = id || `ratelimit-${Math.random().toString(36).substr(2, 9)}`;
|
|
984
|
+
this.windows.forEach((_, index) => {
|
|
985
|
+
this.requestCounts.set(index.toString(), []);
|
|
986
|
+
});
|
|
987
|
+
}
|
|
988
|
+
/**
|
|
989
|
+
* Check if a new request can be accepted based on all rate limit windows
|
|
990
|
+
*/
|
|
991
|
+
canRun() {
|
|
992
|
+
const now = Date.now();
|
|
993
|
+
for (let i = 0; i < this.windows.length; i++) {
|
|
994
|
+
const window = this.windows[i];
|
|
995
|
+
const windowKey = i.toString();
|
|
996
|
+
const requests = this.requestCounts.get(windowKey);
|
|
997
|
+
const windowStart = now - window.windowMs;
|
|
998
|
+
const validCompletedRequests = requests.filter(([timestamp]) => timestamp >= windowStart);
|
|
999
|
+
this.requestCounts.set(windowKey, validCompletedRequests);
|
|
1000
|
+
const validRunningRequests = this.runningStartTimes.filter((startTime) => startTime >= windowStart);
|
|
1001
|
+
const totalRequestsInWindow = validCompletedRequests.length + validRunningRequests.length;
|
|
1002
|
+
if (totalRequestsInWindow >= window.limit) {
|
|
1003
|
+
return false;
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
return true;
|
|
1007
|
+
}
|
|
1008
|
+
getMetrics() {
|
|
1009
|
+
const mostRestrictiveWindow = this.windows.reduce((min, w) => w.limit < min.limit ? w : min, this.windows[0]);
|
|
1010
|
+
return {
|
|
1011
|
+
limit: mostRestrictiveWindow.limit,
|
|
1012
|
+
running: this.running,
|
|
1013
|
+
available: Math.max(0, mostRestrictiveWindow.limit - this.running),
|
|
1014
|
+
utilization: this.running / mostRestrictiveWindow.limit
|
|
1015
|
+
};
|
|
1016
|
+
}
|
|
1017
|
+
getStats() {
|
|
1018
|
+
return {
|
|
1019
|
+
totalAcquired: this.stats.totalAcquired,
|
|
1020
|
+
totalReleased: this.stats.totalReleased
|
|
1021
|
+
};
|
|
1022
|
+
}
|
|
1023
|
+
/**
|
|
1024
|
+
* Called when a task starts - tracks running count and reserves slot in rate limit
|
|
1025
|
+
*/
|
|
1026
|
+
onStart() {
|
|
1027
|
+
const now = Date.now();
|
|
1028
|
+
this.running++;
|
|
1029
|
+
this.runningStartTimes.push(now);
|
|
1030
|
+
this.stats.totalAcquired++;
|
|
1031
|
+
}
|
|
1032
|
+
/**
|
|
1033
|
+
* Called when a task finishes - records completion in rate limit windows and removes from running
|
|
1034
|
+
*/
|
|
1035
|
+
onFinish() {
|
|
1036
|
+
const now = Date.now();
|
|
1037
|
+
this.running = Math.max(0, this.running - 1);
|
|
1038
|
+
this.stats.totalReleased++;
|
|
1039
|
+
if (this.runningStartTimes.length > 0) {
|
|
1040
|
+
this.runningStartTimes.shift();
|
|
1041
|
+
}
|
|
1042
|
+
this.windows.forEach((_, index) => {
|
|
1043
|
+
const windowKey = index.toString();
|
|
1044
|
+
const requests = this.requestCounts.get(windowKey);
|
|
1045
|
+
requests.push([now, 1]);
|
|
1046
|
+
});
|
|
1047
|
+
}
|
|
1048
|
+
/**
|
|
1049
|
+
* Get current status for monitoring
|
|
1050
|
+
*/
|
|
1051
|
+
getStatus() {
|
|
1052
|
+
const now = Date.now();
|
|
1053
|
+
const windowStatus = this.windows.map((window, index) => {
|
|
1054
|
+
const windowKey = index.toString();
|
|
1055
|
+
const requests = this.requestCounts.get(windowKey);
|
|
1056
|
+
const windowStart = now - window.windowMs;
|
|
1057
|
+
const completedInWindow = requests.filter(([timestamp]) => timestamp >= windowStart).length;
|
|
1058
|
+
const runningInWindow = this.runningStartTimes.filter((startTime) => startTime >= windowStart).length;
|
|
1059
|
+
const currentCount = completedInWindow + runningInWindow;
|
|
1060
|
+
return {
|
|
1061
|
+
limit: window.limit,
|
|
1062
|
+
windowMs: window.windowMs,
|
|
1063
|
+
current: currentCount,
|
|
1064
|
+
completed: completedInWindow,
|
|
1065
|
+
running: runningInWindow,
|
|
1066
|
+
remaining: Math.max(0, window.limit - currentCount),
|
|
1067
|
+
resetTime: windowStart + window.windowMs
|
|
1068
|
+
};
|
|
1069
|
+
});
|
|
1070
|
+
return {
|
|
1071
|
+
running: this.running,
|
|
1072
|
+
windows: windowStatus,
|
|
1073
|
+
canAcceptNew: this.canRun()
|
|
1074
|
+
};
|
|
1075
|
+
}
|
|
1076
|
+
/**
|
|
1077
|
+
* Clear all rate limit history (useful for testing or reset)
|
|
1078
|
+
*/
|
|
1079
|
+
reset() {
|
|
1080
|
+
this.requestCounts.clear();
|
|
1081
|
+
this.windows.forEach((_, index) => {
|
|
1082
|
+
this.requestCounts.set(index.toString(), []);
|
|
1083
|
+
});
|
|
1084
|
+
this.runningStartTimes = [];
|
|
1085
|
+
this.running = 0;
|
|
1086
|
+
}
|
|
1087
|
+
/**
|
|
1088
|
+
* Helper to create common rate limit configurations
|
|
1089
|
+
*/
|
|
1090
|
+
static createWindows(configs) {
|
|
1091
|
+
return configs.map((config) => ({
|
|
1092
|
+
limit: config.limit,
|
|
1093
|
+
windowMs: config.seconds * 1e3
|
|
1094
|
+
}));
|
|
1095
|
+
}
|
|
1096
|
+
};
|
|
1097
|
+
|
|
1098
|
+
// groups/keyedRateLimitGroup.ts
|
|
1099
|
+
var KeyedRateLimitGroup = class extends EventTarget {
|
|
1100
|
+
id;
|
|
1101
|
+
type = "keyed-rate-limit";
|
|
1102
|
+
keyed = true;
|
|
1103
|
+
windows;
|
|
1104
|
+
keyFrom;
|
|
1105
|
+
idleKeyTtlMs;
|
|
1106
|
+
keys = /* @__PURE__ */ new Map();
|
|
1107
|
+
stats = {
|
|
1108
|
+
totalAcquired: 0,
|
|
1109
|
+
totalReleased: 0
|
|
1110
|
+
};
|
|
1111
|
+
constructor(windows, options = {}) {
|
|
1112
|
+
super();
|
|
1113
|
+
if (!windows.length) throw new Error("KeyedRateLimitGroup requires at least one window");
|
|
1114
|
+
this.windows = windows;
|
|
1115
|
+
this.id = options.id || `keyed-ratelimit-${Math.random().toString(36).substr(2, 9)}`;
|
|
1116
|
+
this.keyFrom = options.keyFrom ?? ((task) => task.limitKey);
|
|
1117
|
+
this.idleKeyTtlMs = options.idleKeyTtlMs ?? Math.max(...windows.map((w) => w.windowMs)) * 2;
|
|
1118
|
+
}
|
|
1119
|
+
/** Bucket this task belongs to. The scheduler throws if this returns nothing. */
|
|
1120
|
+
keyFor(task) {
|
|
1121
|
+
const key = this.keyFrom(task);
|
|
1122
|
+
return key === void 0 || key === null || key === "" ? void 0 : String(key);
|
|
1123
|
+
}
|
|
1124
|
+
/**
|
|
1125
|
+
* Whether one more task may start.
|
|
1126
|
+
*
|
|
1127
|
+
* With a key, checks that key's bucket. Without one - which is how the
|
|
1128
|
+
* scheduler pre-filters before walking blocked queues - reports whether any
|
|
1129
|
+
* known key has capacity, treating an unknown key as free.
|
|
1130
|
+
*/
|
|
1131
|
+
canRun(key) {
|
|
1132
|
+
this._evictIdleKeys();
|
|
1133
|
+
if (key === void 0) {
|
|
1134
|
+
if (this.keys.size === 0) return true;
|
|
1135
|
+
for (const k of this.keys.keys()) {
|
|
1136
|
+
if (this._keyCanRun(k)) return true;
|
|
1137
|
+
}
|
|
1138
|
+
return false;
|
|
1139
|
+
}
|
|
1140
|
+
return this._keyCanRun(key);
|
|
1141
|
+
}
|
|
1142
|
+
onStart(key) {
|
|
1143
|
+
const state = this._stateFor(key ?? "");
|
|
1144
|
+
const now = Date.now();
|
|
1145
|
+
state.running++;
|
|
1146
|
+
state.runningStartTimes.push(now);
|
|
1147
|
+
state.lastActivity = now;
|
|
1148
|
+
this.stats.totalAcquired++;
|
|
1149
|
+
if (!this._keyCanRun(key ?? "")) {
|
|
1150
|
+
this.dispatchEvent(new CustomEvent("capacity-exhausted", {
|
|
1151
|
+
detail: { groupId: this.id, key: key ?? "" }
|
|
1152
|
+
}));
|
|
1153
|
+
}
|
|
1154
|
+
}
|
|
1155
|
+
onFinish(key) {
|
|
1156
|
+
const resolved = key ?? "";
|
|
1157
|
+
const state = this.keys.get(resolved);
|
|
1158
|
+
if (!state) return;
|
|
1159
|
+
const now = Date.now();
|
|
1160
|
+
state.running = Math.max(0, state.running - 1);
|
|
1161
|
+
state.lastActivity = now;
|
|
1162
|
+
this.stats.totalReleased++;
|
|
1163
|
+
if (state.runningStartTimes.length > 0) state.runningStartTimes.shift();
|
|
1164
|
+
this.windows.forEach((_, index) => state.completed[index].push(now));
|
|
1165
|
+
this.dispatchEvent(new CustomEvent("slot-released", {
|
|
1166
|
+
detail: { groupId: this.id, key: resolved }
|
|
1167
|
+
}));
|
|
1168
|
+
}
|
|
1169
|
+
/**
|
|
1170
|
+
* Aggregate view across keys. `limit` is the per-key limit of the most
|
|
1171
|
+
* restrictive window, `running` is summed over every key. Use
|
|
1172
|
+
* {@link KeyedRateLimitGroup.getKeyMetrics} for one bucket.
|
|
1173
|
+
*/
|
|
1174
|
+
getMetrics() {
|
|
1175
|
+
this._evictIdleKeys();
|
|
1176
|
+
const limit = this._tightestLimit();
|
|
1177
|
+
let running = 0;
|
|
1178
|
+
for (const state of this.keys.values()) running += state.running;
|
|
1179
|
+
const capacity = limit * Math.max(1, this.keys.size);
|
|
1180
|
+
return {
|
|
1181
|
+
limit,
|
|
1182
|
+
running,
|
|
1183
|
+
available: Math.max(0, capacity - running),
|
|
1184
|
+
utilization: capacity > 0 ? running / capacity : 0,
|
|
1185
|
+
activeKeys: this.keys.size
|
|
1186
|
+
};
|
|
1187
|
+
}
|
|
1188
|
+
/** Per-bucket view. Returns zeroes for a key that has not been used. */
|
|
1189
|
+
getKeyMetrics(key) {
|
|
1190
|
+
const limit = this._tightestLimit();
|
|
1191
|
+
const state = this.keys.get(key);
|
|
1192
|
+
const running = state?.running ?? 0;
|
|
1193
|
+
return {
|
|
1194
|
+
limit,
|
|
1195
|
+
running,
|
|
1196
|
+
available: Math.max(0, limit - running),
|
|
1197
|
+
utilization: limit > 0 ? running / limit : 0
|
|
1198
|
+
};
|
|
1199
|
+
}
|
|
1200
|
+
/** Keys currently holding state, busiest first. */
|
|
1201
|
+
getActiveKeys() {
|
|
1202
|
+
this._evictIdleKeys();
|
|
1203
|
+
return [...this.keys.entries()].sort((a, b) => b[1].running - a[1].running).map(([key]) => key);
|
|
1204
|
+
}
|
|
1205
|
+
getStats() {
|
|
1206
|
+
return {
|
|
1207
|
+
totalAcquired: this.stats.totalAcquired,
|
|
1208
|
+
totalReleased: this.stats.totalReleased
|
|
1209
|
+
};
|
|
1210
|
+
}
|
|
1211
|
+
_tightestLimit() {
|
|
1212
|
+
return this.windows.reduce((min, w) => Math.min(min, w.limit), this.windows[0].limit);
|
|
1213
|
+
}
|
|
1214
|
+
_stateFor(key) {
|
|
1215
|
+
let state = this.keys.get(key);
|
|
1216
|
+
if (!state) {
|
|
1217
|
+
state = {
|
|
1218
|
+
completed: this.windows.map(() => []),
|
|
1219
|
+
runningStartTimes: [],
|
|
1220
|
+
running: 0,
|
|
1221
|
+
lastActivity: Date.now()
|
|
1222
|
+
};
|
|
1223
|
+
this.keys.set(key, state);
|
|
1224
|
+
}
|
|
1225
|
+
return state;
|
|
1226
|
+
}
|
|
1227
|
+
_keyCanRun(key) {
|
|
1228
|
+
const state = this.keys.get(key);
|
|
1229
|
+
if (!state) return true;
|
|
1230
|
+
const now = Date.now();
|
|
1231
|
+
for (let i = 0; i < this.windows.length; i++) {
|
|
1232
|
+
const window = this.windows[i];
|
|
1233
|
+
const windowStart = now - window.windowMs;
|
|
1234
|
+
const completed = state.completed[i].filter((ts) => ts >= windowStart);
|
|
1235
|
+
state.completed[i] = completed;
|
|
1236
|
+
const running = state.runningStartTimes.filter((ts) => ts >= windowStart).length;
|
|
1237
|
+
if (completed.length + running >= window.limit) return false;
|
|
1238
|
+
}
|
|
1239
|
+
return true;
|
|
1240
|
+
}
|
|
1241
|
+
/**
|
|
1242
|
+
* Drop keys that hold no running tasks and have been quiet longer than
|
|
1243
|
+
* `idleKeyTtlMs`. Their windows can no longer constrain anything, so the state
|
|
1244
|
+
* is pure memory. Swept lazily rather than on a timer, so an idle group costs
|
|
1245
|
+
* nothing.
|
|
1246
|
+
*/
|
|
1247
|
+
_evictIdleKeys() {
|
|
1248
|
+
if (this.keys.size === 0) return;
|
|
1249
|
+
const cutoff = Date.now() - this.idleKeyTtlMs;
|
|
1250
|
+
for (const [key, state] of this.keys) {
|
|
1251
|
+
if (state.running === 0 && state.lastActivity < cutoff) {
|
|
1252
|
+
this.keys.delete(key);
|
|
1253
|
+
}
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
};
|
|
1257
|
+
|
|
1258
|
+
// core/threadWrapper.ts
|
|
1259
|
+
import { Worker } from "node:worker_threads";
|
|
1260
|
+
|
|
1261
|
+
// core/workerWrapperUrl.bundled.ts
|
|
1262
|
+
var url2;
|
|
1263
|
+
async function getWorkerUrl() {
|
|
1264
|
+
if (url2) return url2;
|
|
1265
|
+
url2 = (await Promise.resolve().then(() => (init_workerWrapper_node(), workerWrapper_node_exports))).default;
|
|
1266
|
+
return url2;
|
|
1267
|
+
}
|
|
1268
|
+
|
|
1269
|
+
// core/workerInterface.ts
|
|
1270
|
+
var WorkerTerminationError = class extends Error {
|
|
1271
|
+
metadata;
|
|
1272
|
+
constructor(message, metadata = {}) {
|
|
1273
|
+
super(message);
|
|
1274
|
+
this.name = "WorkerTerminationError";
|
|
1275
|
+
this.metadata = metadata;
|
|
1276
|
+
}
|
|
1277
|
+
};
|
|
1278
|
+
var BaseWorker = class {
|
|
1279
|
+
config;
|
|
1280
|
+
workerContext;
|
|
1281
|
+
constructor(config = {}, workerContext) {
|
|
1282
|
+
this.config = config;
|
|
1283
|
+
this.workerContext = workerContext;
|
|
1284
|
+
}
|
|
1285
|
+
};
|
|
1286
|
+
|
|
1287
|
+
// core/threadWrapper.ts
|
|
1288
|
+
var ThreadWrapper = class extends EventTarget {
|
|
1289
|
+
worker = null;
|
|
1290
|
+
runningTasks = 0;
|
|
1291
|
+
maxConcurrentTasks;
|
|
1292
|
+
idleTimeout;
|
|
1293
|
+
callbacks = /* @__PURE__ */ new Map();
|
|
1294
|
+
taskStartTimes = /* @__PURE__ */ new Map();
|
|
1295
|
+
// Track task execution start times
|
|
1296
|
+
runningTaskData = /* @__PURE__ */ new Map();
|
|
1297
|
+
// Track task data for requeuing
|
|
1298
|
+
lastActivityTime = Date.now();
|
|
1299
|
+
config;
|
|
1300
|
+
id;
|
|
1301
|
+
initialized = false;
|
|
1302
|
+
originalScriptUrl;
|
|
1303
|
+
initializing = false;
|
|
1304
|
+
// Internal task queue to prevent worker message queue flooding
|
|
1305
|
+
taskQueue = [];
|
|
1306
|
+
// WorkerInstanceState implementation
|
|
1307
|
+
state = "initializing";
|
|
1308
|
+
lastError;
|
|
1309
|
+
// Stats for WorkerStatus
|
|
1310
|
+
tasksCompleted = 0;
|
|
1311
|
+
errorCount = 0;
|
|
1312
|
+
createdAt = Date.now();
|
|
1313
|
+
initStartTime = 0;
|
|
1314
|
+
teardownStartTime = 0;
|
|
1315
|
+
// Pending termination request for post-task handling
|
|
1316
|
+
pendingTerminationRequest = null;
|
|
1317
|
+
//DENO SPECIFIC USE OF WORKER, node uses worker_threads
|
|
1318
|
+
constructor(scriptUrl, idleTimeout = 5e3, maxConcurrentTasks = 1, config = {}) {
|
|
1319
|
+
super();
|
|
1320
|
+
this.originalScriptUrl = (scriptUrl || "").toString();
|
|
1321
|
+
this.idleTimeout = idleTimeout;
|
|
1322
|
+
this.maxConcurrentTasks = maxConcurrentTasks;
|
|
1323
|
+
this.config = config;
|
|
1324
|
+
this.id = `thread-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
|
1325
|
+
}
|
|
1326
|
+
canAcceptTask() {
|
|
1327
|
+
const totalTasks = this.runningTasks + this.taskQueue.length;
|
|
1328
|
+
return totalTasks < this.maxConcurrentTasks && !this.initializing && this.state !== "failed" && this.state !== "terminated";
|
|
1329
|
+
}
|
|
1330
|
+
// WorkerInstanceExtensions implementation
|
|
1331
|
+
getRunningTaskIds() {
|
|
1332
|
+
const runningTaskIds = Array.from(this.callbacks.keys());
|
|
1333
|
+
const queuedTaskIds = this.taskQueue.map((task) => task.taskId);
|
|
1334
|
+
return [...runningTaskIds, ...queuedTaskIds];
|
|
1335
|
+
}
|
|
1336
|
+
getRunningTasks() {
|
|
1337
|
+
const runningTasks = Array.from(this.runningTaskData.values());
|
|
1338
|
+
const queuedTasks = this.taskQueue.map((task) => ({ id: task.taskId, payload: task.payload }));
|
|
1339
|
+
return [...runningTasks, ...queuedTasks];
|
|
1340
|
+
}
|
|
1341
|
+
// terminateWithError implementation - throws error for clean task completion
|
|
1342
|
+
createTerminateWithError() {
|
|
1343
|
+
return (error, metadata = {}) => {
|
|
1344
|
+
this.pendingTerminationRequest = {
|
|
1345
|
+
error,
|
|
1346
|
+
metadata: {
|
|
1347
|
+
failureType: "runtime",
|
|
1348
|
+
canRestart: metadata.canRestart,
|
|
1349
|
+
restartDelay: metadata.restartDelay,
|
|
1350
|
+
...metadata
|
|
1351
|
+
},
|
|
1352
|
+
timestamp: Date.now()
|
|
1353
|
+
};
|
|
1354
|
+
throw new WorkerTerminationError(error.message, metadata);
|
|
1355
|
+
};
|
|
1356
|
+
}
|
|
1357
|
+
// Handle termination request after task completion
|
|
1358
|
+
_handleTerminationRequest(metadata) {
|
|
1359
|
+
this.state = "failed";
|
|
1360
|
+
this.lastError = {
|
|
1361
|
+
timestamp: Date.now(),
|
|
1362
|
+
message: "Worker termination requested",
|
|
1363
|
+
metadata
|
|
1364
|
+
};
|
|
1365
|
+
this.errorCount++;
|
|
1366
|
+
this.dispatchEvent(new CustomEvent("worker.failed", {
|
|
1367
|
+
detail: {
|
|
1368
|
+
workerId: this.id,
|
|
1369
|
+
error: new Error("Worker termination requested"),
|
|
1370
|
+
metadata: {
|
|
1371
|
+
failureType: "self_termination",
|
|
1372
|
+
...metadata
|
|
1373
|
+
},
|
|
1374
|
+
timestamp: Date.now()
|
|
1375
|
+
}
|
|
1376
|
+
}));
|
|
1377
|
+
this.dispatchEvent(new CustomEvent("worker.termination_requested", {
|
|
1378
|
+
detail: {
|
|
1379
|
+
workerId: this.id,
|
|
1380
|
+
metadata,
|
|
1381
|
+
timestamp: Date.now()
|
|
1382
|
+
}
|
|
1383
|
+
}));
|
|
1384
|
+
this.dispatchEvent(new CustomEvent("worker.self_terminated", {
|
|
1385
|
+
detail: {
|
|
1386
|
+
workerId: this.id,
|
|
1387
|
+
metadata,
|
|
1388
|
+
timestamp: Date.now()
|
|
1389
|
+
}
|
|
1390
|
+
}));
|
|
1391
|
+
}
|
|
1392
|
+
// Set up detection for unexpected worker process exits
|
|
1393
|
+
_setupUnexpectedExitDetection() {
|
|
1394
|
+
if (!this.worker) return;
|
|
1395
|
+
this.worker.onerror = (error) => {
|
|
1396
|
+
this._handleUnexpectedExit("error", error);
|
|
1397
|
+
};
|
|
1398
|
+
if ("onclose" in this.worker) {
|
|
1399
|
+
this.worker.onclose = () => {
|
|
1400
|
+
this._handleUnexpectedExit("close", new Error("Worker process closed unexpectedly"));
|
|
1401
|
+
};
|
|
1402
|
+
}
|
|
1403
|
+
this.worker.onmessageerror = (error) => {
|
|
1404
|
+
this._handleUnexpectedExit("messageerror", error);
|
|
1405
|
+
};
|
|
1406
|
+
}
|
|
1407
|
+
// Handle unexpected worker exit/error
|
|
1408
|
+
_handleUnexpectedExit(type, error) {
|
|
1409
|
+
if (this.state === "terminated" || this.state === "failed") {
|
|
1410
|
+
return;
|
|
1411
|
+
}
|
|
1412
|
+
this.state = "failed";
|
|
1413
|
+
this.lastError = {
|
|
1414
|
+
timestamp: Date.now(),
|
|
1415
|
+
message: `Worker ${type}: ${error.message || error}`,
|
|
1416
|
+
metadata: { failureType: "unexpected_exit", exitType: type }
|
|
1417
|
+
};
|
|
1418
|
+
this.errorCount++;
|
|
1419
|
+
this.dispatchEvent(new CustomEvent("worker.failed", {
|
|
1420
|
+
detail: {
|
|
1421
|
+
workerId: this.id,
|
|
1422
|
+
error: new Error(`Worker ${type}: ${error.message || error}`),
|
|
1423
|
+
metadata: {
|
|
1424
|
+
failureType: "unexpected_exit",
|
|
1425
|
+
exitType: type,
|
|
1426
|
+
canRestart: true,
|
|
1427
|
+
// Unexpected exits can typically be restarted
|
|
1428
|
+
restartDelay: 1e3
|
|
1429
|
+
},
|
|
1430
|
+
timestamp: Date.now()
|
|
1431
|
+
}
|
|
1432
|
+
}));
|
|
1433
|
+
this.worker = null;
|
|
1434
|
+
this.initialized = false;
|
|
1435
|
+
this.runningTasks = 0;
|
|
1436
|
+
this.callbacks.clear();
|
|
1437
|
+
this.taskStartTimes.clear();
|
|
1438
|
+
this.runningTaskData.clear();
|
|
1439
|
+
}
|
|
1440
|
+
// Create BaseWorkerContext for worker constructor
|
|
1441
|
+
createBaseWorkerContext() {
|
|
1442
|
+
return {
|
|
1443
|
+
workerId: this.id,
|
|
1444
|
+
terminateWithError: this.createTerminateWithError()
|
|
1445
|
+
};
|
|
1446
|
+
}
|
|
1447
|
+
get idle() {
|
|
1448
|
+
return this.runningTasks === 0 && this.taskQueue.length === 0;
|
|
1449
|
+
}
|
|
1450
|
+
updateActivity() {
|
|
1451
|
+
this.lastActivityTime = Date.now();
|
|
1452
|
+
}
|
|
1453
|
+
async _ensureInitialized() {
|
|
1454
|
+
if (this.initialized) return;
|
|
1455
|
+
this.initializing = true;
|
|
1456
|
+
const workerWrapperURL = await getWorkerUrl();
|
|
1457
|
+
this.worker = new Worker(workerWrapperURL, { type: "module" });
|
|
1458
|
+
this._setupUnexpectedExitDetection();
|
|
1459
|
+
this.worker.onmessage = (e) => {
|
|
1460
|
+
const message = e.data ? e.data : e;
|
|
1461
|
+
const { type, taskId, data } = message;
|
|
1462
|
+
switch (type) {
|
|
1463
|
+
case "init":
|
|
1464
|
+
this.initialized = true;
|
|
1465
|
+
this.dispatchEvent(new CustomEvent("worker.initialization.completed", {
|
|
1466
|
+
detail: {
|
|
1467
|
+
workerId: this.id,
|
|
1468
|
+
workerType: "thread",
|
|
1469
|
+
timestamp: Date.now(),
|
|
1470
|
+
duration: this.initStartTime ? performance.now() - this.initStartTime : 0
|
|
1471
|
+
}
|
|
1472
|
+
}));
|
|
1473
|
+
return;
|
|
1474
|
+
// Setup runs inside the worker thread, which reports its timing back so
|
|
1475
|
+
// threaded workers emit the same setup events inline workers do
|
|
1476
|
+
case "setup_started":
|
|
1477
|
+
this.dispatchEvent(new CustomEvent("worker.setup.started", {
|
|
1478
|
+
detail: { workerId: this.id, workerType: "thread", timestamp: Date.now() }
|
|
1479
|
+
}));
|
|
1480
|
+
return;
|
|
1481
|
+
case "setup_completed":
|
|
1482
|
+
this.dispatchEvent(new CustomEvent("worker.setup.completed", {
|
|
1483
|
+
detail: {
|
|
1484
|
+
workerId: this.id,
|
|
1485
|
+
workerType: "thread",
|
|
1486
|
+
timestamp: Date.now(),
|
|
1487
|
+
duration: data?.duration ?? 0
|
|
1488
|
+
}
|
|
1489
|
+
}));
|
|
1490
|
+
return;
|
|
1491
|
+
case "teardown":
|
|
1492
|
+
this.dispatchEvent(new CustomEvent("worker.teardown.completed", {
|
|
1493
|
+
detail: {
|
|
1494
|
+
workerId: this.id,
|
|
1495
|
+
workerType: "thread",
|
|
1496
|
+
timestamp: Date.now(),
|
|
1497
|
+
duration: this.teardownStartTime ? performance.now() - this.teardownStartTime : 0
|
|
1498
|
+
}
|
|
1499
|
+
}));
|
|
1500
|
+
return;
|
|
1501
|
+
case "progress":
|
|
1502
|
+
if (taskId) {
|
|
1503
|
+
this.dispatchEvent(new CustomEvent("task.progress", {
|
|
1504
|
+
detail: {
|
|
1505
|
+
taskId,
|
|
1506
|
+
workerId: this.id,
|
|
1507
|
+
workerType: "thread",
|
|
1508
|
+
progress: data.progress,
|
|
1509
|
+
message: data.message,
|
|
1510
|
+
details: data.details,
|
|
1511
|
+
timestamp: Date.now()
|
|
1512
|
+
}
|
|
1513
|
+
}));
|
|
1514
|
+
}
|
|
1515
|
+
return;
|
|
1516
|
+
case "spawn_task":
|
|
1517
|
+
if (taskId) {
|
|
1518
|
+
this.dispatchEvent(new CustomEvent("task.spawn_request", {
|
|
1519
|
+
detail: {
|
|
1520
|
+
taskId,
|
|
1521
|
+
workerId: this.id,
|
|
1522
|
+
workerType: "thread",
|
|
1523
|
+
spawnConfig: data,
|
|
1524
|
+
timestamp: Date.now()
|
|
1525
|
+
}
|
|
1526
|
+
}));
|
|
1527
|
+
}
|
|
1528
|
+
return;
|
|
1529
|
+
case "result":
|
|
1530
|
+
case "error": {
|
|
1531
|
+
if (data.isWorkerTerminationRequest) {
|
|
1532
|
+
this._handleTerminationRequest(data.metadata);
|
|
1533
|
+
if (taskId !== "worker_termination_request") {
|
|
1534
|
+
const cb2 = this.callbacks.get(taskId);
|
|
1535
|
+
if (cb2) {
|
|
1536
|
+
this.callbacks.delete(taskId);
|
|
1537
|
+
cb2.reject(new Error(data.message));
|
|
1538
|
+
}
|
|
1539
|
+
}
|
|
1540
|
+
return;
|
|
1541
|
+
}
|
|
1542
|
+
if (taskId === "init") {
|
|
1543
|
+
this.initializing = false;
|
|
1544
|
+
const initError = new Error(data?.message || String(data));
|
|
1545
|
+
this.dispatchEvent(new CustomEvent("worker.initialization.failed", {
|
|
1546
|
+
detail: {
|
|
1547
|
+
workerId: this.id,
|
|
1548
|
+
workerType: "thread",
|
|
1549
|
+
timestamp: Date.now(),
|
|
1550
|
+
error: initError
|
|
1551
|
+
}
|
|
1552
|
+
}));
|
|
1553
|
+
this.dispatchEvent(new CustomEvent("worker.failed", {
|
|
1554
|
+
detail: {
|
|
1555
|
+
workerId: this.id,
|
|
1556
|
+
error: initError,
|
|
1557
|
+
metadata: { failureType: "initialization", stage: "constructor" },
|
|
1558
|
+
timestamp: Date.now()
|
|
1559
|
+
}
|
|
1560
|
+
}));
|
|
1561
|
+
return;
|
|
1562
|
+
}
|
|
1563
|
+
const cb = this.callbacks.get(taskId);
|
|
1564
|
+
if (!cb) return;
|
|
1565
|
+
const startTime = this.taskStartTimes.get(taskId);
|
|
1566
|
+
const executionTime = startTime ? performance.now() - startTime : 0;
|
|
1567
|
+
this.callbacks.delete(taskId);
|
|
1568
|
+
this.taskStartTimes.delete(taskId);
|
|
1569
|
+
this.runningTaskData.delete(taskId);
|
|
1570
|
+
this.runningTasks--;
|
|
1571
|
+
this.tasksCompleted++;
|
|
1572
|
+
if (this.state !== "failed" && this.state !== "terminated") {
|
|
1573
|
+
this.state = this.runningTasks > 0 ? "busy" : "healthy";
|
|
1574
|
+
}
|
|
1575
|
+
if (type === "error") {
|
|
1576
|
+
this.dispatchEvent(new CustomEvent("task.failed", {
|
|
1577
|
+
detail: { taskId, workerId: this.id, workerType: "thread", error: data, timestamp: Date.now(), executionTime }
|
|
1578
|
+
}));
|
|
1579
|
+
cb.reject(new Error(data.message || data));
|
|
1580
|
+
} else {
|
|
1581
|
+
this.dispatchEvent(new CustomEvent("task.completed", {
|
|
1582
|
+
detail: { taskId, workerId: this.id, workerType: "thread", result: data, timestamp: Date.now(), executionTime }
|
|
1583
|
+
}));
|
|
1584
|
+
cb.resolve(data);
|
|
1585
|
+
}
|
|
1586
|
+
this._processTaskQueue();
|
|
1587
|
+
this.updateActivity();
|
|
1588
|
+
return;
|
|
1589
|
+
}
|
|
1590
|
+
default:
|
|
1591
|
+
console.warn(`Unknown message type from worker: ${type}`);
|
|
1592
|
+
}
|
|
1593
|
+
};
|
|
1594
|
+
if ("on" in this.worker) {
|
|
1595
|
+
this.worker.on("message", (message) => {
|
|
1596
|
+
this?.worker?.onmessage?.({ data: message });
|
|
1597
|
+
});
|
|
1598
|
+
}
|
|
1599
|
+
this.initStartTime = performance.now();
|
|
1600
|
+
this.dispatchEvent(new CustomEvent("worker.initialization.started", {
|
|
1601
|
+
detail: { workerId: this.id, workerType: "thread", timestamp: Date.now() }
|
|
1602
|
+
}));
|
|
1603
|
+
return new Promise((resolve, reject) => {
|
|
1604
|
+
const timeout = setTimeout(() => {
|
|
1605
|
+
this.initializing = false;
|
|
1606
|
+
this.removeEventListener("worker.initialization.completed", onInit);
|
|
1607
|
+
const error = new Error("Worker initialization timeout");
|
|
1608
|
+
this.dispatchEvent(new CustomEvent("worker.initialization.failed", {
|
|
1609
|
+
detail: { workerId: this.id, workerType: "thread", timestamp: Date.now(), error }
|
|
1610
|
+
}));
|
|
1611
|
+
reject(error);
|
|
1612
|
+
}, 3e4);
|
|
1613
|
+
const onInit = () => {
|
|
1614
|
+
clearTimeout(timeout);
|
|
1615
|
+
this.initializing = false;
|
|
1616
|
+
this.removeEventListener("worker.initialization.completed", onInit);
|
|
1617
|
+
this.removeEventListener("worker.initialization.failed", onInitFailed);
|
|
1618
|
+
resolve();
|
|
1619
|
+
};
|
|
1620
|
+
const onInitFailed = (e) => {
|
|
1621
|
+
clearTimeout(timeout);
|
|
1622
|
+
this.initializing = false;
|
|
1623
|
+
this.removeEventListener("worker.initialization.completed", onInit);
|
|
1624
|
+
this.removeEventListener("worker.initialization.failed", onInitFailed);
|
|
1625
|
+
reject(e.detail?.error ?? new Error("Worker initialization failed"));
|
|
1626
|
+
};
|
|
1627
|
+
this.addEventListener("worker.initialization.completed", onInit);
|
|
1628
|
+
this.addEventListener("worker.initialization.failed", onInitFailed);
|
|
1629
|
+
const absoluteWorkerUrl = this.originalScriptUrl.startsWith("http") || this.originalScriptUrl.startsWith("file:") ? this.originalScriptUrl : new URL(this.originalScriptUrl, import.meta.url).href;
|
|
1630
|
+
this.worker?.postMessage({
|
|
1631
|
+
taskId: "init",
|
|
1632
|
+
action: "init",
|
|
1633
|
+
config: this.config,
|
|
1634
|
+
workerUrl: absoluteWorkerUrl,
|
|
1635
|
+
workerId: this.id
|
|
1636
|
+
});
|
|
1637
|
+
});
|
|
1638
|
+
}
|
|
1639
|
+
async runTask(taskId, payload) {
|
|
1640
|
+
if (!this.canAcceptTask()) {
|
|
1641
|
+
throw new Error("Worker at maximum concurrent task capacity");
|
|
1642
|
+
}
|
|
1643
|
+
return new Promise((resolve, reject) => {
|
|
1644
|
+
this.taskQueue.push({ taskId, payload, resolve, reject });
|
|
1645
|
+
this._ensureInitialized().then(() => {
|
|
1646
|
+
this.updateActivity();
|
|
1647
|
+
this._processTaskQueue();
|
|
1648
|
+
}).catch((error) => {
|
|
1649
|
+
const index = this.taskQueue.findIndex((t) => t.taskId === taskId);
|
|
1650
|
+
if (index !== -1) this.taskQueue.splice(index, 1);
|
|
1651
|
+
reject(error);
|
|
1652
|
+
});
|
|
1653
|
+
});
|
|
1654
|
+
}
|
|
1655
|
+
_processTaskQueue() {
|
|
1656
|
+
while (this.taskQueue.length > 0 && this.runningTasks < this.maxConcurrentTasks && this.initialized) {
|
|
1657
|
+
const task = this.taskQueue.shift();
|
|
1658
|
+
this.runningTasks++;
|
|
1659
|
+
this.dispatchEvent(new CustomEvent("task.started", {
|
|
1660
|
+
detail: { taskId: task.taskId, workerId: this.id, workerType: "thread", timestamp: Date.now() }
|
|
1661
|
+
}));
|
|
1662
|
+
this.callbacks.set(task.taskId, { resolve: task.resolve, reject: task.reject });
|
|
1663
|
+
this.taskStartTimes.set(task.taskId, performance.now());
|
|
1664
|
+
this.runningTaskData.set(task.taskId, { id: task.taskId, payload: task.payload });
|
|
1665
|
+
this.worker?.postMessage({ taskId: task.taskId, payload: task.payload, action: "run" });
|
|
1666
|
+
}
|
|
1667
|
+
}
|
|
1668
|
+
async terminate() {
|
|
1669
|
+
if (this.initialized) {
|
|
1670
|
+
this.teardownStartTime = performance.now();
|
|
1671
|
+
this.dispatchEvent(new CustomEvent("worker.teardown.started", {
|
|
1672
|
+
detail: { workerId: this.id, workerType: "thread", timestamp: Date.now() }
|
|
1673
|
+
}));
|
|
1674
|
+
try {
|
|
1675
|
+
this.worker?.postMessage({ taskId: "teardown", action: "teardown" });
|
|
1676
|
+
await new Promise((resolve) => {
|
|
1677
|
+
const timeout = setTimeout(resolve, 1e3);
|
|
1678
|
+
const onTeardown = () => {
|
|
1679
|
+
clearTimeout(timeout);
|
|
1680
|
+
this.removeEventListener("worker.teardown.completed", onTeardown);
|
|
1681
|
+
resolve();
|
|
1682
|
+
};
|
|
1683
|
+
this.addEventListener("worker.teardown.completed", onTeardown);
|
|
1684
|
+
});
|
|
1685
|
+
} catch (error) {
|
|
1686
|
+
this.dispatchEvent(new CustomEvent("worker.teardown.failed", {
|
|
1687
|
+
detail: { workerId: this.id, workerType: "thread", timestamp: Date.now(), error }
|
|
1688
|
+
}));
|
|
1689
|
+
}
|
|
1690
|
+
}
|
|
1691
|
+
this.worker?.terminate();
|
|
1692
|
+
this.worker = null;
|
|
1693
|
+
this.runningTasks = 0;
|
|
1694
|
+
this.callbacks.clear();
|
|
1695
|
+
this.initialized = false;
|
|
1696
|
+
}
|
|
1697
|
+
};
|
|
1698
|
+
|
|
1699
|
+
// core/inlineWrapper.ts
|
|
1700
|
+
var InlineWrapper = class extends EventTarget {
|
|
1701
|
+
runningTasks = 0;
|
|
1702
|
+
maxConcurrentTasks;
|
|
1703
|
+
runningTaskIds = /* @__PURE__ */ new Set();
|
|
1704
|
+
runningTaskData = /* @__PURE__ */ new Map();
|
|
1705
|
+
idleTimeout;
|
|
1706
|
+
lastActivityTime = Date.now();
|
|
1707
|
+
workerModule = null;
|
|
1708
|
+
workerInstance = null;
|
|
1709
|
+
scriptUrl;
|
|
1710
|
+
config;
|
|
1711
|
+
id;
|
|
1712
|
+
initializing = false;
|
|
1713
|
+
// WorkerInstanceState implementation
|
|
1714
|
+
state = "initializing";
|
|
1715
|
+
lastError;
|
|
1716
|
+
// Stats for WorkerStatus
|
|
1717
|
+
tasksCompleted = 0;
|
|
1718
|
+
errorCount = 0;
|
|
1719
|
+
createdAt = Date.now();
|
|
1720
|
+
constructor(scriptUrl, idleTimeout = 5e3, maxConcurrentTasks = 1, config = {}) {
|
|
1721
|
+
super();
|
|
1722
|
+
this.scriptUrl = scriptUrl;
|
|
1723
|
+
this.idleTimeout = idleTimeout;
|
|
1724
|
+
this.maxConcurrentTasks = maxConcurrentTasks;
|
|
1725
|
+
this.config = config;
|
|
1726
|
+
this.id = `inline-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
|
1727
|
+
}
|
|
1728
|
+
canAcceptTask() {
|
|
1729
|
+
return this.runningTasks < this.maxConcurrentTasks && !this.initializing && this.state !== "failed" && this.state !== "terminated";
|
|
1730
|
+
}
|
|
1731
|
+
// WorkerInstanceExtensions implementation
|
|
1732
|
+
getRunningTaskIds() {
|
|
1733
|
+
return Array.from(this.runningTaskIds);
|
|
1734
|
+
}
|
|
1735
|
+
getRunningTasks() {
|
|
1736
|
+
return Array.from(this.runningTaskData.values());
|
|
1737
|
+
}
|
|
1738
|
+
// terminateWithError implementation - throws error for clean task completion
|
|
1739
|
+
createTerminateWithError() {
|
|
1740
|
+
return (error, metadata = {}) => {
|
|
1741
|
+
throw new WorkerTerminationError(error.message, metadata);
|
|
1742
|
+
};
|
|
1743
|
+
}
|
|
1744
|
+
// Handle termination request after task completion
|
|
1745
|
+
_handleTerminationRequest(metadata) {
|
|
1746
|
+
this.state = "failed";
|
|
1747
|
+
this.lastError = {
|
|
1748
|
+
timestamp: Date.now(),
|
|
1749
|
+
message: "Worker termination requested",
|
|
1750
|
+
metadata
|
|
1751
|
+
};
|
|
1752
|
+
this.errorCount++;
|
|
1753
|
+
this.dispatchEvent(new CustomEvent("worker.failed", {
|
|
1754
|
+
detail: {
|
|
1755
|
+
workerId: this.id,
|
|
1756
|
+
error: new Error("Worker termination requested"),
|
|
1757
|
+
metadata: {
|
|
1758
|
+
failureType: "self_termination",
|
|
1759
|
+
...metadata
|
|
1760
|
+
},
|
|
1761
|
+
timestamp: Date.now()
|
|
1762
|
+
}
|
|
1763
|
+
}));
|
|
1764
|
+
this.dispatchEvent(new CustomEvent("worker.termination_requested", {
|
|
1765
|
+
detail: {
|
|
1766
|
+
workerId: this.id,
|
|
1767
|
+
metadata,
|
|
1768
|
+
timestamp: Date.now()
|
|
1769
|
+
}
|
|
1770
|
+
}));
|
|
1771
|
+
this.dispatchEvent(new CustomEvent("worker.self_terminated", {
|
|
1772
|
+
detail: {
|
|
1773
|
+
workerId: this.id,
|
|
1774
|
+
metadata,
|
|
1775
|
+
timestamp: Date.now()
|
|
1776
|
+
}
|
|
1777
|
+
}));
|
|
1778
|
+
}
|
|
1779
|
+
// Create BaseWorkerContext for worker constructor
|
|
1780
|
+
createBaseWorkerContext() {
|
|
1781
|
+
return {
|
|
1782
|
+
workerId: this.id,
|
|
1783
|
+
terminateWithError: this.createTerminateWithError()
|
|
1784
|
+
};
|
|
1785
|
+
}
|
|
1786
|
+
get idle() {
|
|
1787
|
+
return this.runningTasks === 0;
|
|
1788
|
+
}
|
|
1789
|
+
updateActivity() {
|
|
1790
|
+
this.lastActivityTime = Date.now();
|
|
1791
|
+
}
|
|
1792
|
+
async _loadWorkerModule() {
|
|
1793
|
+
if (!this.workerModule) {
|
|
1794
|
+
this.workerModule = await import(this.scriptUrl);
|
|
1795
|
+
}
|
|
1796
|
+
return this.workerModule;
|
|
1797
|
+
}
|
|
1798
|
+
async _createWorkerInstance() {
|
|
1799
|
+
if (this.workerInstance) {
|
|
1800
|
+
return this.workerInstance;
|
|
1801
|
+
}
|
|
1802
|
+
this.initializing = true;
|
|
1803
|
+
let instance = null;
|
|
1804
|
+
try {
|
|
1805
|
+
const module = await this._loadWorkerModule();
|
|
1806
|
+
if (!module.default || typeof module.default !== "function" || !module.default.prototype) {
|
|
1807
|
+
throw new Error(`Worker script ${this.scriptUrl} must export a default class`);
|
|
1808
|
+
}
|
|
1809
|
+
const initStartTime = performance.now();
|
|
1810
|
+
this.dispatchEvent(new CustomEvent("worker.initialization.started", {
|
|
1811
|
+
detail: { workerId: this.id, workerType: "inline", timestamp: Date.now() }
|
|
1812
|
+
}));
|
|
1813
|
+
const baseWorkerContext = this.createBaseWorkerContext();
|
|
1814
|
+
instance = new module.default(this.config, baseWorkerContext);
|
|
1815
|
+
if (instance.setup) {
|
|
1816
|
+
this.dispatchEvent(new CustomEvent("worker.setup.started", {
|
|
1817
|
+
detail: { workerId: this.id, workerType: "inline", timestamp: Date.now() }
|
|
1818
|
+
}));
|
|
1819
|
+
const setupStartTime = performance.now();
|
|
1820
|
+
await instance.setup();
|
|
1821
|
+
this.dispatchEvent(new CustomEvent("worker.setup.completed", {
|
|
1822
|
+
detail: {
|
|
1823
|
+
workerId: this.id,
|
|
1824
|
+
workerType: "inline",
|
|
1825
|
+
timestamp: Date.now(),
|
|
1826
|
+
duration: performance.now() - setupStartTime
|
|
1827
|
+
}
|
|
1828
|
+
}));
|
|
1829
|
+
}
|
|
1830
|
+
this.dispatchEvent(new CustomEvent("worker.initialization.completed", {
|
|
1831
|
+
detail: {
|
|
1832
|
+
workerId: this.id,
|
|
1833
|
+
workerType: "inline",
|
|
1834
|
+
timestamp: Date.now(),
|
|
1835
|
+
duration: performance.now() - initStartTime
|
|
1836
|
+
}
|
|
1837
|
+
}));
|
|
1838
|
+
this.workerInstance = instance;
|
|
1839
|
+
this.initializing = false;
|
|
1840
|
+
return instance;
|
|
1841
|
+
} catch (error) {
|
|
1842
|
+
if (instance?.teardown) {
|
|
1843
|
+
try {
|
|
1844
|
+
await instance.teardown();
|
|
1845
|
+
} catch (teardownError) {
|
|
1846
|
+
console.warn("Teardown failed during initialization cleanup:", teardownError);
|
|
1847
|
+
}
|
|
1848
|
+
}
|
|
1849
|
+
this.state = "failed";
|
|
1850
|
+
this.lastError = {
|
|
1851
|
+
timestamp: Date.now(),
|
|
1852
|
+
message: error instanceof Error ? error.message : String(error),
|
|
1853
|
+
metadata: { stage: instance ? "setup" : "constructor" }
|
|
1854
|
+
};
|
|
1855
|
+
this.errorCount++;
|
|
1856
|
+
this.dispatchEvent(new CustomEvent("worker.initialization.failed", {
|
|
1857
|
+
detail: {
|
|
1858
|
+
workerId: this.id,
|
|
1859
|
+
workerType: "inline",
|
|
1860
|
+
timestamp: Date.now(),
|
|
1861
|
+
error,
|
|
1862
|
+
stage: instance ? "setup" : "constructor"
|
|
1863
|
+
}
|
|
1864
|
+
}));
|
|
1865
|
+
this.dispatchEvent(new CustomEvent("worker.failed", {
|
|
1866
|
+
detail: {
|
|
1867
|
+
workerId: this.id,
|
|
1868
|
+
error,
|
|
1869
|
+
metadata: {
|
|
1870
|
+
failureType: "initialization",
|
|
1871
|
+
stage: instance ? "setup" : "constructor"
|
|
1872
|
+
},
|
|
1873
|
+
timestamp: Date.now()
|
|
1874
|
+
}
|
|
1875
|
+
}));
|
|
1876
|
+
this.initializing = false;
|
|
1877
|
+
throw error;
|
|
1878
|
+
}
|
|
1879
|
+
}
|
|
1880
|
+
async runTask(taskId, payload) {
|
|
1881
|
+
if (!this.canAcceptTask()) {
|
|
1882
|
+
throw new Error("Worker at maximum concurrent task capacity");
|
|
1883
|
+
}
|
|
1884
|
+
this.updateActivity();
|
|
1885
|
+
this.runningTasks++;
|
|
1886
|
+
this.runningTaskIds.add(taskId);
|
|
1887
|
+
this.runningTaskData.set(taskId, { id: taskId, payload });
|
|
1888
|
+
this.dispatchEvent(new CustomEvent("task.started", {
|
|
1889
|
+
detail: { taskId, workerId: this.id, workerType: "inline", timestamp: Date.now() }
|
|
1890
|
+
}));
|
|
1891
|
+
let fullExecutionStart;
|
|
1892
|
+
try {
|
|
1893
|
+
const workerInstance = await this._createWorkerInstance();
|
|
1894
|
+
fullExecutionStart = performance.now();
|
|
1895
|
+
const context = {
|
|
1896
|
+
// Worker-level context (inherited from BaseWorkerContext)
|
|
1897
|
+
workerId: this.id,
|
|
1898
|
+
terminateWithError: this.createTerminateWithError(),
|
|
1899
|
+
// Task-level context
|
|
1900
|
+
taskId,
|
|
1901
|
+
sendProgress: (progress, message, details) => {
|
|
1902
|
+
this.dispatchEvent(new CustomEvent("task.progress", {
|
|
1903
|
+
detail: {
|
|
1904
|
+
taskId,
|
|
1905
|
+
workerId: this.id,
|
|
1906
|
+
workerType: "inline",
|
|
1907
|
+
progress,
|
|
1908
|
+
message,
|
|
1909
|
+
details,
|
|
1910
|
+
timestamp: Date.now()
|
|
1911
|
+
}
|
|
1912
|
+
}));
|
|
1913
|
+
},
|
|
1914
|
+
spawnTask: (config) => {
|
|
1915
|
+
this.dispatchEvent(new CustomEvent("task.spawn_request", {
|
|
1916
|
+
detail: {
|
|
1917
|
+
taskId,
|
|
1918
|
+
workerId: this.id,
|
|
1919
|
+
workerType: "inline",
|
|
1920
|
+
spawnConfig: config,
|
|
1921
|
+
timestamp: Date.now()
|
|
1922
|
+
}
|
|
1923
|
+
}));
|
|
1924
|
+
}
|
|
1925
|
+
};
|
|
1926
|
+
const result = await workerInstance.run(payload, context);
|
|
1927
|
+
const fullExecutionTime = performance.now() - fullExecutionStart;
|
|
1928
|
+
this.dispatchEvent(new CustomEvent("task.completed", {
|
|
1929
|
+
detail: {
|
|
1930
|
+
taskId,
|
|
1931
|
+
workerId: this.id,
|
|
1932
|
+
workerType: "inline",
|
|
1933
|
+
result,
|
|
1934
|
+
timestamp: Date.now(),
|
|
1935
|
+
executionTime: fullExecutionTime
|
|
1936
|
+
// Full task execution time including coordination
|
|
1937
|
+
}
|
|
1938
|
+
}));
|
|
1939
|
+
return result;
|
|
1940
|
+
} catch (error) {
|
|
1941
|
+
const fullExecutionTime = 0;
|
|
1942
|
+
if (error instanceof WorkerTerminationError) {
|
|
1943
|
+
this.dispatchEvent(new CustomEvent("task.failed", {
|
|
1944
|
+
detail: { taskId, workerId: this.id, workerType: "inline", error, timestamp: Date.now(), executionTime: fullExecutionTime }
|
|
1945
|
+
}));
|
|
1946
|
+
this._handleTerminationRequest(error.metadata);
|
|
1947
|
+
throw error;
|
|
1948
|
+
} else {
|
|
1949
|
+
this.dispatchEvent(new CustomEvent("task.failed", {
|
|
1950
|
+
detail: { taskId, workerId: this.id, workerType: "inline", error, timestamp: Date.now(), executionTime: fullExecutionTime }
|
|
1951
|
+
}));
|
|
1952
|
+
throw error;
|
|
1953
|
+
}
|
|
1954
|
+
} finally {
|
|
1955
|
+
this.runningTasks--;
|
|
1956
|
+
this.runningTaskIds.delete(taskId);
|
|
1957
|
+
this.runningTaskData.delete(taskId);
|
|
1958
|
+
this.tasksCompleted++;
|
|
1959
|
+
if (this.state !== "failed" && this.state !== "terminated") {
|
|
1960
|
+
this.state = this.runningTasks > 0 ? "busy" : "healthy";
|
|
1961
|
+
}
|
|
1962
|
+
this.updateActivity();
|
|
1963
|
+
}
|
|
1964
|
+
}
|
|
1965
|
+
async terminate() {
|
|
1966
|
+
await this._destroyWorkerInstance();
|
|
1967
|
+
this.workerModule = null;
|
|
1968
|
+
this.runningTasks = 0;
|
|
1969
|
+
this.runningTaskIds.clear();
|
|
1970
|
+
}
|
|
1971
|
+
async _destroyWorkerInstance() {
|
|
1972
|
+
if (this.workerInstance?.teardown) {
|
|
1973
|
+
try {
|
|
1974
|
+
this.dispatchEvent(new CustomEvent("worker.teardown.started", {
|
|
1975
|
+
detail: { workerId: this.id, workerType: "inline", timestamp: Date.now() }
|
|
1976
|
+
}));
|
|
1977
|
+
const teardownStartTime = performance.now();
|
|
1978
|
+
await this.workerInstance.teardown();
|
|
1979
|
+
this.dispatchEvent(new CustomEvent("worker.teardown.completed", {
|
|
1980
|
+
detail: {
|
|
1981
|
+
workerId: this.id,
|
|
1982
|
+
workerType: "inline",
|
|
1983
|
+
timestamp: Date.now(),
|
|
1984
|
+
duration: performance.now() - teardownStartTime
|
|
1985
|
+
}
|
|
1986
|
+
}));
|
|
1987
|
+
} catch (teardownError) {
|
|
1988
|
+
this.dispatchEvent(new CustomEvent("worker.teardown.failed", {
|
|
1989
|
+
detail: { workerId: this.id, workerType: "inline", timestamp: Date.now(), error: teardownError }
|
|
1990
|
+
}));
|
|
1991
|
+
console.warn(`Worker ${this.id} teardown failed:`, teardownError);
|
|
1992
|
+
}
|
|
1993
|
+
}
|
|
1994
|
+
this.workerInstance = null;
|
|
1995
|
+
}
|
|
1996
|
+
};
|
|
1997
|
+
|
|
1998
|
+
// core/workerManager.ts
|
|
1999
|
+
var WorkerManager = class extends EventTarget {
|
|
2000
|
+
threads = [];
|
|
2001
|
+
maxThreads;
|
|
2002
|
+
maxConcurrentTasks;
|
|
2003
|
+
scriptUrl;
|
|
2004
|
+
idleTimeout;
|
|
2005
|
+
inline;
|
|
2006
|
+
config;
|
|
2007
|
+
groups;
|
|
2008
|
+
requeueFailedTasks;
|
|
2009
|
+
maxWorkerRestarts;
|
|
2010
|
+
// Track restart count for entire pool (simpler and more logical)
|
|
2011
|
+
poolRestartCount = 0;
|
|
2012
|
+
// Set for the duration of shutdown, so worker failures during teardown are not
|
|
2013
|
+
// mistaken for crashes worth restarting
|
|
2014
|
+
shuttingDown = false;
|
|
2015
|
+
// Track ALL event listeners for cleanup
|
|
2016
|
+
allListeners = /* @__PURE__ */ new Map();
|
|
2017
|
+
// Track initializing threads
|
|
2018
|
+
initializingThreads = /* @__PURE__ */ new Set();
|
|
2019
|
+
// Centralized idle management
|
|
2020
|
+
idleCheckTimer = null;
|
|
2021
|
+
IDLE_CHECK_INTERVAL = 5e3;
|
|
2022
|
+
// Check every 5 seconds
|
|
2023
|
+
constructor(scriptUrl, options = {}) {
|
|
2024
|
+
super();
|
|
2025
|
+
this.scriptUrl = scriptUrl;
|
|
2026
|
+
this.maxThreads = options.maxThreads ?? 2;
|
|
2027
|
+
this.maxConcurrentTasks = options.maxConcurrentTasks ?? 1;
|
|
2028
|
+
this.idleTimeout = options.idleTimeout ?? 5e3;
|
|
2029
|
+
this.inline = options.inline ?? false;
|
|
2030
|
+
this.config = options.config ?? {};
|
|
2031
|
+
this.groups = options.groups ?? [];
|
|
2032
|
+
this.requeueFailedTasks = options.requeueFailedTasks ?? true;
|
|
2033
|
+
this.maxWorkerRestarts = options.maxWorkerRestarts ?? 3;
|
|
2034
|
+
this._setupWorkerFailureHandling();
|
|
2035
|
+
}
|
|
2036
|
+
// Direct execution without internal queueing
|
|
2037
|
+
async enqueueNoQueue(taskId, payload) {
|
|
2038
|
+
let targetThread = this.threads.find(
|
|
2039
|
+
(t) => t.canAcceptTask && t.canAcceptTask()
|
|
2040
|
+
);
|
|
2041
|
+
if (!targetThread && this.threads.length + this.initializingThreads.size < this.maxThreads) {
|
|
2042
|
+
const newThread = this.inline ? new InlineWrapper(this.scriptUrl, this.idleTimeout, this.maxConcurrentTasks, this.config) : new ThreadWrapper(this.scriptUrl, this.idleTimeout, this.maxConcurrentTasks, this.config);
|
|
2043
|
+
this.initializingThreads.add(newThread);
|
|
2044
|
+
this._setupWorkerEventListeners(newThread);
|
|
2045
|
+
this.threads.push(newThread);
|
|
2046
|
+
this.startIdleTimer();
|
|
2047
|
+
if (this.inline) {
|
|
2048
|
+
this.initializingThreads.delete(newThread);
|
|
2049
|
+
targetThread = newThread;
|
|
2050
|
+
} else {
|
|
2051
|
+
await new Promise((resolve) => {
|
|
2052
|
+
const checkInit = () => {
|
|
2053
|
+
if (newThread.canAcceptTask && newThread.canAcceptTask()) {
|
|
2054
|
+
this.initializingThreads.delete(newThread);
|
|
2055
|
+
resolve();
|
|
2056
|
+
} else {
|
|
2057
|
+
setTimeout(checkInit, 10);
|
|
2058
|
+
}
|
|
2059
|
+
};
|
|
2060
|
+
checkInit();
|
|
2061
|
+
});
|
|
2062
|
+
targetThread = newThread;
|
|
2063
|
+
}
|
|
2064
|
+
}
|
|
2065
|
+
if (!targetThread) {
|
|
2066
|
+
throw new Error(`No worker thread available (${this.threads.length}/${this.maxThreads})`);
|
|
2067
|
+
}
|
|
2068
|
+
return targetThread.runTask(taskId, payload);
|
|
2069
|
+
}
|
|
2070
|
+
_createPredictiveThread() {
|
|
2071
|
+
const thread = this.inline ? new InlineWrapper(this.scriptUrl, this.idleTimeout, this.maxConcurrentTasks, this.config) : new ThreadWrapper(this.scriptUrl, this.idleTimeout, this.maxConcurrentTasks, this.config);
|
|
2072
|
+
this._setupWorkerEventListeners(thread);
|
|
2073
|
+
this.threads.push(thread);
|
|
2074
|
+
this.startIdleTimer();
|
|
2075
|
+
return thread;
|
|
2076
|
+
}
|
|
2077
|
+
// Override addEventListener to track ALL listeners for cleanup
|
|
2078
|
+
//TODO: likely unneccesary, verify
|
|
2079
|
+
addEventListener(event, listener, options) {
|
|
2080
|
+
super.addEventListener(event, listener, options);
|
|
2081
|
+
this._trackListener(this, event, listener);
|
|
2082
|
+
}
|
|
2083
|
+
// Helper to track listeners for cleanup
|
|
2084
|
+
_addInternalListener(target, event, listener) {
|
|
2085
|
+
this._trackListener(target, event, listener);
|
|
2086
|
+
target.addEventListener(event, listener);
|
|
2087
|
+
}
|
|
2088
|
+
_trackListener(target, event, listener) {
|
|
2089
|
+
if (!this.allListeners.has(target)) {
|
|
2090
|
+
this.allListeners.set(target, []);
|
|
2091
|
+
}
|
|
2092
|
+
this.allListeners.get(target).push({ event, listener });
|
|
2093
|
+
}
|
|
2094
|
+
// Remove ALL event listeners (both internal and external)
|
|
2095
|
+
_removeAllListeners() {
|
|
2096
|
+
for (const [target, listeners] of this.allListeners) {
|
|
2097
|
+
for (const { event, listener } of listeners) {
|
|
2098
|
+
try {
|
|
2099
|
+
target.removeEventListener(event, listener);
|
|
2100
|
+
} catch {
|
|
2101
|
+
}
|
|
2102
|
+
}
|
|
2103
|
+
}
|
|
2104
|
+
this.allListeners.clear();
|
|
2105
|
+
}
|
|
2106
|
+
_setupWorkerEventListeners(worker) {
|
|
2107
|
+
const listeners = [
|
|
2108
|
+
{ event: "task.started", handler: (e) => this.dispatchEvent(new CustomEvent("task.started", { detail: e.detail })) },
|
|
2109
|
+
{ event: "task.completed", handler: (e) => this.dispatchEvent(new CustomEvent("task.completed", { detail: e.detail })) },
|
|
2110
|
+
{ event: "task.failed", handler: (e) => this.dispatchEvent(new CustomEvent("task.failed", { detail: e.detail })) },
|
|
2111
|
+
{ event: "task.progress", handler: (e) => this.dispatchEvent(new CustomEvent("task.progress", { detail: e.detail })) },
|
|
2112
|
+
{ event: "task.spawn_request", handler: (e) => this.dispatchEvent(new CustomEvent("task.spawn_request", { detail: e.detail })) },
|
|
2113
|
+
{ event: "worker.failed", handler: (e) => this.dispatchEvent(new CustomEvent("worker.failed", { detail: e.detail })) },
|
|
2114
|
+
// Lifecycle events - forwarded so pool owners can observe worker startup
|
|
2115
|
+
// cost, setup failures and teardown problems. Both wrapper types emit the
|
|
2116
|
+
// same set with the same payload shape.
|
|
2117
|
+
{ event: "worker.initialization.started", handler: (e) => this.dispatchEvent(new CustomEvent("worker.initialization.started", { detail: e.detail })) },
|
|
2118
|
+
{ event: "worker.initialization.completed", handler: (e) => this.dispatchEvent(new CustomEvent("worker.initialization.completed", { detail: e.detail })) },
|
|
2119
|
+
{ event: "worker.initialization.failed", handler: (e) => this.dispatchEvent(new CustomEvent("worker.initialization.failed", { detail: e.detail })) },
|
|
2120
|
+
{ event: "worker.setup.started", handler: (e) => this.dispatchEvent(new CustomEvent("worker.setup.started", { detail: e.detail })) },
|
|
2121
|
+
{ event: "worker.setup.completed", handler: (e) => this.dispatchEvent(new CustomEvent("worker.setup.completed", { detail: e.detail })) },
|
|
2122
|
+
{ event: "worker.teardown.started", handler: (e) => this.dispatchEvent(new CustomEvent("worker.teardown.started", { detail: e.detail })) },
|
|
2123
|
+
{ event: "worker.teardown.completed", handler: (e) => this.dispatchEvent(new CustomEvent("worker.teardown.completed", { detail: e.detail })) },
|
|
2124
|
+
{ event: "worker.teardown.failed", handler: (e) => this.dispatchEvent(new CustomEvent("worker.teardown.failed", { detail: e.detail })) },
|
|
2125
|
+
{ event: "worker.self_terminated", handler: (e) => this.dispatchEvent(new CustomEvent("worker.self_terminated", { detail: e.detail })) },
|
|
2126
|
+
{ event: "worker.termination_requested", handler: (e) => this._handleWorkerTerminationRequest(e) }
|
|
2127
|
+
];
|
|
2128
|
+
for (const { event, handler } of listeners) {
|
|
2129
|
+
this._addInternalListener(worker, event, handler);
|
|
2130
|
+
}
|
|
2131
|
+
}
|
|
2132
|
+
// Handle worker termination requests (new event-driven approach)
|
|
2133
|
+
_handleWorkerTerminationRequest(e) {
|
|
2134
|
+
const { workerId, metadata } = e.detail;
|
|
2135
|
+
const failureEvent = {
|
|
2136
|
+
detail: {
|
|
2137
|
+
workerId,
|
|
2138
|
+
error: new Error("Worker termination requested"),
|
|
2139
|
+
metadata: {
|
|
2140
|
+
failureType: "runtime",
|
|
2141
|
+
...metadata
|
|
2142
|
+
},
|
|
2143
|
+
timestamp: Date.now()
|
|
2144
|
+
}
|
|
2145
|
+
};
|
|
2146
|
+
this._handleWorkerFailure(failureEvent);
|
|
2147
|
+
}
|
|
2148
|
+
// Worker failure handling setup (as specified in task)
|
|
2149
|
+
_setupWorkerFailureHandling() {
|
|
2150
|
+
const failureListener = (e) => this._handleWorkerFailure(e);
|
|
2151
|
+
this._addInternalListener(this, "worker.failed", failureListener);
|
|
2152
|
+
}
|
|
2153
|
+
// Unified worker failure handling (as specified in task)
|
|
2154
|
+
_handleWorkerFailure(e) {
|
|
2155
|
+
if (this.shuttingDown) return;
|
|
2156
|
+
const { workerId, error, metadata } = e.detail;
|
|
2157
|
+
const { canRestart, restartDelay } = metadata || {};
|
|
2158
|
+
const failedWorker = this.threads.find((w) => w.id === workerId);
|
|
2159
|
+
if (!failedWorker) return;
|
|
2160
|
+
const runningTaskIds = failedWorker.getRunningTaskIds();
|
|
2161
|
+
runningTaskIds.forEach((taskId) => {
|
|
2162
|
+
if (this.requeueFailedTasks && canRestart !== false) {
|
|
2163
|
+
this.dispatchEvent(new CustomEvent("task.requeue_required", {
|
|
2164
|
+
detail: { taskId, originalError: error, workerId }
|
|
2165
|
+
}));
|
|
2166
|
+
} else {
|
|
2167
|
+
this.dispatchEvent(new CustomEvent("task.failed", {
|
|
2168
|
+
detail: {
|
|
2169
|
+
taskId,
|
|
2170
|
+
workerId,
|
|
2171
|
+
error: new Error(`Worker failed: ${error.message}`),
|
|
2172
|
+
timestamp: Date.now()
|
|
2173
|
+
}
|
|
2174
|
+
}));
|
|
2175
|
+
}
|
|
2176
|
+
});
|
|
2177
|
+
this._markWorkerAsFailed(workerId, error, metadata);
|
|
2178
|
+
this._releaseWorkerGroupResources(workerId);
|
|
2179
|
+
if (canRestart !== false) {
|
|
2180
|
+
if (this.poolRestartCount < this.maxWorkerRestarts) {
|
|
2181
|
+
this.poolRestartCount++;
|
|
2182
|
+
const delay = restartDelay || 5e3;
|
|
2183
|
+
setTimeout(() => {
|
|
2184
|
+
this._removeAndReplaceWorker(workerId);
|
|
2185
|
+
}, delay);
|
|
2186
|
+
} else {
|
|
2187
|
+
this.dispatchEvent(new CustomEvent("worker.restart_limit_exceeded", {
|
|
2188
|
+
detail: {
|
|
2189
|
+
workerId,
|
|
2190
|
+
poolRestartCount: this.poolRestartCount,
|
|
2191
|
+
maxRestarts: this.maxWorkerRestarts
|
|
2192
|
+
}
|
|
2193
|
+
}));
|
|
2194
|
+
}
|
|
2195
|
+
}
|
|
2196
|
+
}
|
|
2197
|
+
// Resource allocation implementation methods (as specified in task)
|
|
2198
|
+
_markWorkerAsFailed(workerId, error, metadata) {
|
|
2199
|
+
const worker = this.threads.find((w) => w.id === workerId);
|
|
2200
|
+
if (worker) {
|
|
2201
|
+
const alreadyTerminated = worker.state === "terminated";
|
|
2202
|
+
worker.state = "failed";
|
|
2203
|
+
worker.lastError = { timestamp: Date.now(), message: error.message, metadata };
|
|
2204
|
+
if (!alreadyTerminated) {
|
|
2205
|
+
worker.terminate().catch(() => {
|
|
2206
|
+
});
|
|
2207
|
+
}
|
|
2208
|
+
}
|
|
2209
|
+
}
|
|
2210
|
+
_releaseWorkerGroupResources(workerId) {
|
|
2211
|
+
const worker = this.threads.find((w) => w.id === workerId);
|
|
2212
|
+
if (worker) {
|
|
2213
|
+
}
|
|
2214
|
+
}
|
|
2215
|
+
_removeAndReplaceWorker(workerId) {
|
|
2216
|
+
const workerIndex = this.threads.findIndex((w) => w.id === workerId);
|
|
2217
|
+
if (workerIndex !== -1) {
|
|
2218
|
+
this.threads.splice(workerIndex, 1);
|
|
2219
|
+
if (this.threads.length < this.maxThreads) {
|
|
2220
|
+
this._createPredictiveThread();
|
|
2221
|
+
}
|
|
2222
|
+
}
|
|
2223
|
+
}
|
|
2224
|
+
// Worker Status Inspection APIs (as specified in task)
|
|
2225
|
+
/** Ids of the workers that currently exist in this pool (created lazily). */
|
|
2226
|
+
getWorkerIds() {
|
|
2227
|
+
return this.threads.map((w) => w.id);
|
|
2228
|
+
}
|
|
2229
|
+
/**
|
|
2230
|
+
* Health and activity of one worker, or `null` if no such worker exists.
|
|
2231
|
+
* Useful for building supervision on top of `worker.failed`.
|
|
2232
|
+
*/
|
|
2233
|
+
getWorkerStatus(workerId) {
|
|
2234
|
+
const worker = this.threads.find((w) => w.id === workerId);
|
|
2235
|
+
if (!worker) return null;
|
|
2236
|
+
return {
|
|
2237
|
+
id: worker.id,
|
|
2238
|
+
state: worker.state || "healthy",
|
|
2239
|
+
tasksCompleted: worker.tasksCompleted || 0,
|
|
2240
|
+
errorCount: worker.errorCount || 0,
|
|
2241
|
+
uptime: Date.now() - (worker.createdAt || Date.now()),
|
|
2242
|
+
currentTasks: worker.getRunningTaskIds(),
|
|
2243
|
+
lastError: worker.lastError,
|
|
2244
|
+
resourcesHeld: this.groups || []
|
|
2245
|
+
// Groups this worker is assigned to
|
|
2246
|
+
};
|
|
2247
|
+
}
|
|
2248
|
+
/** `getWorkerStatus` for every live worker, keyed by worker id. */
|
|
2249
|
+
getAllWorkerStatuses() {
|
|
2250
|
+
const statuses = /* @__PURE__ */ new Map();
|
|
2251
|
+
this.threads.forEach((worker) => {
|
|
2252
|
+
const status = this.getWorkerStatus(worker.id);
|
|
2253
|
+
if (status) {
|
|
2254
|
+
statuses.set(worker.id, status);
|
|
2255
|
+
}
|
|
2256
|
+
});
|
|
2257
|
+
return statuses;
|
|
2258
|
+
}
|
|
2259
|
+
// Worker Management APIs (as specified in task)
|
|
2260
|
+
/**
|
|
2261
|
+
* Terminate a worker and create a replacement, optionally merging `newConfig`
|
|
2262
|
+
* over the pool's `config`. The replacement gets a new id.
|
|
2263
|
+
*
|
|
2264
|
+
* @returns false if no worker with that id exists.
|
|
2265
|
+
*/
|
|
2266
|
+
async restartWorker(workerId, newConfig) {
|
|
2267
|
+
const workerIndex = this.threads.findIndex((w) => w.id === workerId);
|
|
2268
|
+
if (workerIndex === -1) return false;
|
|
2269
|
+
const oldWorker = this.threads[workerIndex];
|
|
2270
|
+
await oldWorker.terminate();
|
|
2271
|
+
this.threads.splice(workerIndex, 1);
|
|
2272
|
+
const config = { ...this.config, ...newConfig || {} };
|
|
2273
|
+
const newWorker = this.inline ? new InlineWrapper(this.scriptUrl, this.idleTimeout, this.maxConcurrentTasks, config) : new ThreadWrapper(this.scriptUrl, this.idleTimeout, this.maxConcurrentTasks, config);
|
|
2274
|
+
this._setupWorkerEventListeners(newWorker);
|
|
2275
|
+
this.threads.push(newWorker);
|
|
2276
|
+
this.startIdleTimer();
|
|
2277
|
+
return true;
|
|
2278
|
+
}
|
|
2279
|
+
/** Alias for {@link WorkerManager.restartWorker}. */
|
|
2280
|
+
async replaceWorker(workerId, newConfig) {
|
|
2281
|
+
return await this.restartWorker(workerId, newConfig);
|
|
2282
|
+
}
|
|
2283
|
+
/**
|
|
2284
|
+
* Merge `config` into a live worker's config object.
|
|
2285
|
+
*
|
|
2286
|
+
* Note this mutates the config a worker instance already holds - a worker that
|
|
2287
|
+
* copied values in its constructor will not see the change. Use
|
|
2288
|
+
* {@link WorkerManager.restartWorker} when the worker must be rebuilt.
|
|
2289
|
+
*
|
|
2290
|
+
* @returns false if no worker with that id exists.
|
|
2291
|
+
*/
|
|
2292
|
+
async updateWorkerConfig(workerId, config) {
|
|
2293
|
+
const worker = this.threads.find((w) => w.id === workerId);
|
|
2294
|
+
if (!worker) return false;
|
|
2295
|
+
worker.config = { ...worker.config, ...config };
|
|
2296
|
+
return true;
|
|
2297
|
+
}
|
|
2298
|
+
// Centralized idle management methods
|
|
2299
|
+
startIdleTimer() {
|
|
2300
|
+
if (this.idleCheckTimer || this.threads.length === 0) return;
|
|
2301
|
+
this.idleCheckTimer = setInterval(() => {
|
|
2302
|
+
this.checkIdleWorkers();
|
|
2303
|
+
}, this.IDLE_CHECK_INTERVAL);
|
|
2304
|
+
}
|
|
2305
|
+
stopIdleTimer() {
|
|
2306
|
+
if (this.idleCheckTimer) {
|
|
2307
|
+
clearInterval(this.idleCheckTimer);
|
|
2308
|
+
this.idleCheckTimer = null;
|
|
2309
|
+
}
|
|
2310
|
+
}
|
|
2311
|
+
checkIdleWorkers() {
|
|
2312
|
+
const now = Date.now();
|
|
2313
|
+
const workersToTerminate = [];
|
|
2314
|
+
for (const worker of this.threads) {
|
|
2315
|
+
if (worker.idleTimeout > 0 && worker.idle && now - worker.lastActivityTime > worker.idleTimeout) {
|
|
2316
|
+
workersToTerminate.push(worker);
|
|
2317
|
+
}
|
|
2318
|
+
}
|
|
2319
|
+
for (const worker of workersToTerminate) {
|
|
2320
|
+
this._removeIdleWorker(worker).catch((error) => {
|
|
2321
|
+
console.warn(`Failed to remove idle worker ${worker.id}:`, error);
|
|
2322
|
+
});
|
|
2323
|
+
}
|
|
2324
|
+
if (this.threads.length === 0) {
|
|
2325
|
+
this.stopIdleTimer();
|
|
2326
|
+
}
|
|
2327
|
+
}
|
|
2328
|
+
async _removeIdleWorker(worker) {
|
|
2329
|
+
const workerIndex = this.threads.findIndex((w) => w.id === worker.id);
|
|
2330
|
+
if (workerIndex === -1) return;
|
|
2331
|
+
this.threads.splice(workerIndex, 1);
|
|
2332
|
+
try {
|
|
2333
|
+
await worker.terminate();
|
|
2334
|
+
} catch (error) {
|
|
2335
|
+
console.warn(`Failed to terminate idle worker ${worker.id}:`, error);
|
|
2336
|
+
}
|
|
2337
|
+
}
|
|
2338
|
+
// Shutdown all workers and cleanup resources
|
|
2339
|
+
/** Terminate every worker in the pool and drop its listeners. */
|
|
2340
|
+
async shutdown() {
|
|
2341
|
+
this.shuttingDown = true;
|
|
2342
|
+
this.stopIdleTimer();
|
|
2343
|
+
const shutdownPromises = this.threads.map(async (worker) => {
|
|
2344
|
+
try {
|
|
2345
|
+
return new Promise((resolve, reject) => {
|
|
2346
|
+
const timeout = setTimeout(() => {
|
|
2347
|
+
console.warn(`\u26A0\uFE0F Timeout terminating worker ${worker.id}`);
|
|
2348
|
+
reject(new Error("Failed to terminate worker"));
|
|
2349
|
+
}, 1e3);
|
|
2350
|
+
const terminatePromise = worker.terminate();
|
|
2351
|
+
terminatePromise.then(() => {
|
|
2352
|
+
clearTimeout(timeout);
|
|
2353
|
+
resolve();
|
|
2354
|
+
}).catch((error) => {
|
|
2355
|
+
clearTimeout(timeout);
|
|
2356
|
+
reject(error);
|
|
2357
|
+
});
|
|
2358
|
+
});
|
|
2359
|
+
} catch (error) {
|
|
2360
|
+
console.warn(`\u26A0\uFE0F Error terminating worker ${worker.id}:`, error);
|
|
2361
|
+
return Promise.resolve();
|
|
2362
|
+
}
|
|
2363
|
+
});
|
|
2364
|
+
await Promise.all(shutdownPromises);
|
|
2365
|
+
this.threads = [];
|
|
2366
|
+
this._removeAllListeners();
|
|
2367
|
+
}
|
|
2368
|
+
};
|
|
2369
|
+
export {
|
|
2370
|
+
BaseWorker,
|
|
2371
|
+
ConcurrentLimitGroup,
|
|
2372
|
+
FyflowScheduler,
|
|
2373
|
+
FyflowTask,
|
|
2374
|
+
InlineWrapper,
|
|
2375
|
+
KeyedRateLimitGroup,
|
|
2376
|
+
RateLimitGroup,
|
|
2377
|
+
ThreadWrapper,
|
|
2378
|
+
WorkerManager,
|
|
2379
|
+
WorkerTerminationError
|
|
2380
|
+
};
|