@nmakarov/cli-toolkit 0.21.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/args.cjs +1 -4
- package/dist/args.cjs.map +1 -1
- package/dist/args.js +1 -1
- package/dist/args.js.map +1 -1
- package/dist/cli-runner.cjs +1252 -604
- package/dist/cli-runner.cjs.map +1 -1
- package/dist/cli-runner.js +1299 -650
- package/dist/cli-runner.js.map +1 -1
- package/dist/db.cjs +85 -157
- package/dist/db.cjs.map +1 -1
- package/dist/db.js +84 -150
- package/dist/db.js.map +1 -1
- package/dist/errors.cjs +2 -2
- package/dist/errors.cjs.map +1 -1
- package/dist/errors.js +2 -1
- package/dist/errors.js.map +1 -1
- package/dist/filedatabase.cjs +19 -19
- package/dist/filedatabase.cjs.map +1 -1
- package/dist/filedatabase.js +19 -16
- package/dist/filedatabase.js.map +1 -1
- package/dist/http-client.cjs +9 -11
- package/dist/http-client.cjs.map +1 -1
- package/dist/http-client.js +10 -9
- package/dist/http-client.js.map +1 -1
- package/dist/http-client2.cjs +34 -37
- package/dist/http-client2.cjs.map +1 -1
- package/dist/http-client2.js +34 -34
- package/dist/http-client2.js.map +1 -1
- package/dist/index.cjs +1739 -720
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +1746 -721
- package/dist/index.js.map +1 -1
- package/dist/init.cjs +81 -68
- package/dist/init.cjs.map +1 -1
- package/dist/init.js +96 -82
- package/dist/init.js.map +1 -1
- package/dist/logger.cjs +5 -5
- package/dist/logger.cjs.map +1 -1
- package/dist/logger.js +5 -4
- package/dist/logger.js.map +1 -1
- package/dist/mock-server.cjs +21 -33
- package/dist/mock-server.cjs.map +1 -1
- package/dist/mock-server.js +21 -28
- package/dist/mock-server.js.map +1 -1
- package/dist/params.cjs +6 -9
- package/dist/params.cjs.map +1 -1
- package/dist/params.js +6 -6
- package/dist/params.js.map +1 -1
- package/dist/s3.cjs +286 -0
- package/dist/s3.cjs.map +1 -0
- package/dist/s3.js +273 -0
- package/dist/s3.js.map +1 -0
- package/dist/screen.cjs +34 -39
- package/dist/screen.cjs.map +1 -1
- package/dist/screen.js +48 -46
- package/dist/screen.js.map +1 -1
- package/dist/tasks.cjs +1354 -501
- package/dist/tasks.cjs.map +1 -1
- package/dist/tasks.js +1369 -527
- package/dist/tasks.js.map +1 -1
- package/dist/utils.cjs +7 -8
- package/dist/utils.cjs.map +1 -1
- package/dist/utils.js +6 -6
- package/dist/utils.js.map +1 -1
- package/package.json +32 -47
- package/scripts/ssm/{parse-cli.ts → parse-cli.js} +4 -4
- package/scripts/ssm/{ssm-admin.ts → ssm-admin.js} +12 -12
- package/scripts/ssm/{ssm-pull.ts → ssm-pull.js} +10 -13
package/dist/tasks.js
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
// src/
|
|
1
|
+
// src/tasks/index.js
|
|
2
|
+
import os3 from "os";
|
|
3
|
+
|
|
4
|
+
// src/utils/date-utils.js
|
|
2
5
|
function isTimestampFolder(folderName) {
|
|
3
6
|
const isoRegex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(Z|\.\d{3}Z)$/;
|
|
4
7
|
if (!isoRegex.test(folderName)) {
|
|
@@ -8,7 +11,7 @@ function isTimestampFolder(folderName) {
|
|
|
8
11
|
return !isNaN(date.getTime()) && date.getTime() > 0;
|
|
9
12
|
}
|
|
10
13
|
|
|
11
|
-
// src/utils/fs-utils.
|
|
14
|
+
// src/utils/fs-utils.js
|
|
12
15
|
import fs from "fs";
|
|
13
16
|
import path from "path";
|
|
14
17
|
async function ensurePath(...pathParts) {
|
|
@@ -32,7 +35,7 @@ function getFileExtension(dataType) {
|
|
|
32
35
|
}
|
|
33
36
|
}
|
|
34
37
|
|
|
35
|
-
// src/utils/os-utils.
|
|
38
|
+
// src/utils/os-utils.js
|
|
36
39
|
import fs2 from "fs";
|
|
37
40
|
import path2 from "path";
|
|
38
41
|
import { execSync } from "child_process";
|
|
@@ -61,7 +64,7 @@ function getFreeDiskSpace(targetPath) {
|
|
|
61
64
|
}
|
|
62
65
|
}
|
|
63
66
|
|
|
64
|
-
// src/utils/format-utils.
|
|
67
|
+
// src/utils/format-utils.js
|
|
65
68
|
function bytesToHumanReadable(bytes) {
|
|
66
69
|
if (bytes === 0) return "0 B";
|
|
67
70
|
const k = 1024;
|
|
@@ -70,7 +73,7 @@ function bytesToHumanReadable(bytes) {
|
|
|
70
73
|
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + " " + sizes[i];
|
|
71
74
|
}
|
|
72
75
|
|
|
73
|
-
// src/utils/core-utils.
|
|
76
|
+
// src/utils/core-utils.js
|
|
74
77
|
function sleepMs(ms) {
|
|
75
78
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
76
79
|
}
|
|
@@ -79,14 +82,82 @@ function toJsonColumn(value) {
|
|
|
79
82
|
return JSON.stringify(value);
|
|
80
83
|
}
|
|
81
84
|
|
|
82
|
-
// src/tasks/servicesRegistry.
|
|
83
|
-
import { randomUUID as randomUUID2 } from "crypto";
|
|
84
|
-
import { mkdir, readFile, writeFile } from "fs/promises";
|
|
85
|
+
// src/tasks/servicesRegistry.js
|
|
85
86
|
import os from "os";
|
|
86
|
-
import path3 from "path";
|
|
87
87
|
|
|
88
|
-
// src/tasks/taskUtils.
|
|
88
|
+
// src/tasks/taskUtils.js
|
|
89
89
|
import { randomUUID } from "crypto";
|
|
90
|
+
|
|
91
|
+
// src/tasks/time-matcher.js
|
|
92
|
+
var RANGES = ["0-59", "0-59", "0-23", "1-31", "1-12", "0-6"];
|
|
93
|
+
function resolveAsterisks(field, range) {
|
|
94
|
+
return field.includes("*") ? field.replace("*", range) : field;
|
|
95
|
+
}
|
|
96
|
+
function resolveRanges(field) {
|
|
97
|
+
const regex = /(\d+)-(\d+)/;
|
|
98
|
+
let current = field;
|
|
99
|
+
while (true) {
|
|
100
|
+
const match = regex.exec(current);
|
|
101
|
+
if (!match) break;
|
|
102
|
+
const raw = match[0];
|
|
103
|
+
let first = Number(match[1]);
|
|
104
|
+
let last = Number(match[2]);
|
|
105
|
+
if (last < first) {
|
|
106
|
+
[first, last] = [last, first];
|
|
107
|
+
}
|
|
108
|
+
const values = [];
|
|
109
|
+
for (let i = first; i <= last; i += 1) {
|
|
110
|
+
values.push(i);
|
|
111
|
+
}
|
|
112
|
+
current = current.replace(raw, values.join(","));
|
|
113
|
+
}
|
|
114
|
+
return current;
|
|
115
|
+
}
|
|
116
|
+
function resolveSteps(field) {
|
|
117
|
+
const match = /^(.+)\/(\d+)$/.exec(field);
|
|
118
|
+
if (!match) return field;
|
|
119
|
+
const base = match[1];
|
|
120
|
+
const step = Number(match[2]);
|
|
121
|
+
if (!Number.isFinite(step) || step <= 0) return field;
|
|
122
|
+
return base.split(",").map((v) => Number(v)).filter((v) => Number.isFinite(v) && v % step === 0).join(",");
|
|
123
|
+
}
|
|
124
|
+
function convertPattern(pattern) {
|
|
125
|
+
const parts = pattern.trim().split(/\s+/);
|
|
126
|
+
if (parts.length !== 6) {
|
|
127
|
+
throw new Error(`Invalid schedule "${pattern}". Expected 6 fields: sec min hour day month weekday`);
|
|
128
|
+
}
|
|
129
|
+
return parts.map((field, idx) => resolveAsterisks(field, RANGES[idx])).map((field) => resolveRanges(field)).map((field) => resolveSteps(field));
|
|
130
|
+
}
|
|
131
|
+
function fieldMatches(field, value) {
|
|
132
|
+
const allowed = field.split(",").map((v) => Number(v));
|
|
133
|
+
return allowed.includes(value);
|
|
134
|
+
}
|
|
135
|
+
function matchesParsedPattern(parsed, date) {
|
|
136
|
+
return fieldMatches(parsed[0], date.getSeconds()) && fieldMatches(parsed[1], date.getMinutes()) && fieldMatches(parsed[2], date.getHours()) && fieldMatches(parsed[3], date.getDate()) && fieldMatches(parsed[4], date.getMonth() + 1) && fieldMatches(parsed[5], date.getDay());
|
|
137
|
+
}
|
|
138
|
+
function timeMatcher(pattern, date = /* @__PURE__ */ new Date()) {
|
|
139
|
+
const parsed = convertPattern(pattern);
|
|
140
|
+
return matchesParsedPattern(parsed, date);
|
|
141
|
+
}
|
|
142
|
+
var MS_PER_SECOND = 1e3;
|
|
143
|
+
var DEFAULT_SEARCH_HORIZON_MS = 10 * 365 * 24 * 60 * 60 * MS_PER_SECOND;
|
|
144
|
+
function nextTimeMatch(pattern, from = /* @__PURE__ */ new Date(), maxSearchMs = DEFAULT_SEARCH_HORIZON_MS) {
|
|
145
|
+
const parsed = convertPattern(pattern);
|
|
146
|
+
let t = Math.ceil((from.getTime() + 1) / MS_PER_SECOND) * MS_PER_SECOND;
|
|
147
|
+
const end = t + maxSearchMs;
|
|
148
|
+
while (t <= end) {
|
|
149
|
+
const date = new Date(t);
|
|
150
|
+
if (matchesParsedPattern(parsed, date)) {
|
|
151
|
+
return date;
|
|
152
|
+
}
|
|
153
|
+
t += MS_PER_SECOND;
|
|
154
|
+
}
|
|
155
|
+
throw new Error(
|
|
156
|
+
`nextTimeMatch: no match for "${pattern}" within ${maxSearchMs}ms after ${from.toISOString()}`
|
|
157
|
+
);
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// src/tasks/taskUtils.js
|
|
90
161
|
function getDb(context) {
|
|
91
162
|
const db = context.db;
|
|
92
163
|
if (!db) {
|
|
@@ -94,113 +165,117 @@ function getDb(context) {
|
|
|
94
165
|
}
|
|
95
166
|
return db;
|
|
96
167
|
}
|
|
97
|
-
function queueToTableNames(
|
|
98
|
-
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(queue)) {
|
|
99
|
-
throw new Error(`Invalid queue name "${queue}". Use letters, numbers, underscore only.`);
|
|
100
|
-
}
|
|
168
|
+
function queueToTableNames(queueName) {
|
|
101
169
|
return {
|
|
102
|
-
tasksTable:
|
|
103
|
-
historyTable: `${
|
|
170
|
+
tasksTable: queueName,
|
|
171
|
+
historyTable: `${queueName}_history`,
|
|
172
|
+
registryTable: `${queueName}_services_registry`
|
|
104
173
|
};
|
|
105
174
|
}
|
|
106
|
-
function
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
175
|
+
function defineTasksTable(t, db, tableNameForIndex) {
|
|
176
|
+
t.uuid("id").primary().defaultTo(db.raw("uuid_generate_v4()"));
|
|
177
|
+
t.timestamp("created_at").notNullable().defaultTo(db.fn.now());
|
|
178
|
+
t.timestamp("started_at");
|
|
179
|
+
t.timestamp("completed_at");
|
|
180
|
+
t.integer("priority").notNullable().defaultTo(50);
|
|
181
|
+
t.text("schedule");
|
|
182
|
+
t.timestamp("next_run_at").defaultTo(null);
|
|
183
|
+
t.timestamp("past_due").defaultTo(null);
|
|
184
|
+
t.text("name").notNullable();
|
|
185
|
+
t.text("opid");
|
|
186
|
+
t.json("params");
|
|
187
|
+
t.text("service_group");
|
|
188
|
+
t.integer("instance_number");
|
|
189
|
+
t.text("service_name");
|
|
190
|
+
t.text("server_name");
|
|
191
|
+
t.text("status").notNullable().defaultTo("idle");
|
|
192
|
+
t.timestamp("status_changed_at").defaultTo(null);
|
|
193
|
+
t.text("progress");
|
|
194
|
+
t.boolean("success");
|
|
195
|
+
t.json("results");
|
|
196
|
+
t.index(["service_group", "status", "priority", "created_at"], `${tableNameForIndex}_claim_idx`);
|
|
197
|
+
t.index(["service_group", "name"], `${tableNameForIndex}_group_name_idx`);
|
|
198
|
+
}
|
|
199
|
+
function taskHistoryInsertFromQueueRow(row, overrides) {
|
|
200
|
+
const { id, ...snapshot } = row;
|
|
201
|
+
void id;
|
|
202
|
+
return {
|
|
203
|
+
...snapshot,
|
|
204
|
+
...overrides
|
|
205
|
+
};
|
|
111
206
|
}
|
|
112
207
|
async function ensureTaskTables(context, options = {}) {
|
|
113
|
-
const
|
|
208
|
+
const queueName = options.queueName ?? "tasks";
|
|
114
209
|
const recreate = options.recreate ?? false;
|
|
115
210
|
const db = getDb(context);
|
|
116
|
-
const { tasksTable, historyTable } = queueToTableNames(
|
|
211
|
+
const { tasksTable, historyTable, registryTable } = queueToTableNames(queueName);
|
|
117
212
|
const needsTasks = recreate ? true : !await db.tableExists(tasksTable);
|
|
118
213
|
const needsHistory = recreate ? true : !await db.tableExists(historyTable);
|
|
214
|
+
const needsRegistry = recreate ? true : !await db.tableExists(registryTable);
|
|
119
215
|
if (recreate) {
|
|
120
216
|
await db.schema.dropTableIfExists(historyTable);
|
|
121
217
|
await db.schema.dropTableIfExists(tasksTable);
|
|
218
|
+
await db.schema.dropTableIfExists(registryTable);
|
|
122
219
|
}
|
|
123
220
|
if (needsTasks) {
|
|
124
221
|
await db.raw(`CREATE EXTENSION IF NOT EXISTS "uuid-ossp"`);
|
|
125
222
|
await db.schema.createTable(tasksTable, (t) => {
|
|
126
|
-
t
|
|
127
|
-
t.timestamp("created_at").notNullable().defaultTo(db.fn.now());
|
|
128
|
-
t.timestamp("started_at");
|
|
129
|
-
t.timestamp("completed_at");
|
|
130
|
-
t.integer("priority").notNullable().defaultTo(0);
|
|
131
|
-
t.text("schedule");
|
|
132
|
-
t.timestamp("past_due").defaultTo(null);
|
|
133
|
-
t.text("target").notNullable();
|
|
134
|
-
t.text("task").notNullable();
|
|
135
|
-
t.json("params");
|
|
136
|
-
t.text("opid");
|
|
137
|
-
t.timestamp("paused_at").defaultTo(null);
|
|
138
|
-
t.text("progress");
|
|
139
|
-
t.boolean("success");
|
|
140
|
-
t.json("results");
|
|
141
|
-
});
|
|
142
|
-
await db.schema.alterTable(tasksTable, (t) => {
|
|
143
|
-
t.index(["target", "started_at", "created_at"], `${tasksTable}_target_started_created_idx`);
|
|
144
|
-
t.index(["target", "past_due", "priority", "created_at"], `${tasksTable}_target_past_due_priority_created_idx`);
|
|
145
|
-
t.index(["target", "task"], `${tasksTable}_target_task_idx`);
|
|
223
|
+
defineTasksTable(t, db, tasksTable);
|
|
146
224
|
});
|
|
147
225
|
}
|
|
148
226
|
if (needsHistory) {
|
|
149
227
|
await db.schema.createTable(historyTable, (t) => {
|
|
150
|
-
t
|
|
151
|
-
t.timestamp("created_at").notNullable();
|
|
152
|
-
t.timestamp("started_at");
|
|
153
|
-
t.timestamp("completed_at");
|
|
154
|
-
t.integer("priority").notNullable().defaultTo(0);
|
|
155
|
-
t.text("schedule");
|
|
156
|
-
t.timestamp("past_due").defaultTo(null);
|
|
157
|
-
t.text("target").notNullable();
|
|
158
|
-
t.text("task").notNullable();
|
|
159
|
-
t.json("params");
|
|
160
|
-
t.text("opid");
|
|
161
|
-
t.text("progress");
|
|
162
|
-
t.boolean("success");
|
|
163
|
-
t.json("results");
|
|
164
|
-
});
|
|
165
|
-
await db.schema.alterTable(historyTable, (t) => {
|
|
166
|
-
t.index(["target", "created_at"], `${historyTable}_target_created_idx`);
|
|
167
|
-
t.index(["task", "created_at"], `${historyTable}_task_created_idx`);
|
|
228
|
+
defineTasksTable(t, db, historyTable);
|
|
168
229
|
});
|
|
169
230
|
}
|
|
170
|
-
const registryTable = servicesRegistryTable(queue);
|
|
171
|
-
const needsRegistry = !await db.tableExists(registryTable);
|
|
172
231
|
if (needsRegistry) {
|
|
173
232
|
await db.schema.createTable(registryTable, (t) => {
|
|
174
233
|
t.uuid("id").primary().defaultTo(db.raw("uuid_generate_v4()"));
|
|
175
|
-
t.
|
|
176
|
-
t.text("queue").notNullable();
|
|
234
|
+
t.text("queue_name").notNullable();
|
|
177
235
|
t.text("service_group").notNullable();
|
|
236
|
+
t.integer("instance_number").notNullable().defaultTo(1);
|
|
178
237
|
t.text("service_name").notNullable();
|
|
179
|
-
t.text("
|
|
180
|
-
t.text("hostname");
|
|
238
|
+
t.text("server_name").notNullable();
|
|
181
239
|
t.integer("pid");
|
|
182
240
|
t.json("metadata");
|
|
183
241
|
t.timestamp("created_at").notNullable().defaultTo(db.fn.now());
|
|
184
242
|
t.timestamp("last_seen_at").notNullable().defaultTo(db.fn.now());
|
|
185
|
-
t.unique(["
|
|
186
|
-
t.index(["
|
|
187
|
-
t.index(["
|
|
243
|
+
t.unique(["queue_name", "service_name"], `${registryTable}_queue_name_service_name_uniq`);
|
|
244
|
+
t.index(["queue_name", "service_group", "last_seen_at"], `${registryTable}_queue_group_seen_idx`);
|
|
245
|
+
t.index(["queue_name", "last_seen_at"], `${registryTable}_queue_seen_idx`);
|
|
188
246
|
});
|
|
189
247
|
}
|
|
190
248
|
}
|
|
191
249
|
async function enqueueTask(context, options) {
|
|
192
250
|
const db = getDb(context);
|
|
193
|
-
const
|
|
194
|
-
const { tasksTable } = queueToTableNames(
|
|
251
|
+
const queueName = options.queueName ?? "tasks";
|
|
252
|
+
const { tasksTable } = queueToTableNames(queueName);
|
|
195
253
|
const id = randomUUID();
|
|
254
|
+
const name = options.name ?? options.task;
|
|
255
|
+
if (!name) {
|
|
256
|
+
throw new Error("enqueueTask: name (or task) is required");
|
|
257
|
+
}
|
|
258
|
+
const schedule = options.schedule?.trim() ? options.schedule : null;
|
|
259
|
+
let nextRunAt = null;
|
|
260
|
+
if (options.nextRunAt !== void 0) {
|
|
261
|
+
nextRunAt = options.nextRunAt == null ? null : new Date(options.nextRunAt);
|
|
262
|
+
} else if (schedule) {
|
|
263
|
+
nextRunAt = nextTimeMatch(schedule, /* @__PURE__ */ new Date());
|
|
264
|
+
}
|
|
196
265
|
await db(tasksTable).insert({
|
|
197
266
|
id,
|
|
198
|
-
|
|
199
|
-
task: options.task,
|
|
267
|
+
name,
|
|
200
268
|
params: toJsonColumn(options.params ?? null),
|
|
201
269
|
opid: options.opid ?? null,
|
|
202
|
-
priority: options.priority ??
|
|
203
|
-
schedule
|
|
270
|
+
priority: options.priority ?? 50,
|
|
271
|
+
schedule,
|
|
272
|
+
next_run_at: nextRunAt,
|
|
273
|
+
service_group: options.serviceGroup ?? null,
|
|
274
|
+
instance_number: options.instanceNumber ?? null,
|
|
275
|
+
service_name: options.serviceName ?? null,
|
|
276
|
+
server_name: options.serverName ?? null,
|
|
277
|
+
status: "idle",
|
|
278
|
+
status_changed_at: db.fn.now()
|
|
204
279
|
});
|
|
205
280
|
return id;
|
|
206
281
|
}
|
|
@@ -211,7 +286,7 @@ async function updateTaskProgress(context, tasksTable, taskId, progress) {
|
|
|
211
286
|
});
|
|
212
287
|
}
|
|
213
288
|
|
|
214
|
-
// src/tasks/servicesRegistry.
|
|
289
|
+
// src/tasks/servicesRegistry.js
|
|
215
290
|
function getDb2(context) {
|
|
216
291
|
const db = context.db;
|
|
217
292
|
if (!db) {
|
|
@@ -225,7 +300,7 @@ function parseMetadataColumn(value) {
|
|
|
225
300
|
if (typeof value === "string") {
|
|
226
301
|
try {
|
|
227
302
|
const p = JSON.parse(value);
|
|
228
|
-
return p && typeof p === "object" && !Array.isArray(
|
|
303
|
+
return p && typeof p === "object" && !Array.isArray(value) ? p : {};
|
|
229
304
|
} catch {
|
|
230
305
|
return {};
|
|
231
306
|
}
|
|
@@ -235,6 +310,7 @@ function parseMetadataColumn(value) {
|
|
|
235
310
|
var DEFAULT_GROUP_MAX_INSTANCES = {
|
|
236
311
|
intake: 1,
|
|
237
312
|
harvest: 1,
|
|
313
|
+
harvester: 0,
|
|
238
314
|
loader: 0,
|
|
239
315
|
photos: 0,
|
|
240
316
|
photosprocessor: 0,
|
|
@@ -244,25 +320,6 @@ function sanitizeNamePart(raw) {
|
|
|
244
320
|
const s = String(raw || "").trim().replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
245
321
|
return s.slice(0, 80) || "runner";
|
|
246
322
|
}
|
|
247
|
-
function identityFilePath(identityDir, queue, serviceGroup) {
|
|
248
|
-
const safeQ = sanitizeNamePart(queue);
|
|
249
|
-
const safeG = sanitizeNamePart(serviceGroup);
|
|
250
|
-
return path3.join(identityDir, `${safeQ}_${safeG}.json`);
|
|
251
|
-
}
|
|
252
|
-
async function readIdentityFile(filePath) {
|
|
253
|
-
try {
|
|
254
|
-
const text = await readFile(filePath, "utf8");
|
|
255
|
-
const parsed = JSON.parse(text);
|
|
256
|
-
return parsed && typeof parsed === "object" ? parsed : {};
|
|
257
|
-
} catch {
|
|
258
|
-
return {};
|
|
259
|
-
}
|
|
260
|
-
}
|
|
261
|
-
async function writeIdentityFile(filePath, data) {
|
|
262
|
-
await mkdir(path3.dirname(filePath), { recursive: true });
|
|
263
|
-
await writeFile(filePath, `${JSON.stringify(data, null, 2)}
|
|
264
|
-
`, "utf8");
|
|
265
|
-
}
|
|
266
323
|
function resolveMaxInstances(serviceGroup, override) {
|
|
267
324
|
if (override !== void 0 && Number.isFinite(override)) {
|
|
268
325
|
return Math.max(0, Math.floor(Number(override)));
|
|
@@ -270,179 +327,190 @@ function resolveMaxInstances(serviceGroup, override) {
|
|
|
270
327
|
const g = serviceGroup.trim().toLowerCase();
|
|
271
328
|
return DEFAULT_GROUP_MAX_INSTANCES[g] ?? 0;
|
|
272
329
|
}
|
|
273
|
-
async function countAliveInGroup(db, registryTable,
|
|
330
|
+
async function countAliveInGroup(db, registryTable, queueName, serviceGroup, staleMs, excludeRowId) {
|
|
274
331
|
const cutoff = new Date(Date.now() - staleMs);
|
|
275
|
-
let q = db(registryTable).where({
|
|
276
|
-
if (
|
|
277
|
-
q = q.whereNot("
|
|
332
|
+
let q = db(registryTable).where({ queue_name: queueName, service_group: serviceGroup }).where("last_seen_at", ">", cutoff);
|
|
333
|
+
if (excludeRowId) {
|
|
334
|
+
q = q.whereNot("id", excludeRowId);
|
|
278
335
|
}
|
|
279
336
|
const row = await q.count("id as count").first();
|
|
280
337
|
return Number(row?.count ?? 0);
|
|
281
338
|
}
|
|
339
|
+
async function getOccupiedInstanceSlots(db, registryTable, queueName, serviceGroup, staleMs) {
|
|
340
|
+
const cutoff = new Date(Date.now() - staleMs);
|
|
341
|
+
const rows = await db(registryTable).where({ queue_name: queueName, service_group: serviceGroup }).where("last_seen_at", ">", cutoff).select("instance_number");
|
|
342
|
+
const set = /* @__PURE__ */ new Set();
|
|
343
|
+
for (const r of rows) {
|
|
344
|
+
const n = Number(r.instance_number);
|
|
345
|
+
if (Number.isFinite(n) && n >= 1) set.add(Math.floor(n));
|
|
346
|
+
}
|
|
347
|
+
return set;
|
|
348
|
+
}
|
|
282
349
|
function isUniqueViolation(error) {
|
|
283
350
|
const code = error?.code ?? error?.errno;
|
|
284
351
|
return code === "23505" || String(error?.message || "").includes("duplicate key");
|
|
285
352
|
}
|
|
353
|
+
function buildMetadata(options) {
|
|
354
|
+
const base = options.metadata && typeof options.metadata === "object" ? { ...options.metadata } : {};
|
|
355
|
+
if (options.target) {
|
|
356
|
+
base.runnerTarget = options.target;
|
|
357
|
+
}
|
|
358
|
+
return toJsonColumn(Object.keys(base).length ? base : null);
|
|
359
|
+
}
|
|
360
|
+
function allocateInstanceNumber(occupied, explicit, maxSlots) {
|
|
361
|
+
if (explicit !== void 0 && explicit !== null && Number.isFinite(Number(explicit))) {
|
|
362
|
+
const e = Math.max(1, Math.floor(Number(explicit)));
|
|
363
|
+
if (occupied.has(e)) {
|
|
364
|
+
throw new Error(`[services-registry] instance slot ${e} is already occupied by an alive peer`);
|
|
365
|
+
}
|
|
366
|
+
if (maxSlots !== void 0 && maxSlots > 0 && e > maxSlots) {
|
|
367
|
+
throw new Error(`[services-registry] instance ${e} exceeds configured max slots ${maxSlots} for this group`);
|
|
368
|
+
}
|
|
369
|
+
return e;
|
|
370
|
+
}
|
|
371
|
+
const cap = maxSlots !== void 0 && maxSlots > 0 ? maxSlots : 1e4;
|
|
372
|
+
for (let n = 1; n <= cap; n++) {
|
|
373
|
+
if (!occupied.has(n)) return n;
|
|
374
|
+
}
|
|
375
|
+
throw new Error(`[services-registry] no free instance slot (searched 1..${cap})`);
|
|
376
|
+
}
|
|
377
|
+
function defaultServiceName(groupBase, hostBase, instanceNumber) {
|
|
378
|
+
return `${groupBase}-${hostBase}-${instanceNumber}`;
|
|
379
|
+
}
|
|
286
380
|
async function registerInServicesRegistry(context, options) {
|
|
287
381
|
const db = getDb2(context);
|
|
288
|
-
const registryTable =
|
|
382
|
+
const registryTable = queueToTableNames(options.queueName).registryTable;
|
|
289
383
|
const serviceGroup = options.serviceGroup.trim();
|
|
290
384
|
if (!serviceGroup) {
|
|
291
385
|
throw new Error("registerInServicesRegistry: serviceGroup is required");
|
|
292
386
|
}
|
|
293
|
-
const
|
|
294
|
-
let identity = await readIdentityFile(identityPath);
|
|
295
|
-
let instanceId = typeof identity.instanceId === "string" && identity.instanceId.trim() ? identity.instanceId.trim() : randomUUID2();
|
|
296
|
-
identity.instanceId = instanceId;
|
|
297
|
-
await writeIdentityFile(identityPath, identity);
|
|
298
|
-
const hostname = os.hostname();
|
|
387
|
+
const serverName = os.hostname();
|
|
299
388
|
const pid = typeof process.pid === "number" ? process.pid : null;
|
|
300
|
-
const meta =
|
|
301
|
-
const
|
|
302
|
-
|
|
303
|
-
await db(registryTable).where({ instance_id: instanceId }).update({
|
|
304
|
-
target: options.target,
|
|
305
|
-
hostname,
|
|
306
|
-
pid,
|
|
307
|
-
metadata: meta,
|
|
308
|
-
last_seen_at: db.fn.now()
|
|
309
|
-
});
|
|
310
|
-
const serviceName = String(existing.service_name);
|
|
311
|
-
identity.serviceName = serviceName;
|
|
312
|
-
await writeIdentityFile(identityPath, identity);
|
|
313
|
-
const reg = {
|
|
314
|
-
instanceId,
|
|
315
|
-
serviceName,
|
|
316
|
-
serviceGroup,
|
|
317
|
-
queue: options.queue,
|
|
318
|
-
target: options.target,
|
|
319
|
-
rowId: String(existing.id)
|
|
320
|
-
};
|
|
321
|
-
context.servicesRegistry = reg;
|
|
322
|
-
context.runnerHeartbeat = reg;
|
|
323
|
-
context.logger.info?.(
|
|
324
|
-
`[services-registry] resumed instance_id=${instanceId} name=${serviceName} group=${serviceGroup} queue=${options.queue}`
|
|
325
|
-
);
|
|
326
|
-
return {
|
|
327
|
-
instanceId,
|
|
328
|
-
serviceName,
|
|
329
|
-
serviceGroup,
|
|
330
|
-
queue: options.queue,
|
|
331
|
-
target: options.target,
|
|
332
|
-
rowId: String(existing.id),
|
|
333
|
-
registryTable
|
|
334
|
-
};
|
|
335
|
-
}
|
|
389
|
+
const meta = buildMetadata(options);
|
|
390
|
+
const groupBase = sanitizeNamePart(serviceGroup);
|
|
391
|
+
const hostBase = sanitizeNamePart(serverName);
|
|
336
392
|
const maxAllowed = resolveMaxInstances(serviceGroup, options.groupMaxInstances);
|
|
337
|
-
const
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
options.queue,
|
|
341
|
-
serviceGroup,
|
|
342
|
-
options.staleMs,
|
|
343
|
-
instanceId
|
|
344
|
-
);
|
|
345
|
-
if (maxAllowed > 0 && aliveOthers >= maxAllowed) {
|
|
346
|
-
const msg = `[services-registry] group limit reached for "${serviceGroup}": ${aliveOthers} alive (max ${maxAllowed}, queue=${options.queue}).`;
|
|
393
|
+
const aliveCount = await countAliveInGroup(db, registryTable, options.queueName, serviceGroup, options.staleMs, void 0);
|
|
394
|
+
if (maxAllowed > 0 && aliveCount >= maxAllowed) {
|
|
395
|
+
const msg = `[services-registry] group limit reached for "${serviceGroup}": ${aliveCount} alive (max ${maxAllowed}, queue=${options.queueName}).`;
|
|
347
396
|
if (options.enforceMaxInstances) {
|
|
348
397
|
throw new Error(msg);
|
|
349
398
|
}
|
|
350
399
|
context.logger.warn?.(`${msg} Starting anyway (runnerEnforceMaxInstances=false).`);
|
|
351
400
|
}
|
|
352
|
-
const
|
|
353
|
-
const
|
|
354
|
-
const
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
401
|
+
const maxSlots = maxAllowed > 0 ? maxAllowed : void 0;
|
|
402
|
+
const cutoff = new Date(Date.now() - options.staleMs);
|
|
403
|
+
const MAX_ATTEMPTS = 8;
|
|
404
|
+
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
|
|
405
|
+
const occupied = await getOccupiedInstanceSlots(db, registryTable, options.queueName, serviceGroup, options.staleMs);
|
|
406
|
+
const instanceNumber = allocateInstanceNumber(occupied, options.instanceNumber, maxSlots);
|
|
407
|
+
const serviceNameRaw = options.serviceName?.trim() ? sanitizeNamePart(options.serviceName.trim()) : defaultServiceName(groupBase, hostBase, instanceNumber);
|
|
408
|
+
const existing = await db(registryTable).where({ queue_name: options.queueName, service_name: serviceNameRaw }).first();
|
|
409
|
+
if (existing) {
|
|
410
|
+
const lastSeen = new Date(existing.last_seen_at);
|
|
411
|
+
const isAlive = !Number.isNaN(lastSeen.getTime()) && lastSeen > cutoff;
|
|
412
|
+
if (isAlive) {
|
|
413
|
+
if (options.serviceName?.trim()) {
|
|
414
|
+
throw new Error(
|
|
415
|
+
`[services-registry] service_name "${serviceNameRaw}" is already registered by an alive peer`
|
|
416
|
+
);
|
|
417
|
+
}
|
|
418
|
+
context.logger.warn?.(
|
|
419
|
+
`[services-registry] service_name "${serviceNameRaw}" already alive; retrying allocation (attempt ${attempt + 1})`
|
|
420
|
+
);
|
|
421
|
+
if (options.instanceNumber !== void 0 && options.instanceNumber !== null) {
|
|
422
|
+
throw new Error(
|
|
423
|
+
`[services-registry] instance slot ${instanceNumber} / name "${serviceNameRaw}" is already held by an alive peer`
|
|
424
|
+
);
|
|
425
|
+
}
|
|
426
|
+
await new Promise((r) => setTimeout(r, 50 + attempt * 30));
|
|
427
|
+
continue;
|
|
372
428
|
}
|
|
429
|
+
await db(registryTable).where({ id: existing.id }).update({
|
|
430
|
+
server_name: serverName,
|
|
431
|
+
pid,
|
|
432
|
+
metadata: meta,
|
|
433
|
+
service_group: serviceGroup,
|
|
434
|
+
instance_number: instanceNumber,
|
|
435
|
+
last_seen_at: db.fn.now()
|
|
436
|
+
});
|
|
437
|
+
const reg = {
|
|
438
|
+
serviceName: serviceNameRaw,
|
|
439
|
+
serviceGroup,
|
|
440
|
+
queueName: options.queueName,
|
|
441
|
+
target: options.target,
|
|
442
|
+
rowId: String(existing.id),
|
|
443
|
+
registryTable,
|
|
444
|
+
instanceNumber
|
|
445
|
+
};
|
|
446
|
+
context.servicesRegistry = reg;
|
|
447
|
+
context.runnerHeartbeat = reg;
|
|
448
|
+
context.logger.info?.(
|
|
449
|
+
`[services-registry] took over stale row name=${serviceNameRaw} instance=${instanceNumber} group=${serviceGroup} queue=${options.queueName}`
|
|
450
|
+
);
|
|
451
|
+
return reg;
|
|
373
452
|
}
|
|
374
|
-
}
|
|
375
|
-
let inserted;
|
|
376
|
-
for (const candidate of eachServiceNameCandidate(baseCandidates)) {
|
|
377
453
|
try {
|
|
378
454
|
const rows = await db(registryTable).insert({
|
|
379
|
-
|
|
380
|
-
queue: options.queue,
|
|
455
|
+
queue_name: options.queueName,
|
|
381
456
|
service_group: serviceGroup,
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
457
|
+
instance_number: instanceNumber,
|
|
458
|
+
service_name: serviceNameRaw,
|
|
459
|
+
server_name: serverName,
|
|
385
460
|
pid,
|
|
386
461
|
metadata: meta,
|
|
387
|
-
last_seen_at: db.fn.now()
|
|
462
|
+
last_seen_at: db.fn.now(),
|
|
463
|
+
created_at: db.fn.now()
|
|
388
464
|
}).returning(["id", "service_name"]);
|
|
389
465
|
const row = Array.isArray(rows) ? rows[0] : rows;
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
466
|
+
let rowId = row && typeof row === "object" ? String(row.id ?? "") : "";
|
|
467
|
+
if (!rowId) {
|
|
468
|
+
const again = await db(registryTable).where({ queue_name: options.queueName, service_name: serviceNameRaw }).first();
|
|
469
|
+
rowId = again?.id != null ? String(again.id) : "";
|
|
393
470
|
}
|
|
471
|
+
if (!rowId) continue;
|
|
472
|
+
const regNew = {
|
|
473
|
+
serviceName: String(row?.service_name ?? serviceNameRaw),
|
|
474
|
+
serviceGroup,
|
|
475
|
+
queueName: options.queueName,
|
|
476
|
+
target: options.target,
|
|
477
|
+
rowId,
|
|
478
|
+
registryTable,
|
|
479
|
+
instanceNumber
|
|
480
|
+
};
|
|
481
|
+
context.servicesRegistry = regNew;
|
|
482
|
+
context.runnerHeartbeat = regNew;
|
|
483
|
+
context.logger.info?.(
|
|
484
|
+
`[services-registry] registered name=${regNew.serviceName} instance=${instanceNumber} group=${serviceGroup} queue=${options.queueName}`
|
|
485
|
+
);
|
|
486
|
+
return regNew;
|
|
394
487
|
} catch (error) {
|
|
395
488
|
if (!isUniqueViolation(error)) {
|
|
396
489
|
throw error;
|
|
397
490
|
}
|
|
491
|
+
context.logger.warn?.(`[services-registry] insert race on "${serviceNameRaw}", retrying (attempt ${attempt + 1})`);
|
|
398
492
|
}
|
|
399
493
|
}
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
`[services-registry] could not allocate a unique service_name for group=${serviceGroup} queue=${options.queue} (too many collisions).`
|
|
403
|
-
);
|
|
404
|
-
}
|
|
405
|
-
identity.serviceName = inserted.service_name;
|
|
406
|
-
await writeIdentityFile(identityPath, identity);
|
|
407
|
-
const regNew = {
|
|
408
|
-
instanceId,
|
|
409
|
-
serviceName: inserted.service_name,
|
|
410
|
-
serviceGroup,
|
|
411
|
-
queue: options.queue,
|
|
412
|
-
target: options.target,
|
|
413
|
-
rowId: inserted.id
|
|
414
|
-
};
|
|
415
|
-
context.servicesRegistry = regNew;
|
|
416
|
-
context.runnerHeartbeat = regNew;
|
|
417
|
-
context.logger.info?.(
|
|
418
|
-
`[services-registry] registered instance_id=${instanceId} name=${inserted.service_name} group=${serviceGroup} queue=${options.queue} target=${options.target}`
|
|
494
|
+
throw new Error(
|
|
495
|
+
`[services-registry] could not allocate a registry row for group=${serviceGroup} queue=${options.queueName} after ${MAX_ATTEMPTS} attempts`
|
|
419
496
|
);
|
|
420
|
-
return {
|
|
421
|
-
instanceId,
|
|
422
|
-
serviceName: inserted.service_name,
|
|
423
|
-
serviceGroup,
|
|
424
|
-
queue: options.queue,
|
|
425
|
-
target: options.target,
|
|
426
|
-
rowId: inserted.id,
|
|
427
|
-
registryTable
|
|
428
|
-
};
|
|
429
497
|
}
|
|
430
498
|
async function touchServicesRegistry(context, registration) {
|
|
431
499
|
const db = getDb2(context);
|
|
432
|
-
const
|
|
500
|
+
const serverName = os.hostname();
|
|
433
501
|
const pid = typeof process.pid === "number" ? process.pid : null;
|
|
434
|
-
await db(registration.registryTable).where({
|
|
502
|
+
await db(registration.registryTable).where({ id: registration.rowId }).update({
|
|
435
503
|
last_seen_at: db.fn.now(),
|
|
436
|
-
|
|
504
|
+
server_name: serverName,
|
|
437
505
|
pid
|
|
438
506
|
});
|
|
439
507
|
}
|
|
440
508
|
async function updateServicesRegistryMetadata(context, registration, patch) {
|
|
441
509
|
const db = getDb2(context);
|
|
442
|
-
const row = await db(registration.registryTable).where({
|
|
510
|
+
const row = await db(registration.registryTable).where({ id: registration.rowId }).first();
|
|
443
511
|
const prev = parseMetadataColumn(row?.metadata);
|
|
444
512
|
const merged = { ...prev, ...patch };
|
|
445
|
-
await db(registration.registryTable).where({
|
|
513
|
+
await db(registration.registryTable).where({ id: registration.rowId }).update({
|
|
446
514
|
metadata: toJsonColumn(merged),
|
|
447
515
|
last_seen_at: db.fn.now()
|
|
448
516
|
});
|
|
@@ -450,14 +518,14 @@ async function updateServicesRegistryMetadata(context, registration, patch) {
|
|
|
450
518
|
}
|
|
451
519
|
async function unregisterServicesRegistry(context, registration) {
|
|
452
520
|
const db = getDb2(context);
|
|
453
|
-
await db(registration.registryTable).where({
|
|
454
|
-
context.logger.info?.(`[services-registry] unregistered name=${registration.serviceName}
|
|
521
|
+
await db(registration.registryTable).where({ id: registration.rowId }).delete();
|
|
522
|
+
context.logger.info?.(`[services-registry] unregistered name=${registration.serviceName} id=${registration.rowId}`);
|
|
455
523
|
}
|
|
456
|
-
async function listServicesRegistry(context, options = {
|
|
524
|
+
async function listServicesRegistry(context, options = { queueName: "tasks" }) {
|
|
457
525
|
const db = getDb2(context);
|
|
458
526
|
const staleMs = options.staleMs ?? 6e4;
|
|
459
527
|
const cutoff = new Date(Date.now() - staleMs);
|
|
460
|
-
const table =
|
|
528
|
+
const table = queueToTableNames(options.queueName).registryTable;
|
|
461
529
|
let q = db(table).where("last_seen_at", ">", cutoff).orderBy([{ column: "service_group", order: "asc" }, { column: "service_name", order: "asc" }]);
|
|
462
530
|
if (options.serviceGroup?.trim()) {
|
|
463
531
|
q = q.where({ service_group: options.serviceGroup.trim() });
|
|
@@ -465,11 +533,14 @@ async function listServicesRegistry(context, options = { queue: "tasks" }) {
|
|
|
465
533
|
return await q;
|
|
466
534
|
}
|
|
467
535
|
|
|
468
|
-
// src/
|
|
469
|
-
import fs3 from "fs";
|
|
536
|
+
// src/tasks/taskLogs.js
|
|
470
537
|
import path4 from "path";
|
|
471
538
|
|
|
472
|
-
// src/filedatabase/
|
|
539
|
+
// src/filedatabase/index.js
|
|
540
|
+
import fs3 from "fs";
|
|
541
|
+
import path3 from "path";
|
|
542
|
+
|
|
543
|
+
// src/filedatabase/serializers.js
|
|
473
544
|
function detectDataType(data) {
|
|
474
545
|
if (Array.isArray(data)) {
|
|
475
546
|
return "json-array";
|
|
@@ -501,7 +572,7 @@ function deserializeData(rawData, dataType) {
|
|
|
501
572
|
}
|
|
502
573
|
}
|
|
503
574
|
|
|
504
|
-
// src/errors.
|
|
575
|
+
// src/errors.js
|
|
505
576
|
var FrameworkError = class extends Error {
|
|
506
577
|
constructor(message) {
|
|
507
578
|
super(message);
|
|
@@ -521,7 +592,7 @@ var FileDatabaseError = class extends FrameworkError {
|
|
|
521
592
|
}
|
|
522
593
|
};
|
|
523
594
|
|
|
524
|
-
// src/filedatabase/index.
|
|
595
|
+
// src/filedatabase/index.js
|
|
525
596
|
var FileDatabase = class _FileDatabase {
|
|
526
597
|
basePath;
|
|
527
598
|
namespace;
|
|
@@ -607,14 +678,14 @@ var FileDatabase = class _FileDatabase {
|
|
|
607
678
|
if (errors.length) {
|
|
608
679
|
throw new FileDatabaseError(`[FileDatabase] ${errors.join("; ")}`);
|
|
609
680
|
}
|
|
610
|
-
|
|
681
|
+
const parts = [this.basePath, this.namespace];
|
|
611
682
|
if (this.tableName) {
|
|
612
683
|
parts.push(...this.tableName.split("/"));
|
|
613
684
|
}
|
|
614
685
|
if (this.versioned && version) {
|
|
615
686
|
parts.push(version);
|
|
616
687
|
}
|
|
617
|
-
return
|
|
688
|
+
return path3.resolve(...parts);
|
|
618
689
|
}
|
|
619
690
|
/**
|
|
620
691
|
* Set current version and version folder
|
|
@@ -651,7 +722,7 @@ var FileDatabase = class _FileDatabase {
|
|
|
651
722
|
this.currentFileNumber = 0;
|
|
652
723
|
const versions = await this.getVersions();
|
|
653
724
|
while (versions.length > this.maxVersions) {
|
|
654
|
-
const versionToDelete =
|
|
725
|
+
const versionToDelete = path3.resolve(this.getDestinationPath(), versions.shift());
|
|
655
726
|
this.logger.silly?.(`[FileDatabase] Deleting old version: ${versionToDelete}`);
|
|
656
727
|
await fs3.promises.rm(versionToDelete, { recursive: true, force: true });
|
|
657
728
|
}
|
|
@@ -670,7 +741,7 @@ var FileDatabase = class _FileDatabase {
|
|
|
670
741
|
await ensurePath(destPath);
|
|
671
742
|
const items = await fs3.promises.readdir(destPath);
|
|
672
743
|
const versions = items.filter((item) => {
|
|
673
|
-
const itemPath =
|
|
744
|
+
const itemPath = path3.join(destPath, item);
|
|
674
745
|
const stat = fs3.statSync(itemPath);
|
|
675
746
|
return stat.isDirectory() && isTimestampFolder(item);
|
|
676
747
|
});
|
|
@@ -727,7 +798,7 @@ var FileDatabase = class _FileDatabase {
|
|
|
727
798
|
const items = await fs3.promises.readdir(tablePath);
|
|
728
799
|
if (items.includes("metadata.json")) {
|
|
729
800
|
const metadata = JSON.parse(
|
|
730
|
-
await fs3.promises.readFile(
|
|
801
|
+
await fs3.promises.readFile(path3.join(tablePath, "metadata.json"), "utf8")
|
|
731
802
|
);
|
|
732
803
|
return {
|
|
733
804
|
versioned: false,
|
|
@@ -736,13 +807,13 @@ var FileDatabase = class _FileDatabase {
|
|
|
736
807
|
};
|
|
737
808
|
}
|
|
738
809
|
const versionFolders = items.filter((item) => {
|
|
739
|
-
const itemPath =
|
|
810
|
+
const itemPath = path3.join(tablePath, item);
|
|
740
811
|
const stat = fs3.statSync(itemPath);
|
|
741
812
|
return stat.isDirectory() && isTimestampFolder(item);
|
|
742
813
|
});
|
|
743
814
|
if (versionFolders.length > 0) {
|
|
744
815
|
const latestVersion = versionFolders.sort().pop();
|
|
745
|
-
const versionMetadataPath =
|
|
816
|
+
const versionMetadataPath = path3.join(tablePath, latestVersion, "metadata.json");
|
|
746
817
|
return {
|
|
747
818
|
versioned: true,
|
|
748
819
|
hasMetadata: fs3.existsSync(versionMetadataPath),
|
|
@@ -763,7 +834,7 @@ var FileDatabase = class _FileDatabase {
|
|
|
763
834
|
* Load metadata from JSON file
|
|
764
835
|
*/
|
|
765
836
|
async loadMetadataJson(version) {
|
|
766
|
-
const metadataFile =
|
|
837
|
+
const metadataFile = path3.join(this.getDestinationPath(), version, "metadata.json");
|
|
767
838
|
if (fs3.existsSync(metadataFile)) {
|
|
768
839
|
try {
|
|
769
840
|
const rawData = await fs3.promises.readFile(metadataFile, "utf8");
|
|
@@ -779,7 +850,7 @@ var FileDatabase = class _FileDatabase {
|
|
|
779
850
|
* Reads all files to get accurate counts - used when synopsis calculation is needed
|
|
780
851
|
*/
|
|
781
852
|
async figureMetadataFromVersionFiles(version) {
|
|
782
|
-
const versionPath =
|
|
853
|
+
const versionPath = path3.join(this.getDestinationPath(), version);
|
|
783
854
|
if (!fs3.existsSync(versionPath)) {
|
|
784
855
|
return this.getDefaultMetadata();
|
|
785
856
|
}
|
|
@@ -791,10 +862,10 @@ var FileDatabase = class _FileDatabase {
|
|
|
791
862
|
let detectedDataType = null;
|
|
792
863
|
for (let i = 0; i < files.length; i++) {
|
|
793
864
|
const fileName = files[i];
|
|
794
|
-
const filePath =
|
|
865
|
+
const filePath = path3.join(versionPath, fileName);
|
|
795
866
|
try {
|
|
796
867
|
const rawData = await fs3.promises.readFile(filePath, "utf8");
|
|
797
|
-
const extension =
|
|
868
|
+
const extension = path3.extname(fileName).toLowerCase();
|
|
798
869
|
let dataType = "text";
|
|
799
870
|
if (extension === ".json") {
|
|
800
871
|
dataType = "json-array";
|
|
@@ -827,7 +898,7 @@ var FileDatabase = class _FileDatabase {
|
|
|
827
898
|
* Much faster for large datasets with many files
|
|
828
899
|
*/
|
|
829
900
|
async buildMetadataOptimized(version) {
|
|
830
|
-
const versionPath =
|
|
901
|
+
const versionPath = path3.join(this.getDestinationPath(), version);
|
|
831
902
|
if (!fs3.existsSync(versionPath)) {
|
|
832
903
|
return this.getDefaultMetadata();
|
|
833
904
|
}
|
|
@@ -843,7 +914,7 @@ var FileDatabase = class _FileDatabase {
|
|
|
843
914
|
fileName
|
|
844
915
|
}));
|
|
845
916
|
const firstFile = metadata.files[0];
|
|
846
|
-
const firstFilePath =
|
|
917
|
+
const firstFilePath = path3.join(versionPath, firstFile.fileName);
|
|
847
918
|
const firstFileRaw = await fs3.promises.readFile(firstFilePath, "utf8");
|
|
848
919
|
let firstFileData;
|
|
849
920
|
try {
|
|
@@ -860,7 +931,7 @@ var FileDatabase = class _FileDatabase {
|
|
|
860
931
|
}
|
|
861
932
|
if (files.length > 1) {
|
|
862
933
|
const lastFile = metadata.files[metadata.files.length - 1];
|
|
863
|
-
const lastFilePath =
|
|
934
|
+
const lastFilePath = path3.join(versionPath, lastFile.fileName);
|
|
864
935
|
const lastFileRaw = await fs3.promises.readFile(lastFilePath, "utf8");
|
|
865
936
|
const lastFileData = deserializeData(lastFileRaw, metadata.dataType);
|
|
866
937
|
lastFile.recordsCount = Array.isArray(lastFileData) ? lastFileData.length : 1;
|
|
@@ -911,9 +982,9 @@ var FileDatabase = class _FileDatabase {
|
|
|
911
982
|
if (!this.currentVersion) {
|
|
912
983
|
return;
|
|
913
984
|
}
|
|
914
|
-
metadataFile =
|
|
985
|
+
metadataFile = path3.join(this.getDestinationPath(), this.currentVersion, "metadata.json");
|
|
915
986
|
} else {
|
|
916
|
-
metadataFile =
|
|
987
|
+
metadataFile = path3.join(this.getDestinationPath(), "metadata.json");
|
|
917
988
|
}
|
|
918
989
|
await fs3.promises.writeFile(metadataFile, JSON.stringify(metadataToSave, null, 4), "utf8");
|
|
919
990
|
}
|
|
@@ -976,7 +1047,7 @@ var FileDatabase = class _FileDatabase {
|
|
|
976
1047
|
}
|
|
977
1048
|
}
|
|
978
1049
|
if (!Array.isArray(data) && !forceNewFile) {
|
|
979
|
-
const lastFileExtension =
|
|
1050
|
+
const lastFileExtension = path3.extname(lastFile.fileName);
|
|
980
1051
|
const expectedExtension = `.${getFileExtension(incomingDataType)}`;
|
|
981
1052
|
if (lastFileExtension !== expectedExtension) {
|
|
982
1053
|
if (lastFileRecordsCount > 0) {
|
|
@@ -986,7 +1057,7 @@ var FileDatabase = class _FileDatabase {
|
|
|
986
1057
|
}
|
|
987
1058
|
}
|
|
988
1059
|
} else if (!Array.isArray(data) && forceNewFile) {
|
|
989
|
-
const lastFileExtension =
|
|
1060
|
+
const lastFileExtension = path3.extname(lastFile.fileName);
|
|
990
1061
|
const expectedExtension = `.${getFileExtension(incomingDataType)}`;
|
|
991
1062
|
if (lastFileExtension !== expectedExtension) {
|
|
992
1063
|
lastFile.fileName = `${this.currentFileNumber.toString().padStart(6, "0")}.${getFileExtension(incomingDataType)}`;
|
|
@@ -1076,7 +1147,7 @@ var FileDatabase = class _FileDatabase {
|
|
|
1076
1147
|
*/
|
|
1077
1148
|
async safeWrite(filePath, data) {
|
|
1078
1149
|
const serializedData = serializeData(data);
|
|
1079
|
-
const dir =
|
|
1150
|
+
const dir = path3.dirname(filePath);
|
|
1080
1151
|
const requiredBytes = Buffer.byteLength(serializedData, "utf8");
|
|
1081
1152
|
const freeBytes = getFreeDiskSpace(dir);
|
|
1082
1153
|
if (freeBytes !== null) {
|
|
@@ -1100,14 +1171,17 @@ var FileDatabase = class _FileDatabase {
|
|
|
1100
1171
|
* Prepare the instance for read or write operations
|
|
1101
1172
|
* This discovers state and sets up internal members based on mode and current data
|
|
1102
1173
|
*/
|
|
1103
|
-
async prepare(
|
|
1174
|
+
async prepare(options) {
|
|
1175
|
+
const { write, read, version, deferInitialVersion } = options;
|
|
1104
1176
|
if (write) {
|
|
1105
1177
|
if (this.versioned) {
|
|
1106
1178
|
if (this.currentVersion === null) {
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1179
|
+
if (!deferInitialVersion) {
|
|
1180
|
+
await this.makeNewVersion();
|
|
1181
|
+
this.metadata = this.getDefaultMetadata();
|
|
1182
|
+
this.metadata.version = this.currentVersion;
|
|
1183
|
+
this.makeNewFile();
|
|
1184
|
+
}
|
|
1111
1185
|
} else {
|
|
1112
1186
|
if (!this.metadata.files.length) {
|
|
1113
1187
|
this.metadata = await this.figureMetadata(this.currentVersion);
|
|
@@ -1121,7 +1195,7 @@ var FileDatabase = class _FileDatabase {
|
|
|
1121
1195
|
} else {
|
|
1122
1196
|
await ensurePath(this.getDestinationPath());
|
|
1123
1197
|
if (this.useMetadata === true) {
|
|
1124
|
-
const metadataPath =
|
|
1198
|
+
const metadataPath = path3.join(this.getDestinationPath(), "metadata.json");
|
|
1125
1199
|
if (fs3.existsSync(metadataPath)) {
|
|
1126
1200
|
try {
|
|
1127
1201
|
const rawData = await fs3.promises.readFile(metadataPath, "utf8");
|
|
@@ -1174,7 +1248,7 @@ var FileDatabase = class _FileDatabase {
|
|
|
1174
1248
|
}
|
|
1175
1249
|
if (this.useMetadata) {
|
|
1176
1250
|
const destPath = this.getDestinationPath();
|
|
1177
|
-
const metadataPath =
|
|
1251
|
+
const metadataPath = path3.join(destPath, "metadata.json");
|
|
1178
1252
|
if (fs3.existsSync(metadataPath)) {
|
|
1179
1253
|
try {
|
|
1180
1254
|
const rawData = await fs3.promises.readFile(metadataPath, "utf8");
|
|
@@ -1210,14 +1284,14 @@ var FileDatabase = class _FileDatabase {
|
|
|
1210
1284
|
if (options.filename) {
|
|
1211
1285
|
const destPath2 = this.getDestinationPath();
|
|
1212
1286
|
await ensurePath(destPath2);
|
|
1213
|
-
const filePath =
|
|
1287
|
+
const filePath = path3.join(destPath2, options.filename);
|
|
1214
1288
|
await this.safeWrite(filePath, data);
|
|
1215
1289
|
return;
|
|
1216
1290
|
}
|
|
1217
1291
|
if (options.forceNewVersion && !this.versioned) {
|
|
1218
1292
|
throw new FileDatabaseError("Cannot use forceNewVersion in non-versioned mode");
|
|
1219
1293
|
}
|
|
1220
|
-
await this.prepare({ write: true });
|
|
1294
|
+
await this.prepare({ write: true, deferInitialVersion: !!(options.forceNewVersion && this.versioned) });
|
|
1221
1295
|
const incomingDataType = detectDataType(data);
|
|
1222
1296
|
this.metadata.dataType = incomingDataType;
|
|
1223
1297
|
if (options.forceNewVersion) {
|
|
@@ -1259,11 +1333,11 @@ var FileDatabase = class _FileDatabase {
|
|
|
1259
1333
|
const forceNewFile = hasCustomMetadata && targetFileIndex === null;
|
|
1260
1334
|
let { dataToWrite, dataLeftOver, fileName } = this.figureOutDataAndFileToWrite(data, targetFileIndex, forceNewFile);
|
|
1261
1335
|
const destPath = this.getDestinationPath(this.currentVersion || void 0);
|
|
1262
|
-
await this.safeWrite(
|
|
1336
|
+
await this.safeWrite(path3.join(destPath, fileName), dataToWrite);
|
|
1263
1337
|
this.updateMetadata(dataToWrite, fileName, options.customMetadata);
|
|
1264
1338
|
while (dataLeftOver && dataLeftOver.length > 0 && targetFileIndex === null) {
|
|
1265
1339
|
const writeContext = this.figureOutDataAndFileToWrite(dataLeftOver);
|
|
1266
|
-
await this.safeWrite(
|
|
1340
|
+
await this.safeWrite(path3.join(destPath, writeContext.fileName), writeContext.dataToWrite);
|
|
1267
1341
|
this.updateMetadata(writeContext.dataToWrite, writeContext.fileName, options.customMetadata);
|
|
1268
1342
|
dataLeftOver = writeContext.dataLeftOver;
|
|
1269
1343
|
}
|
|
@@ -1279,7 +1353,7 @@ var FileDatabase = class _FileDatabase {
|
|
|
1279
1353
|
const { version, nextPage = false, pageSize, filename } = options;
|
|
1280
1354
|
if (filename) {
|
|
1281
1355
|
const destPath = this.getDestinationPath(version);
|
|
1282
|
-
const filePath =
|
|
1356
|
+
const filePath = path3.join(destPath, filename);
|
|
1283
1357
|
try {
|
|
1284
1358
|
const rawData = await fs3.promises.readFile(filePath, "utf8");
|
|
1285
1359
|
return JSON.parse(rawData);
|
|
@@ -1291,7 +1365,7 @@ var FileDatabase = class _FileDatabase {
|
|
|
1291
1365
|
const isNonPaginatedData = this.metadata.dataType === "text" || this.metadata.dataType === "xml" || this.metadata.dataType === "json-object";
|
|
1292
1366
|
if (isNonPaginatedData) {
|
|
1293
1367
|
const file = this.metadata.files[0];
|
|
1294
|
-
const filePath =
|
|
1368
|
+
const filePath = path3.join(this.getDestinationPath(this.currentVersion || void 0), file.fileName);
|
|
1295
1369
|
try {
|
|
1296
1370
|
const rawData = await fs3.promises.readFile(filePath, "utf8");
|
|
1297
1371
|
return deserializeData(rawData, this.metadata.dataType);
|
|
@@ -1329,7 +1403,7 @@ var FileDatabase = class _FileDatabase {
|
|
|
1329
1403
|
let cumulativeRecords = currentFileOffset;
|
|
1330
1404
|
for (let i = currentFileIndex; i < this.metadata.files.length && recordsRead < effectivePageSize; i++) {
|
|
1331
1405
|
const file = this.metadata.files[i];
|
|
1332
|
-
const filePath =
|
|
1406
|
+
const filePath = path3.join(this.getDestinationPath(this.currentVersion || void 0), file.fileName);
|
|
1333
1407
|
try {
|
|
1334
1408
|
const rawData = await fs3.promises.readFile(filePath, "utf8");
|
|
1335
1409
|
const fileData = deserializeData(rawData, this.metadata.dataType);
|
|
@@ -1373,7 +1447,7 @@ var FileDatabase = class _FileDatabase {
|
|
|
1373
1447
|
* Returns data file names (.json, .txt, .xml) excluding metadata.json.
|
|
1374
1448
|
*/
|
|
1375
1449
|
async listFilenames() {
|
|
1376
|
-
const destPath = this.versioned && this.currentVersion ?
|
|
1450
|
+
const destPath = this.versioned && this.currentVersion ? path3.join(this.getDestinationPath(), this.currentVersion) : this.getDestinationPath();
|
|
1377
1451
|
try {
|
|
1378
1452
|
const entries = await fs3.promises.readdir(destPath, { withFileTypes: true });
|
|
1379
1453
|
return entries.filter((e) => e.isFile() && e.name !== "metadata.json" && /\.(json|txt|xml)$/i.test(e.name)).map((e) => e.name);
|
|
@@ -1387,8 +1461,8 @@ var FileDatabase = class _FileDatabase {
|
|
|
1387
1461
|
* Use with listFilenames() to manage individual files.
|
|
1388
1462
|
*/
|
|
1389
1463
|
async removeFile(filename) {
|
|
1390
|
-
const destPath = this.versioned && this.currentVersion ?
|
|
1391
|
-
const filePath =
|
|
1464
|
+
const destPath = this.versioned && this.currentVersion ? path3.join(this.getDestinationPath(), this.currentVersion) : this.getDestinationPath();
|
|
1465
|
+
const filePath = path3.join(destPath, filename);
|
|
1392
1466
|
try {
|
|
1393
1467
|
await fs3.promises.unlink(filePath);
|
|
1394
1468
|
} catch (err) {
|
|
@@ -1414,7 +1488,7 @@ var FileDatabase = class _FileDatabase {
|
|
|
1414
1488
|
this.metadata.files.splice(idx, 1);
|
|
1415
1489
|
this.metadata.totalRecords = Math.max(0, (this.metadata.totalRecords || 0) - recordsCount);
|
|
1416
1490
|
const destPath = this.getDestinationPath();
|
|
1417
|
-
const filePath =
|
|
1491
|
+
const filePath = path3.join(destPath, filename);
|
|
1418
1492
|
try {
|
|
1419
1493
|
await fs3.promises.unlink(filePath);
|
|
1420
1494
|
} catch (err) {
|
|
@@ -1470,7 +1544,7 @@ var FileDatabase = class _FileDatabase {
|
|
|
1470
1544
|
});
|
|
1471
1545
|
if (matches) {
|
|
1472
1546
|
const destPath = this.getDestinationPath();
|
|
1473
|
-
const filePath =
|
|
1547
|
+
const filePath = path3.join(destPath, fileEntry.fileName);
|
|
1474
1548
|
const fileData = await fs3.promises.readFile(filePath, "utf8");
|
|
1475
1549
|
const data = deserializeData(fileData, metadata.dataType || "json-object");
|
|
1476
1550
|
results.push({
|
|
@@ -1493,7 +1567,7 @@ var FileDatabase = class _FileDatabase {
|
|
|
1493
1567
|
});
|
|
1494
1568
|
if (matches) {
|
|
1495
1569
|
const destPath = this.getDestinationPath(version);
|
|
1496
|
-
const filePath =
|
|
1570
|
+
const filePath = path3.join(destPath, fileEntry.fileName);
|
|
1497
1571
|
const fileData = await fs3.promises.readFile(filePath, "utf8");
|
|
1498
1572
|
const data = deserializeData(fileData, metadata.dataType || "json-object");
|
|
1499
1573
|
results.push({
|
|
@@ -1511,7 +1585,7 @@ var FileDatabase = class _FileDatabase {
|
|
|
1511
1585
|
}
|
|
1512
1586
|
};
|
|
1513
1587
|
|
|
1514
|
-
// src/tasks/taskLogs.
|
|
1588
|
+
// src/tasks/taskLogs.js
|
|
1515
1589
|
function getLogsState(context) {
|
|
1516
1590
|
const holder = context;
|
|
1517
1591
|
if (holder.__tasksLogsState) return holder.__tasksLogsState;
|
|
@@ -1564,6 +1638,96 @@ function getLogsState(context) {
|
|
|
1564
1638
|
holder.__tasksLogsState = state;
|
|
1565
1639
|
return state;
|
|
1566
1640
|
}
|
|
1641
|
+
function ipcLogTargetKey(target) {
|
|
1642
|
+
const bp = target.basePath ?? "";
|
|
1643
|
+
const ns = target.namespace ?? "";
|
|
1644
|
+
return `${bp}::${ns}::${target.tableName}`;
|
|
1645
|
+
}
|
|
1646
|
+
function ipcFileLogsTableNameForSourceResource(source, resource) {
|
|
1647
|
+
const seg = (s) => {
|
|
1648
|
+
const t = String(s).trim().replace(/[^a-zA-Z0-9._-]+/g, "_").replace(/^_+|_+$/g, "");
|
|
1649
|
+
return t.length ? t : "x";
|
|
1650
|
+
};
|
|
1651
|
+
return `${seg(source)}/${seg(resource)}`;
|
|
1652
|
+
}
|
|
1653
|
+
async function readTaskIpcLogsSnapshot(context, options) {
|
|
1654
|
+
const holder = context;
|
|
1655
|
+
const basePath = holder.params?.get?.("tasksLogsBasePath") ?? "./data";
|
|
1656
|
+
const namespace = holder.params?.get?.("tasksLogsNamespace") ?? "tasks-logs";
|
|
1657
|
+
const tableName = ipcFileLogsTableNameForSourceResource(options.source, options.resource);
|
|
1658
|
+
const tail = Math.max(1, Math.min(1e4, Number(options.tail) > 0 ? Number(options.tail) : 100));
|
|
1659
|
+
const fd = new FileDatabase({
|
|
1660
|
+
basePath,
|
|
1661
|
+
namespace,
|
|
1662
|
+
tableName,
|
|
1663
|
+
versioned: true,
|
|
1664
|
+
useMetadata: true,
|
|
1665
|
+
maxVersions: 30,
|
|
1666
|
+
pageSize: 2e3,
|
|
1667
|
+
logger: holder.logger
|
|
1668
|
+
});
|
|
1669
|
+
const versions = await fd.getVersions();
|
|
1670
|
+
if (versions.length === 0) {
|
|
1671
|
+
return { records: [], latestTs: null };
|
|
1672
|
+
}
|
|
1673
|
+
const latest = versions[versions.length - 1];
|
|
1674
|
+
const raw = await fd.read({ version: latest });
|
|
1675
|
+
const arr = Array.isArray(raw) ? raw : [];
|
|
1676
|
+
let filtered = arr;
|
|
1677
|
+
if (options.afterTs && String(options.afterTs).trim()) {
|
|
1678
|
+
const cut = String(options.afterTs).trim();
|
|
1679
|
+
filtered = arr.filter((r) => r && typeof r.ts === "string" && String(r.ts) > cut);
|
|
1680
|
+
}
|
|
1681
|
+
let latestTs = null;
|
|
1682
|
+
for (const r of filtered) {
|
|
1683
|
+
const ts = typeof r?.ts === "string" ? String(r.ts) : null;
|
|
1684
|
+
if (ts && (!latestTs || ts > latestTs)) latestTs = ts;
|
|
1685
|
+
}
|
|
1686
|
+
const incremental = !!(options.afterTs && String(options.afterTs).trim());
|
|
1687
|
+
const maxReturn = incremental ? 1e4 : tail;
|
|
1688
|
+
const sliced = filtered.length > maxReturn ? filtered.slice(-maxReturn) : filtered;
|
|
1689
|
+
return { records: sliced, latestTs };
|
|
1690
|
+
}
|
|
1691
|
+
function resolveIpcFileLogsDir(context, target) {
|
|
1692
|
+
const holder = context;
|
|
1693
|
+
const basePath = target.basePath ?? (holder.params?.get?.("tasksLogsBasePath") || "./data");
|
|
1694
|
+
const namespace = target.namespace ?? (holder.params?.get?.("tasksLogsNamespace") || "tasks-logs");
|
|
1695
|
+
const segments = target.tableName.split("/").filter(Boolean);
|
|
1696
|
+
return path4.resolve(basePath, namespace, ...segments);
|
|
1697
|
+
}
|
|
1698
|
+
function getLogsStateForTarget(context, target) {
|
|
1699
|
+
const holder = context;
|
|
1700
|
+
const enabledRaw = holder.params?.get?.("tasksLogsEnabled");
|
|
1701
|
+
const enabled = enabledRaw === void 0 ? true : !!enabledRaw;
|
|
1702
|
+
if (!enabled) return null;
|
|
1703
|
+
if (!holder.__tasksLogsTargetStates) holder.__tasksLogsTargetStates = /* @__PURE__ */ new Map();
|
|
1704
|
+
const map = holder.__tasksLogsTargetStates;
|
|
1705
|
+
const key = ipcLogTargetKey(target);
|
|
1706
|
+
if (map.has(key)) return map.get(key);
|
|
1707
|
+
const basePath = target.basePath ?? (holder.params?.get?.("tasksLogsBasePath") || "./data");
|
|
1708
|
+
const namespace = target.namespace ?? (holder.params?.get?.("tasksLogsNamespace") || "tasks-logs");
|
|
1709
|
+
const maxVersionsRaw = Number(holder.params?.get?.("tasksLogsMaxVersions"));
|
|
1710
|
+
const pageSizeRaw = Number(holder.params?.get?.("tasksLogsPageSize"));
|
|
1711
|
+
const db = new FileDatabase({
|
|
1712
|
+
basePath,
|
|
1713
|
+
namespace,
|
|
1714
|
+
tableName: target.tableName,
|
|
1715
|
+
versioned: true,
|
|
1716
|
+
useMetadata: true,
|
|
1717
|
+
maxVersions: Number.isFinite(maxVersionsRaw) && maxVersionsRaw > 0 ? maxVersionsRaw : 20,
|
|
1718
|
+
pageSize: Number.isFinite(pageSizeRaw) && pageSizeRaw > 0 ? pageSizeRaw : 2e3,
|
|
1719
|
+
logger: holder.logger
|
|
1720
|
+
});
|
|
1721
|
+
const state = {
|
|
1722
|
+
db,
|
|
1723
|
+
errorDb: null,
|
|
1724
|
+
queue: Promise.resolve(),
|
|
1725
|
+
initialized: false,
|
|
1726
|
+
errorInitialized: false
|
|
1727
|
+
};
|
|
1728
|
+
map.set(key, state);
|
|
1729
|
+
return state;
|
|
1730
|
+
}
|
|
1567
1731
|
function isErrorPayload(payload) {
|
|
1568
1732
|
if (!payload) return false;
|
|
1569
1733
|
if (typeof payload === "object") {
|
|
@@ -1583,14 +1747,26 @@ function buildLogRecord(task, payload) {
|
|
|
1583
1747
|
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1584
1748
|
opid: task.opid ?? null,
|
|
1585
1749
|
taskId: task.id,
|
|
1586
|
-
taskName: task.
|
|
1587
|
-
target: task.
|
|
1750
|
+
taskName: task.name,
|
|
1751
|
+
target: task.service_group,
|
|
1588
1752
|
source: typeof params.source === "string" ? params.source : null,
|
|
1589
1753
|
resource: typeof params.resource === "string" ? params.resource : null,
|
|
1590
1754
|
payload
|
|
1591
1755
|
};
|
|
1592
1756
|
}
|
|
1593
|
-
function appendTaskIpcLog(context, task, payload) {
|
|
1757
|
+
function appendTaskIpcLog(context, task, payload, target) {
|
|
1758
|
+
if (target) {
|
|
1759
|
+
const state2 = getLogsStateForTarget(context, target);
|
|
1760
|
+
if (!state2?.db) return;
|
|
1761
|
+
const record2 = buildLogRecord(task, payload);
|
|
1762
|
+
state2.queue = state2.queue.then(async () => {
|
|
1763
|
+
await state2.db.write([record2], { forceNewVersion: !state2.initialized });
|
|
1764
|
+
state2.initialized = true;
|
|
1765
|
+
}).catch((error) => {
|
|
1766
|
+
context.logger.warn?.("[tasks] failed to persist IPC log entry (targeted):", error);
|
|
1767
|
+
});
|
|
1768
|
+
return;
|
|
1769
|
+
}
|
|
1594
1770
|
const state = getLogsState(context);
|
|
1595
1771
|
if (!state.db && !state.errorDb) return;
|
|
1596
1772
|
const record = buildLogRecord(task, payload);
|
|
@@ -1607,84 +1783,293 @@ function appendTaskIpcLog(context, task, payload) {
|
|
|
1607
1783
|
context.logger.warn?.("[tasks] failed to persist IPC log entry:", error);
|
|
1608
1784
|
});
|
|
1609
1785
|
}
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
let current = field;
|
|
1619
|
-
while (true) {
|
|
1620
|
-
const match = regex.exec(current);
|
|
1621
|
-
if (!match) break;
|
|
1622
|
-
const raw = match[0];
|
|
1623
|
-
let first = Number(match[1]);
|
|
1624
|
-
let last = Number(match[2]);
|
|
1625
|
-
if (last < first) {
|
|
1626
|
-
[first, last] = [last, first];
|
|
1627
|
-
}
|
|
1628
|
-
const values = [];
|
|
1629
|
-
for (let i = first; i <= last; i += 1) {
|
|
1630
|
-
values.push(i);
|
|
1786
|
+
async function flushTaskIpcLogs(context) {
|
|
1787
|
+
const holder = context;
|
|
1788
|
+
const promises = [];
|
|
1789
|
+
if (holder.__tasksLogsState?.queue) promises.push(holder.__tasksLogsState.queue);
|
|
1790
|
+
const map = holder.__tasksLogsTargetStates;
|
|
1791
|
+
if (map) {
|
|
1792
|
+
for (const s of map.values()) {
|
|
1793
|
+
if (s.queue) promises.push(s.queue);
|
|
1631
1794
|
}
|
|
1632
|
-
current = current.replace(raw, values.join(","));
|
|
1633
|
-
}
|
|
1634
|
-
return current;
|
|
1635
|
-
}
|
|
1636
|
-
function resolveSteps(field) {
|
|
1637
|
-
const match = /^(.+)\/(\d+)$/.exec(field);
|
|
1638
|
-
if (!match) return field;
|
|
1639
|
-
const base = match[1];
|
|
1640
|
-
const step = Number(match[2]);
|
|
1641
|
-
if (!Number.isFinite(step) || step <= 0) return field;
|
|
1642
|
-
return base.split(",").map((v) => Number(v)).filter((v) => Number.isFinite(v) && v % step === 0).join(",");
|
|
1643
|
-
}
|
|
1644
|
-
function convertPattern(pattern) {
|
|
1645
|
-
const parts = pattern.trim().split(/\s+/);
|
|
1646
|
-
if (parts.length !== 6) {
|
|
1647
|
-
throw new Error(`Invalid schedule "${pattern}". Expected 6 fields: sec min hour day month weekday`);
|
|
1648
1795
|
}
|
|
1649
|
-
|
|
1650
|
-
}
|
|
1651
|
-
function fieldMatches(field, value) {
|
|
1652
|
-
const allowed = field.split(",").map((v) => Number(v));
|
|
1653
|
-
return allowed.includes(value);
|
|
1654
|
-
}
|
|
1655
|
-
function timeMatcher(pattern, date = /* @__PURE__ */ new Date()) {
|
|
1656
|
-
const parsed = convertPattern(pattern);
|
|
1657
|
-
return fieldMatches(parsed[0], date.getSeconds()) && fieldMatches(parsed[1], date.getMinutes()) && fieldMatches(parsed[2], date.getHours()) && fieldMatches(parsed[3], date.getDate()) && fieldMatches(parsed[4], date.getMonth() + 1) && fieldMatches(parsed[5], date.getDay());
|
|
1796
|
+
await Promise.all(promises);
|
|
1658
1797
|
}
|
|
1659
1798
|
|
|
1660
|
-
// src/tasks/
|
|
1661
|
-
var
|
|
1662
|
-
|
|
1663
|
-
|
|
1799
|
+
// src/tasks/AbstractTask.js
|
|
1800
|
+
var AbstractTask = class _AbstractTask {
|
|
1801
|
+
/**
|
|
1802
|
+
* Whether `send-task` should wait for completion (and print a result
|
|
1803
|
+
* report) when no explicit `--wait` / `--noWait` flag is given. Defaults
|
|
1804
|
+
* to false; short-lived probe tasks (e.g. `ping`) override to true.
|
|
1805
|
+
*
|
|
1806
|
+
* @type {boolean}
|
|
1807
|
+
*/
|
|
1808
|
+
static defaultWaitForResult = false;
|
|
1809
|
+
/**
|
|
1810
|
+
* @param {object} context Runner context (db, logger, params, emitter...).
|
|
1811
|
+
* @param {object} task Task row as claimed from the queue.
|
|
1812
|
+
*/
|
|
1664
1813
|
constructor(context, task) {
|
|
1665
1814
|
this.context = context;
|
|
1666
1815
|
this.task = task;
|
|
1667
1816
|
}
|
|
1817
|
+
/**
|
|
1818
|
+
* Return a short reason string when the task should be deferred (e.g. "locked
|
|
1819
|
+
* by source"), or `false`/falsy when it is free to run. Default: always `false`.
|
|
1820
|
+
*
|
|
1821
|
+
* @returns {string | false | Promise<string | false>}
|
|
1822
|
+
*/
|
|
1668
1823
|
cantRunReason() {
|
|
1669
1824
|
return false;
|
|
1670
1825
|
}
|
|
1826
|
+
/**
|
|
1827
|
+
* Called by the runner when a stop has been requested. Subclasses running
|
|
1828
|
+
* long loops should flip a flag here and check it between iterations.
|
|
1829
|
+
*
|
|
1830
|
+
* @param {number} [_allowanceMs] Grace period the runner promises before hard exit.
|
|
1831
|
+
*/
|
|
1671
1832
|
requestStop(_allowanceMs) {
|
|
1672
1833
|
}
|
|
1834
|
+
/**
|
|
1835
|
+
* Perform the task. Must be implemented by subclasses.
|
|
1836
|
+
*
|
|
1837
|
+
* @param {(progress: unknown) => Promise<void>} _reportProgress
|
|
1838
|
+
* Updates the DB `progress` column. Accepts any serializable value;
|
|
1839
|
+
* strings are stored verbatim, objects are JSON-stringified.
|
|
1840
|
+
* @returns {Promise<{ success: boolean, results: unknown }>}
|
|
1841
|
+
*/
|
|
1842
|
+
async run(_reportProgress) {
|
|
1843
|
+
throw new Error("AbstractTask.run must be implemented by subclass");
|
|
1844
|
+
}
|
|
1845
|
+
/**
|
|
1846
|
+
* Resolve a complete row payload for this task — envelope fields (queue,
|
|
1847
|
+
* priority, targeting, schedule…) plus the inner `params` blob produced by
|
|
1848
|
+
* {@link AbstractTask.resolveCustomParams}. Output shape matches
|
|
1849
|
+
* {@link enqueueTask}'s `options` argument, so the typical call is:
|
|
1850
|
+
*
|
|
1851
|
+
* const payload = await TaskClass.resolveParams(context, { name });
|
|
1852
|
+
* await enqueueTask(context, payload);
|
|
1853
|
+
*
|
|
1854
|
+
* Validation failures throw {@link ParamError} so the script aborts before
|
|
1855
|
+
* a malformed row hits the DB.
|
|
1856
|
+
*
|
|
1857
|
+
* @param {object} context
|
|
1858
|
+
* @param {Record<string, unknown>} [overrides] Partial overrides; takes precedence over CLI/env.
|
|
1859
|
+
* Recognised keys: `name`, `queueName`, `priority`, `serviceGroup`,
|
|
1860
|
+
* `serviceName`, `instanceNumber`, `serverName`, `opid`, `schedule`,
|
|
1861
|
+
* `nextRunAt`, plus `params` (object — overlay onto inner blob).
|
|
1862
|
+
* @returns {Promise<object>}
|
|
1863
|
+
*/
|
|
1864
|
+
static async resolveParams(context, overrides = {}) {
|
|
1865
|
+
const main = _AbstractTask._resolveMainFields(context, overrides);
|
|
1866
|
+
const params = await this.resolveCustomParams(context, overrides);
|
|
1867
|
+
return { ...main, params };
|
|
1868
|
+
}
|
|
1869
|
+
/**
|
|
1870
|
+
* Resolve the inner JSON blob stored in the `params` column. Default
|
|
1871
|
+
* implementation passes through `--paramsJson` (parsed as a JSON object)
|
|
1872
|
+
* overlaid with `overrides.params` when supplied; returns `null` when
|
|
1873
|
+
* neither is provided.
|
|
1874
|
+
*
|
|
1875
|
+
* Subclasses with typed fields should override and call
|
|
1876
|
+
* {@link AbstractTask._mergeTypedParams} for layered CLI/env/JSON/override
|
|
1877
|
+
* resolution, then validate and throw {@link ParamError} on bad input.
|
|
1878
|
+
*
|
|
1879
|
+
* @param {object} context
|
|
1880
|
+
* @param {Record<string, unknown>} [overrides]
|
|
1881
|
+
* @returns {Promise<object|null>}
|
|
1882
|
+
*/
|
|
1883
|
+
static async resolveCustomParams(context, overrides = {}) {
|
|
1884
|
+
return _AbstractTask._defaultParamsBlob(context, overrides);
|
|
1885
|
+
}
|
|
1886
|
+
/**
|
|
1887
|
+
* Read main task envelope fields from `context.params` (CLI/env), with
|
|
1888
|
+
* any matching key on `overrides` taking precedence. Internal; called by
|
|
1889
|
+
* {@link AbstractTask.resolveParams}.
|
|
1890
|
+
*
|
|
1891
|
+
* @param {object} context
|
|
1892
|
+
* @param {Record<string, unknown>} [overrides]
|
|
1893
|
+
* @returns {object}
|
|
1894
|
+
*/
|
|
1895
|
+
static _resolveMainFields(context, overrides = {}) {
|
|
1896
|
+
const defs = {
|
|
1897
|
+
queueName: "string default tasks",
|
|
1898
|
+
priority: "number default 50",
|
|
1899
|
+
serviceGroup: "string",
|
|
1900
|
+
serviceName: "string",
|
|
1901
|
+
instanceNumber: "number",
|
|
1902
|
+
serverName: "string",
|
|
1903
|
+
opid: "string",
|
|
1904
|
+
schedule: "string"
|
|
1905
|
+
};
|
|
1906
|
+
const cli = context.params.getAllForModule("task-envelope", defs);
|
|
1907
|
+
const name = emptyToUndef(overrides.name) ?? emptyToUndef(context.params.get("name", "string"));
|
|
1908
|
+
if (!name) {
|
|
1909
|
+
throw new ParamError("Task --name is required (e.g. ping, stop, dummyHarvest)");
|
|
1910
|
+
}
|
|
1911
|
+
let instanceNumber;
|
|
1912
|
+
const rawInstance = overrides.instanceNumber ?? cli.instanceNumber;
|
|
1913
|
+
if (rawInstance !== void 0 && rawInstance !== null && String(rawInstance).trim() !== "") {
|
|
1914
|
+
const n = Number(rawInstance);
|
|
1915
|
+
if (!Number.isFinite(n) || !Number.isInteger(n) || n < 1) {
|
|
1916
|
+
throw new ParamError("--instanceNumber must be a positive integer when set");
|
|
1917
|
+
}
|
|
1918
|
+
instanceNumber = n;
|
|
1919
|
+
} else {
|
|
1920
|
+
instanceNumber = null;
|
|
1921
|
+
}
|
|
1922
|
+
const priorityRaw = overrides.priority ?? cli.priority ?? 50;
|
|
1923
|
+
const priority = Number(priorityRaw);
|
|
1924
|
+
if (!Number.isFinite(priority)) {
|
|
1925
|
+
throw new ParamError(`--priority must be a number (got ${JSON.stringify(priorityRaw)})`);
|
|
1926
|
+
}
|
|
1927
|
+
return {
|
|
1928
|
+
name,
|
|
1929
|
+
queueName: overrides.queueName ?? emptyToUndef(cli.queueName) ?? "tasks",
|
|
1930
|
+
priority,
|
|
1931
|
+
serviceGroup: emptyToUndef(overrides.serviceGroup) ?? emptyToUndef(cli.serviceGroup) ?? null,
|
|
1932
|
+
serviceName: emptyToUndef(overrides.serviceName) ?? emptyToUndef(cli.serviceName) ?? null,
|
|
1933
|
+
instanceNumber,
|
|
1934
|
+
serverName: emptyToUndef(overrides.serverName) ?? emptyToUndef(cli.serverName) ?? null,
|
|
1935
|
+
opid: emptyToUndef(overrides.opid) ?? emptyToUndef(cli.opid) ?? null,
|
|
1936
|
+
schedule: emptyToUndef(overrides.schedule) ?? emptyToUndef(cli.schedule) ?? null,
|
|
1937
|
+
nextRunAt: overrides.nextRunAt ?? null
|
|
1938
|
+
};
|
|
1939
|
+
}
|
|
1940
|
+
/**
|
|
1941
|
+
* Default inner-params resolver: parses `--paramsJson` (must be a JSON
|
|
1942
|
+
* object), then overlays `overrides.params` on top. Returns `null` when
|
|
1943
|
+
* neither is provided.
|
|
1944
|
+
*
|
|
1945
|
+
* @param {object} context
|
|
1946
|
+
* @param {Record<string, unknown>} [overrides]
|
|
1947
|
+
* @returns {object|null}
|
|
1948
|
+
*/
|
|
1949
|
+
static _defaultParamsBlob(context, overrides = {}) {
|
|
1950
|
+
const cli = context.params.getAllForModule("task-params", { paramsJson: "string" });
|
|
1951
|
+
const fromJson = parseParamsJson(cli.paramsJson);
|
|
1952
|
+
const fromOverride = pickParamsObject(overrides);
|
|
1953
|
+
if (!fromJson && !fromOverride) return null;
|
|
1954
|
+
return { ...fromJson ?? {}, ...fromOverride ?? {} };
|
|
1955
|
+
}
|
|
1956
|
+
/**
|
|
1957
|
+
* Helper for subclass `resolveCustomParams` overrides. Reads typed CLI
|
|
1958
|
+
* params (per `defs`) plus `--paramsJson` under a module namespace, then
|
|
1959
|
+
* merges them with explicit `overrides.params` in increasing priority:
|
|
1960
|
+
*
|
|
1961
|
+
* typed CLI flags → --paramsJson → overrides.params
|
|
1962
|
+
*
|
|
1963
|
+
* Undefined values are dropped so defaults declared in `defs` aren't
|
|
1964
|
+
* overwritten by missing-flag noise. Returns the merged object; the
|
|
1965
|
+
* caller is responsible for validation and throwing `ParamError`.
|
|
1966
|
+
*
|
|
1967
|
+
* @param {object} context
|
|
1968
|
+
* @param {string} moduleName Namespace for `--showUsedParams` grouping.
|
|
1969
|
+
* @param {Record<string, string>} defs Param defs in `getAllForModule` syntax.
|
|
1970
|
+
* @param {Record<string, unknown>} [overrides] As passed to `resolveCustomParams`.
|
|
1971
|
+
* @returns {Record<string, unknown>}
|
|
1972
|
+
*/
|
|
1973
|
+
static _mergeTypedParams(context, moduleName, defs, overrides = {}) {
|
|
1974
|
+
const fullDefs = { ...defs, paramsJson: "string" };
|
|
1975
|
+
const cliRaw = context.params.getAllForModule(moduleName, fullDefs);
|
|
1976
|
+
const fromJson = parseParamsJson(cliRaw.paramsJson) ?? {};
|
|
1977
|
+
const fromCli = {};
|
|
1978
|
+
for (const [k, v] of Object.entries(cliRaw)) {
|
|
1979
|
+
if (k === "paramsJson") continue;
|
|
1980
|
+
if (v !== void 0 && v !== null) fromCli[k] = v;
|
|
1981
|
+
}
|
|
1982
|
+
const fromOverride = pickParamsObject(overrides) ?? {};
|
|
1983
|
+
return { ...fromCli, ...fromJson, ...fromOverride };
|
|
1984
|
+
}
|
|
1673
1985
|
};
|
|
1986
|
+
function emptyToUndef(s) {
|
|
1987
|
+
if (s === void 0 || s === null) return void 0;
|
|
1988
|
+
if (typeof s !== "string") return s;
|
|
1989
|
+
const t = s.trim();
|
|
1990
|
+
return t.length ? t : void 0;
|
|
1991
|
+
}
|
|
1992
|
+
function parseParamsJson(raw) {
|
|
1993
|
+
if (raw == null) return null;
|
|
1994
|
+
const t = String(raw).trim();
|
|
1995
|
+
if (!t) return null;
|
|
1996
|
+
let parsed;
|
|
1997
|
+
try {
|
|
1998
|
+
parsed = JSON.parse(t);
|
|
1999
|
+
} catch (e) {
|
|
2000
|
+
throw new ParamError(`--paramsJson: not valid JSON: ${e?.message ?? String(e)}`);
|
|
2001
|
+
}
|
|
2002
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
2003
|
+
throw new ParamError("--paramsJson must be a JSON object");
|
|
2004
|
+
}
|
|
2005
|
+
return parsed;
|
|
2006
|
+
}
|
|
2007
|
+
function pickParamsObject(overrides) {
|
|
2008
|
+
const p = overrides?.params;
|
|
2009
|
+
if (p && typeof p === "object" && !Array.isArray(p)) return p;
|
|
2010
|
+
return void 0;
|
|
2011
|
+
}
|
|
1674
2012
|
|
|
1675
|
-
// src/tasks/coreTasks/TaskPing.
|
|
1676
|
-
var TaskPing = class extends
|
|
2013
|
+
// src/tasks/coreTasks/TaskPing.js
|
|
2014
|
+
var TaskPing = class extends AbstractTask {
|
|
2015
|
+
/** Default-wait so `send-task --name=ping` prints a result without `--wait`. */
|
|
2016
|
+
static defaultWaitForResult = true;
|
|
2017
|
+
/** Ping takes no params. */
|
|
2018
|
+
static async resolveCustomParams() {
|
|
2019
|
+
return null;
|
|
2020
|
+
}
|
|
2021
|
+
/**
|
|
2022
|
+
* @returns {Promise<{ success: true, results: "pong" }>}
|
|
2023
|
+
*/
|
|
1677
2024
|
async run() {
|
|
1678
2025
|
this.context.logger.info?.(`[TaskPing] pong (${this.task.id})`);
|
|
1679
2026
|
return { success: true, results: "pong" };
|
|
1680
2027
|
}
|
|
1681
2028
|
};
|
|
1682
2029
|
|
|
1683
|
-
// src/tasks/coreTasks/TaskSampleProcess.
|
|
1684
|
-
var TaskSampleProcess = class extends
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
2030
|
+
// src/tasks/coreTasks/TaskSampleProcess.js
|
|
2031
|
+
var TaskSampleProcess = class extends AbstractTask {
|
|
2032
|
+
/**
|
|
2033
|
+
* @param {object} context
|
|
2034
|
+
* @param {Record<string, unknown>} [overrides]
|
|
2035
|
+
* @returns {Promise<{ total: number, delay: number, name?: string }>}
|
|
2036
|
+
*/
|
|
2037
|
+
static async resolveCustomParams(context, overrides = {}) {
|
|
2038
|
+
const merged = AbstractTask._mergeTypedParams(context, "task-sample-process", {
|
|
2039
|
+
total: "number default 10",
|
|
2040
|
+
delay: "number default 1000",
|
|
2041
|
+
name: "string"
|
|
2042
|
+
}, overrides);
|
|
2043
|
+
const total = Number(merged.total);
|
|
2044
|
+
const delay = Number(merged.delay);
|
|
2045
|
+
if (!Number.isInteger(total) || total <= 0) {
|
|
2046
|
+
throw new ParamError(`sampleProcess: param "total" must be a positive integer (got ${JSON.stringify(merged.total)})`);
|
|
2047
|
+
}
|
|
2048
|
+
if (!Number.isInteger(delay) || delay < 0) {
|
|
2049
|
+
throw new ParamError(`sampleProcess: param "delay" must be an integer >= 0 (got ${JSON.stringify(merged.delay)})`);
|
|
2050
|
+
}
|
|
2051
|
+
const out = { total, delay };
|
|
2052
|
+
if (typeof merged.name === "string" && merged.name.trim()) {
|
|
2053
|
+
out.name = merged.name.trim();
|
|
2054
|
+
}
|
|
2055
|
+
return out;
|
|
2056
|
+
}
|
|
2057
|
+
/**
|
|
2058
|
+
* @param {object} context
|
|
2059
|
+
* @param {object} task
|
|
2060
|
+
*/
|
|
2061
|
+
constructor(context, task) {
|
|
2062
|
+
super(context, task);
|
|
2063
|
+
this.stopRequested = false;
|
|
2064
|
+
this.stopAllowanceMs = 0;
|
|
2065
|
+
this.stopDecisionLogged = false;
|
|
2066
|
+
}
|
|
2067
|
+
/**
|
|
2068
|
+
* Runner-facing stop signal. Records the allowance window so the main loop
|
|
2069
|
+
* can decide per-iteration whether to finish or abort early.
|
|
2070
|
+
*
|
|
2071
|
+
* @param {number} allowanceMs
|
|
2072
|
+
*/
|
|
1688
2073
|
requestStop(allowanceMs) {
|
|
1689
2074
|
this.stopRequested = true;
|
|
1690
2075
|
this.stopAllowanceMs = Number.isFinite(allowanceMs) && allowanceMs > 0 ? allowanceMs : 0;
|
|
@@ -1692,6 +2077,14 @@ var TaskSampleProcess = class extends TaskMaster {
|
|
|
1692
2077
|
`[TaskSampleProcess] stop signal received (${this.task.id}), allowanceMs=${this.stopAllowanceMs}`
|
|
1693
2078
|
);
|
|
1694
2079
|
}
|
|
2080
|
+
/**
|
|
2081
|
+
* Iterate `total` times, sleeping `delay` ms between ticks and reporting
|
|
2082
|
+
* progress every iteration. Validates params up front; invalid values short-
|
|
2083
|
+
* circuit to a structured failure without starting the loop.
|
|
2084
|
+
*
|
|
2085
|
+
* @param {(progress: object) => Promise<void>} reportProgress
|
|
2086
|
+
* @returns {Promise<{ success: boolean, results: unknown }>}
|
|
2087
|
+
*/
|
|
1695
2088
|
async run(reportProgress) {
|
|
1696
2089
|
const totalRaw = this.task?.params?.total ?? 10;
|
|
1697
2090
|
const delayRaw = this.task?.params?.delay ?? 1e3;
|
|
@@ -1773,7 +2166,7 @@ var TaskSampleProcess = class extends TaskMaster {
|
|
|
1773
2166
|
}
|
|
1774
2167
|
};
|
|
1775
2168
|
|
|
1776
|
-
// src/tasks/coreTasks/TaskShellCommand.
|
|
2169
|
+
// src/tasks/coreTasks/TaskShellCommand.js
|
|
1777
2170
|
import { spawn } from "child_process";
|
|
1778
2171
|
function runShellCommand(command, cwd) {
|
|
1779
2172
|
return new Promise((resolve, reject) => {
|
|
@@ -1803,7 +2196,27 @@ function runShellCommand(command, cwd) {
|
|
|
1803
2196
|
});
|
|
1804
2197
|
});
|
|
1805
2198
|
}
|
|
1806
|
-
var TaskShellCommand = class extends
|
|
2199
|
+
var TaskShellCommand = class extends AbstractTask {
|
|
2200
|
+
/**
|
|
2201
|
+
* @param {object} context
|
|
2202
|
+
* @param {Record<string, unknown>} [overrides]
|
|
2203
|
+
* @returns {Promise<{ command: string, cwd?: string }>}
|
|
2204
|
+
*/
|
|
2205
|
+
static async resolveCustomParams(context, overrides = {}) {
|
|
2206
|
+
const merged = AbstractTask._mergeTypedParams(context, "task-shell", {
|
|
2207
|
+
command: "string",
|
|
2208
|
+
cwd: "string"
|
|
2209
|
+
}, overrides);
|
|
2210
|
+
const command = typeof merged.command === "string" ? merged.command.trim() : "";
|
|
2211
|
+
if (!command) {
|
|
2212
|
+
throw new ParamError('shellCommand: param "command" must be a non-empty string');
|
|
2213
|
+
}
|
|
2214
|
+
const cwd = typeof merged.cwd === "string" && merged.cwd.trim() ? merged.cwd.trim() : null;
|
|
2215
|
+
return cwd ? { command, cwd } : { command };
|
|
2216
|
+
}
|
|
2217
|
+
/**
|
|
2218
|
+
* @returns {Promise<{ success: boolean, results: unknown }>}
|
|
2219
|
+
*/
|
|
1807
2220
|
async run() {
|
|
1808
2221
|
const params = this.task?.params;
|
|
1809
2222
|
const commandRaw = typeof params === "string" ? params : params?.command;
|
|
@@ -1852,7 +2265,7 @@ var TaskShellCommand = class extends TaskMaster {
|
|
|
1852
2265
|
}
|
|
1853
2266
|
};
|
|
1854
2267
|
|
|
1855
|
-
// src/tasks/coreTasks/TaskSystemInfo.
|
|
2268
|
+
// src/tasks/coreTasks/TaskSystemInfo.js
|
|
1856
2269
|
import os2 from "os";
|
|
1857
2270
|
import fs4 from "fs/promises";
|
|
1858
2271
|
function toGb(valueBytes) {
|
|
@@ -1872,7 +2285,16 @@ async function getDiskStats() {
|
|
|
1872
2285
|
free: toGb(free)
|
|
1873
2286
|
};
|
|
1874
2287
|
}
|
|
1875
|
-
var TaskSystemInfo = class extends
|
|
2288
|
+
var TaskSystemInfo = class extends AbstractTask {
|
|
2289
|
+
/** Same UX expectation as `ping` — short probe, print the result. */
|
|
2290
|
+
static defaultWaitForResult = true;
|
|
2291
|
+
/** systemInfo takes no params. */
|
|
2292
|
+
static async resolveCustomParams() {
|
|
2293
|
+
return null;
|
|
2294
|
+
}
|
|
2295
|
+
/**
|
|
2296
|
+
* @returns {Promise<{ success: boolean, results: unknown }>}
|
|
2297
|
+
*/
|
|
1876
2298
|
async run() {
|
|
1877
2299
|
try {
|
|
1878
2300
|
const totalMemory = os2.totalmem();
|
|
@@ -1924,8 +2346,31 @@ var TaskSystemInfo = class extends TaskMaster {
|
|
|
1924
2346
|
}
|
|
1925
2347
|
};
|
|
1926
2348
|
|
|
1927
|
-
// src/tasks/coreTasks/TaskSumAB.
|
|
1928
|
-
var TaskSumAB = class extends
|
|
2349
|
+
// src/tasks/coreTasks/TaskSumAB.js
|
|
2350
|
+
var TaskSumAB = class extends AbstractTask {
|
|
2351
|
+
/** Short, deterministic — wait by default so callers see the sum. */
|
|
2352
|
+
static defaultWaitForResult = true;
|
|
2353
|
+
/**
|
|
2354
|
+
* @param {object} context
|
|
2355
|
+
* @param {Record<string, unknown>} [overrides]
|
|
2356
|
+
* @returns {Promise<{ a: number, b: number }>}
|
|
2357
|
+
*/
|
|
2358
|
+
static async resolveCustomParams(context, overrides = {}) {
|
|
2359
|
+
const merged = AbstractTask._mergeTypedParams(context, "task-sumab", {
|
|
2360
|
+
a: "number",
|
|
2361
|
+
b: "number"
|
|
2362
|
+
}, overrides);
|
|
2363
|
+
if (typeof merged.a !== "number" || Number.isNaN(merged.a)) {
|
|
2364
|
+
throw new ParamError(`taskSumAB: param "a" must be a valid number (got ${JSON.stringify(merged.a)})`);
|
|
2365
|
+
}
|
|
2366
|
+
if (typeof merged.b !== "number" || Number.isNaN(merged.b)) {
|
|
2367
|
+
throw new ParamError(`taskSumAB: param "b" must be a valid number (got ${JSON.stringify(merged.b)})`);
|
|
2368
|
+
}
|
|
2369
|
+
return { a: merged.a, b: merged.b };
|
|
2370
|
+
}
|
|
2371
|
+
/**
|
|
2372
|
+
* @returns {Promise<{ success: boolean, results: unknown }>}
|
|
2373
|
+
*/
|
|
1929
2374
|
async run() {
|
|
1930
2375
|
const a = this.task?.params?.a;
|
|
1931
2376
|
const b = this.task?.params?.b;
|
|
@@ -1956,8 +2401,46 @@ var TaskSumAB = class extends TaskMaster {
|
|
|
1956
2401
|
}
|
|
1957
2402
|
};
|
|
1958
2403
|
|
|
1959
|
-
// src/tasks/coreTasks/TaskStopRunner.
|
|
1960
|
-
var TaskStopRunner = class extends
|
|
2404
|
+
// src/tasks/coreTasks/TaskStopRunner.js
|
|
2405
|
+
var TaskStopRunner = class extends AbstractTask {
|
|
2406
|
+
/**
|
|
2407
|
+
* Stop tasks must target a concrete instance — without `serviceName` the
|
|
2408
|
+
* row would race against any worker on the queue. Layered on top of the
|
|
2409
|
+
* envelope built by {@link AbstractTask.resolveParams}.
|
|
2410
|
+
*
|
|
2411
|
+
* @param {object} context
|
|
2412
|
+
* @param {Record<string, unknown>} [overrides]
|
|
2413
|
+
* @returns {Promise<object>}
|
|
2414
|
+
*/
|
|
2415
|
+
static async resolveParams(context, overrides = {}) {
|
|
2416
|
+
const main = await super.resolveParams(context, overrides);
|
|
2417
|
+
if (!main.serviceName) {
|
|
2418
|
+
throw new ParamError(
|
|
2419
|
+
"stop/stopRunner requires --serviceName (registry instance name; optional --serviceGroup, --instanceNumber, --serverName to narrow targeting)"
|
|
2420
|
+
);
|
|
2421
|
+
}
|
|
2422
|
+
return main;
|
|
2423
|
+
}
|
|
2424
|
+
/**
|
|
2425
|
+
* @param {object} context
|
|
2426
|
+
* @param {Record<string, unknown>} [overrides]
|
|
2427
|
+
* @returns {Promise<{ allowanceMs: number }>}
|
|
2428
|
+
*/
|
|
2429
|
+
static async resolveCustomParams(context, overrides = {}) {
|
|
2430
|
+
const merged = AbstractTask._mergeTypedParams(context, "task-stop", {
|
|
2431
|
+
allowanceMs: "number default 5000"
|
|
2432
|
+
}, overrides);
|
|
2433
|
+
const allowanceMs = Number(merged.allowanceMs);
|
|
2434
|
+
if (!Number.isFinite(allowanceMs) || allowanceMs < 0) {
|
|
2435
|
+
throw new ParamError(
|
|
2436
|
+
`stopRunner: allowanceMs must be a non-negative number (got ${JSON.stringify(merged.allowanceMs)})`
|
|
2437
|
+
);
|
|
2438
|
+
}
|
|
2439
|
+
return { allowanceMs };
|
|
2440
|
+
}
|
|
2441
|
+
/**
|
|
2442
|
+
* @returns {Promise<{ success: true, results: { stopRunner: true, allowanceMs: number, message: string } }>}
|
|
2443
|
+
*/
|
|
1961
2444
|
async run() {
|
|
1962
2445
|
const allowanceMs = Number(this.task?.params?.allowanceMs ?? 5e3);
|
|
1963
2446
|
this.context.logger.warn?.(`[TaskStopRunner] stop requested (allowanceMs=${allowanceMs})`);
|
|
@@ -1972,172 +2455,396 @@ var TaskStopRunner = class extends TaskMaster {
|
|
|
1972
2455
|
}
|
|
1973
2456
|
};
|
|
1974
2457
|
|
|
1975
|
-
// src/tasks/
|
|
2458
|
+
// src/tasks/coreTasks/TaskGetLogs.js
|
|
2459
|
+
var TaskGetLogs = class extends AbstractTask {
|
|
2460
|
+
/**
|
|
2461
|
+
* @param {object} context
|
|
2462
|
+
* @param {Record<string, unknown>} [overrides]
|
|
2463
|
+
* @returns {Promise<{ source: string, resource: string, tail: number, afterTs?: string }>}
|
|
2464
|
+
*/
|
|
2465
|
+
static async resolveCustomParams(context, overrides = {}) {
|
|
2466
|
+
const merged = AbstractTask._mergeTypedParams(context, "task-get-logs", {
|
|
2467
|
+
source: "string",
|
|
2468
|
+
resource: "string",
|
|
2469
|
+
tail: "number default 100",
|
|
2470
|
+
afterTs: "string"
|
|
2471
|
+
}, overrides);
|
|
2472
|
+
const source = typeof merged.source === "string" ? merged.source.trim() : "";
|
|
2473
|
+
const resource = typeof merged.resource === "string" ? merged.resource.trim() : "";
|
|
2474
|
+
if (!source) throw new ParamError('getLogs: param "source" is required');
|
|
2475
|
+
if (!resource) throw new ParamError('getLogs: param "resource" is required');
|
|
2476
|
+
let tail = Number(merged.tail);
|
|
2477
|
+
if (!Number.isFinite(tail) || tail < 1) tail = 100;
|
|
2478
|
+
tail = Math.min(1e4, Math.max(1, Math.floor(tail)));
|
|
2479
|
+
const out = { source, resource, tail };
|
|
2480
|
+
if (typeof merged.afterTs === "string" && merged.afterTs.trim()) {
|
|
2481
|
+
out.afterTs = merged.afterTs.trim();
|
|
2482
|
+
}
|
|
2483
|
+
return out;
|
|
2484
|
+
}
|
|
2485
|
+
/**
|
|
2486
|
+
* @param {unknown} _reportProgress Unused (single-shot read, no progress events).
|
|
2487
|
+
* @returns {Promise<{ success: boolean, results: unknown }>}
|
|
2488
|
+
*/
|
|
2489
|
+
async run(_reportProgress) {
|
|
2490
|
+
const p = this.task.params ?? {};
|
|
2491
|
+
const source = String(p.source ?? "").trim();
|
|
2492
|
+
const resource = String(p.resource ?? "").trim();
|
|
2493
|
+
const tail = Math.max(1, Math.min(1e4, Number(p.tail) > 0 ? Number(p.tail) : 100));
|
|
2494
|
+
const afterTs = p.afterTs != null && String(p.afterTs).trim() ? String(p.afterTs).trim() : null;
|
|
2495
|
+
if (!source || !resource) {
|
|
2496
|
+
return {
|
|
2497
|
+
success: false,
|
|
2498
|
+
results: { error: 'getLogs requires params "source" and "resource"' }
|
|
2499
|
+
};
|
|
2500
|
+
}
|
|
2501
|
+
try {
|
|
2502
|
+
const { records, latestTs } = await readTaskIpcLogsSnapshot(this.context, {
|
|
2503
|
+
source,
|
|
2504
|
+
resource,
|
|
2505
|
+
tail,
|
|
2506
|
+
afterTs
|
|
2507
|
+
});
|
|
2508
|
+
return {
|
|
2509
|
+
success: true,
|
|
2510
|
+
results: { records, latestTs, source, resource }
|
|
2511
|
+
};
|
|
2512
|
+
} catch (e) {
|
|
2513
|
+
return {
|
|
2514
|
+
success: false,
|
|
2515
|
+
results: { error: e?.message ?? String(e) }
|
|
2516
|
+
};
|
|
2517
|
+
}
|
|
2518
|
+
}
|
|
2519
|
+
};
|
|
2520
|
+
|
|
2521
|
+
// src/tasks/TasksRegistry.js
|
|
1976
2522
|
var TasksRegistry = class _TasksRegistry {
|
|
1977
|
-
|
|
2523
|
+
/**
|
|
2524
|
+
* @param {Record<string, Function>} [initial] Optional seed entries to copy in.
|
|
2525
|
+
*/
|
|
1978
2526
|
constructor(initial) {
|
|
2527
|
+
this.map = {};
|
|
1979
2528
|
if (initial) {
|
|
1980
2529
|
this.addMany(initial);
|
|
1981
2530
|
}
|
|
1982
2531
|
}
|
|
2532
|
+
/**
|
|
2533
|
+
* Build a registry pre-populated with every core task plus legacy aliases.
|
|
2534
|
+
* Prefer this over `new TasksRegistry()` for anything that wants `ping`/`stop`/etc.
|
|
2535
|
+
*
|
|
2536
|
+
* @returns {TasksRegistry}
|
|
2537
|
+
*/
|
|
1983
2538
|
static withCoreTasks() {
|
|
1984
|
-
return new _TasksRegistry().add("ping", TaskPing).add("sampleProcess", TaskSampleProcess).add("shellCommand", TaskShellCommand).add("systemInfo", TaskSystemInfo).add("taskSumAB", TaskSumAB).add("stopRunner", TaskStopRunner).add("stop", TaskStopRunner);
|
|
2539
|
+
return new _TasksRegistry().add("ping", TaskPing).add("sampleProcess", TaskSampleProcess).add("shellCommand", TaskShellCommand).add("systemInfo", TaskSystemInfo).add("info", TaskSystemInfo).add("taskSumAB", TaskSumAB).add("stopRunner", TaskStopRunner).add("stop", TaskStopRunner).add("getLogs", TaskGetLogs);
|
|
1985
2540
|
}
|
|
2541
|
+
/**
|
|
2542
|
+
* Register a single task class under a name. Overwrites any previous entry.
|
|
2543
|
+
*
|
|
2544
|
+
* @param {string} taskName
|
|
2545
|
+
* @param {Function} taskClass Subclass of `AbstractTask`.
|
|
2546
|
+
* @returns {this}
|
|
2547
|
+
*/
|
|
1986
2548
|
add(taskName, taskClass) {
|
|
1987
2549
|
this.map[taskName] = taskClass;
|
|
1988
2550
|
return this;
|
|
1989
2551
|
}
|
|
2552
|
+
/**
|
|
2553
|
+
* Bulk-register a name → class map. Later calls override earlier ones.
|
|
2554
|
+
*
|
|
2555
|
+
* @param {Record<string, Function>} entries
|
|
2556
|
+
* @returns {this}
|
|
2557
|
+
*/
|
|
1990
2558
|
addMany(entries) {
|
|
1991
2559
|
for (const [name, klass] of Object.entries(entries)) {
|
|
1992
2560
|
this.add(name, klass);
|
|
1993
2561
|
}
|
|
1994
2562
|
return this;
|
|
1995
2563
|
}
|
|
2564
|
+
/**
|
|
2565
|
+
* Look up a task class by name. Returns `undefined` when the name is unknown;
|
|
2566
|
+
* the runner treats that as "some other worker may handle this" and skips.
|
|
2567
|
+
*
|
|
2568
|
+
* @param {string} taskName
|
|
2569
|
+
* @returns {Function | undefined}
|
|
2570
|
+
*/
|
|
1996
2571
|
get(taskName) {
|
|
1997
2572
|
return this.map[taskName];
|
|
1998
2573
|
}
|
|
2574
|
+
/**
|
|
2575
|
+
* Strict variant of {@link get}: throws {@link ParamError} (with the list
|
|
2576
|
+
* of supported names) when `taskName` is unknown. Use from enqueuer code
|
|
2577
|
+
* paths where an unknown name is a hard CLI/programmer error.
|
|
2578
|
+
*
|
|
2579
|
+
* @param {string} taskName
|
|
2580
|
+
* @returns {Function}
|
|
2581
|
+
*/
|
|
2582
|
+
requireClass(taskName) {
|
|
2583
|
+
const TaskClass = taskName ? this.map[taskName] : void 0;
|
|
2584
|
+
if (!TaskClass) {
|
|
2585
|
+
const supported = this.listSupportedTasks().join(", ") || "(none)";
|
|
2586
|
+
throw new ParamError(
|
|
2587
|
+
`Unknown task "${taskName ?? ""}". Supported on this registry: ${supported}`
|
|
2588
|
+
);
|
|
2589
|
+
}
|
|
2590
|
+
return TaskClass;
|
|
2591
|
+
}
|
|
2592
|
+
/**
|
|
2593
|
+
* Dispatcher used by `send-task` / programmatic enqueuers: pick `name`
|
|
2594
|
+
* from `overrides` or `context.params`, look up the class, and delegate
|
|
2595
|
+
* to its static {@link AbstractTask.resolveParams} with `name` seeded into
|
|
2596
|
+
* the overrides. The returned object is shaped for {@link enqueueTask}.
|
|
2597
|
+
*
|
|
2598
|
+
* Validation failures (unknown task, missing required custom params, etc.)
|
|
2599
|
+
* surface as {@link ParamError} so the caller aborts cleanly before any
|
|
2600
|
+
* row is inserted.
|
|
2601
|
+
*
|
|
2602
|
+
* @param {object} context
|
|
2603
|
+
* @param {Record<string, unknown>} [overrides]
|
|
2604
|
+
* @returns {Promise<object>}
|
|
2605
|
+
*/
|
|
2606
|
+
async resolveTaskParams(context, overrides = {}) {
|
|
2607
|
+
const overrideName = typeof overrides.name === "string" ? overrides.name.trim() : overrides.name;
|
|
2608
|
+
const fromCli = context.params.get("name", "string");
|
|
2609
|
+
const cliName = typeof fromCli === "string" ? fromCli.trim() : fromCli;
|
|
2610
|
+
const name = overrideName || cliName;
|
|
2611
|
+
if (!name) {
|
|
2612
|
+
throw new ParamError("Task --name is required (e.g. ping, stop, dummyHarvest)");
|
|
2613
|
+
}
|
|
2614
|
+
const TaskClass = this.requireClass(name);
|
|
2615
|
+
return TaskClass.resolveParams(context, { ...overrides, name });
|
|
2616
|
+
}
|
|
2617
|
+
/**
|
|
2618
|
+
* Names of every registered task, sorted alphabetically (useful for CLI output
|
|
2619
|
+
* and allowlist sanity checks).
|
|
2620
|
+
*
|
|
2621
|
+
* @returns {string[]}
|
|
2622
|
+
*/
|
|
1999
2623
|
listSupportedTasks() {
|
|
2000
2624
|
return Object.keys(this.map).sort();
|
|
2001
2625
|
}
|
|
2626
|
+
/**
|
|
2627
|
+
* Shallow copy of the internal map, for handing to `addMany` on another registry
|
|
2628
|
+
* or for serialization.
|
|
2629
|
+
*
|
|
2630
|
+
* @returns {Record<string, Function>}
|
|
2631
|
+
*/
|
|
2002
2632
|
toObject() {
|
|
2003
2633
|
return { ...this.map };
|
|
2004
2634
|
}
|
|
2005
2635
|
};
|
|
2006
2636
|
|
|
2007
|
-
// src/tasks/
|
|
2637
|
+
// src/tasks/serviceTaskAllowlist.js
|
|
2638
|
+
var SERVICE_TASK_NAMES = [
|
|
2639
|
+
"ping",
|
|
2640
|
+
"stop",
|
|
2641
|
+
"stopRunner",
|
|
2642
|
+
"shellCommand",
|
|
2643
|
+
"systemInfo",
|
|
2644
|
+
"info",
|
|
2645
|
+
"getLogs"
|
|
2646
|
+
];
|
|
2647
|
+
function normalizeAllowedTasks(value) {
|
|
2648
|
+
if (!value) return void 0;
|
|
2649
|
+
if (Array.isArray(value)) {
|
|
2650
|
+
const out2 = value.map((v) => String(v).trim()).filter(Boolean);
|
|
2651
|
+
return out2.length ? out2 : void 0;
|
|
2652
|
+
}
|
|
2653
|
+
const out = String(value).split(",").map((v) => v.trim()).filter(Boolean);
|
|
2654
|
+
return out.length ? out : void 0;
|
|
2655
|
+
}
|
|
2656
|
+
function mergeAllowedTasksWithServiceTasks(names) {
|
|
2657
|
+
const set = /* @__PURE__ */ new Set([...SERVICE_TASK_NAMES, ...names ?? []]);
|
|
2658
|
+
return Array.from(set).sort();
|
|
2659
|
+
}
|
|
2660
|
+
|
|
2661
|
+
// src/tasks/taskScriptRunner.js
|
|
2008
2662
|
import { spawn as spawn2 } from "child_process";
|
|
2663
|
+
var MAX_PROGRESS_TEXT_LEN = 4e3;
|
|
2009
2664
|
function toCliArgs(args = []) {
|
|
2010
2665
|
return args.filter((a) => typeof a === "string" && a.length > 0);
|
|
2011
2666
|
}
|
|
2012
2667
|
function formatChildLogPrefix(task) {
|
|
2013
|
-
return `${task.
|
|
2668
|
+
return `${task.name}:${task.id.slice(0, 8)}${task.opid ? `:${task.opid}` : ""}`;
|
|
2014
2669
|
}
|
|
2015
|
-
|
|
2016
|
-
|
|
2670
|
+
function isProgressPayload(payload) {
|
|
2671
|
+
if (!payload || typeof payload !== "object") return false;
|
|
2672
|
+
if (payload.level !== "progress") return false;
|
|
2673
|
+
const count = Number(payload.count);
|
|
2674
|
+
const total = Number(payload.total);
|
|
2675
|
+
return Number.isFinite(count) && Number.isFinite(total) && total > 0;
|
|
2676
|
+
}
|
|
2677
|
+
function formatProgressText(payload, fallbackPrefix) {
|
|
2678
|
+
const pfx = payload.prefix ? `${payload.prefix} ` : fallbackPrefix ? `${fallbackPrefix} ` : "";
|
|
2679
|
+
const label = typeof payload.message === "string" && payload.message ? `${payload.message} ` : "";
|
|
2680
|
+
return `${pfx}${label}${payload.count}/${payload.total}`;
|
|
2681
|
+
}
|
|
2682
|
+
function forwardChildLogToParent(context, prefix, message) {
|
|
2683
|
+
if (!message || typeof message !== "object") return;
|
|
2684
|
+
const text = typeof message.message === "string" ? message.message : null;
|
|
2685
|
+
if (!text) return;
|
|
2686
|
+
const level = typeof message.level === "string" ? message.level.toLowerCase() : "";
|
|
2687
|
+
const line = `[child:${prefix}] ${text}`;
|
|
2688
|
+
const logger = context.logger;
|
|
2689
|
+
switch (level) {
|
|
2690
|
+
case "error":
|
|
2691
|
+
case "fatal":
|
|
2692
|
+
logger.error?.(line);
|
|
2693
|
+
return;
|
|
2694
|
+
case "warn":
|
|
2695
|
+
case "warning":
|
|
2696
|
+
logger.warn?.(line);
|
|
2697
|
+
return;
|
|
2698
|
+
case "debug":
|
|
2699
|
+
logger.debug?.(line);
|
|
2700
|
+
return;
|
|
2701
|
+
case "info":
|
|
2702
|
+
default:
|
|
2703
|
+
logger.info?.(line);
|
|
2704
|
+
}
|
|
2705
|
+
}
|
|
2706
|
+
function buildNodeArgs(scriptPath, cliArgs) {
|
|
2017
2707
|
const inheritedExecArgs = Array.isArray(process.execArgv) ? [...process.execArgv] : [];
|
|
2018
2708
|
const hasTsRuntimeInParent = inheritedExecArgs.some((arg) => /tsx|ts-node/i.test(arg));
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
}
|
|
2032
|
-
}
|
|
2709
|
+
return hasTsRuntimeInParent ? [...inheritedExecArgs, scriptPath, ...cliArgs] : ["--import", "tsx", scriptPath, ...cliArgs];
|
|
2710
|
+
}
|
|
2711
|
+
function resolveTasksTableName(context) {
|
|
2712
|
+
return context.tasksQueueName || context.params?.get?.("table") || "tasks";
|
|
2713
|
+
}
|
|
2714
|
+
function announceIpcFileLogsTarget(context, options) {
|
|
2715
|
+
if (!options.ipcFileLogs) return;
|
|
2716
|
+
const logsDir = resolveIpcFileLogsDir(context, options.ipcFileLogs);
|
|
2717
|
+
const enabledRaw = context.params?.get?.("tasksLogsEnabled");
|
|
2718
|
+
const logsEnabled = enabledRaw === void 0 ? true : !!enabledRaw;
|
|
2719
|
+
context.logger.info?.(
|
|
2720
|
+
`[tasks] IPC file logs: ${logsDir}` + (logsEnabled ? "" : " (tasksLogsEnabled=false; not persisted)")
|
|
2033
2721
|
);
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
let
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
await db(tasksTable).where({ id: options.task.id }).update({ progress: text.slice(0, 4e3) });
|
|
2047
|
-
}).catch((error) => {
|
|
2048
|
-
context.logger.warn?.(
|
|
2049
|
-
`[tasks] failed to update progress for task=${options.task.id}: ${error?.message ?? String(error)}`
|
|
2050
|
-
);
|
|
2051
|
-
});
|
|
2052
|
-
if (options.onProgress) {
|
|
2053
|
-
progressCallbackChain = progressCallbackChain.then(async () => {
|
|
2054
|
-
await options.onProgress?.(text.slice(0, 4e3));
|
|
2055
|
-
}).catch((error) => {
|
|
2056
|
-
context.logger.warn?.(
|
|
2057
|
-
`[tasks] reportProgress callback failed for task=${options.task.id}: ${error?.message ?? String(error)}`
|
|
2058
|
-
);
|
|
2722
|
+
}
|
|
2723
|
+
function createSerializedQueue() {
|
|
2724
|
+
let chain = Promise.resolve();
|
|
2725
|
+
return {
|
|
2726
|
+
push(fn) {
|
|
2727
|
+
chain = chain.then(fn, () => {
|
|
2728
|
+
}).catch(() => {
|
|
2729
|
+
});
|
|
2730
|
+
return chain;
|
|
2731
|
+
},
|
|
2732
|
+
drain() {
|
|
2733
|
+
return chain.catch(() => {
|
|
2059
2734
|
});
|
|
2060
2735
|
}
|
|
2061
2736
|
};
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2737
|
+
}
|
|
2738
|
+
async function runNodeTaskScript(context, options) {
|
|
2739
|
+
const cliArgs = toCliArgs(["--route=ipc", "--mode=json", ...options.args || []]);
|
|
2740
|
+
const nodeArgs = buildNodeArgs(options.scriptPath, cliArgs);
|
|
2741
|
+
const child = spawn2(process.execPath, nodeArgs, {
|
|
2742
|
+
cwd: options.cwd || process.cwd(),
|
|
2743
|
+
stdio: ["ignore", "pipe", "pipe", "ipc"],
|
|
2744
|
+
env: {
|
|
2745
|
+
...process.env,
|
|
2746
|
+
TASK_ID: options.task.id,
|
|
2747
|
+
TASK_NAME: options.task.name,
|
|
2748
|
+
TASK_OPID: options.task.opid || ""
|
|
2749
|
+
}
|
|
2750
|
+
});
|
|
2751
|
+
announceIpcFileLogsTarget(context, options);
|
|
2752
|
+
const prefix = formatChildLogPrefix(options.task);
|
|
2753
|
+
const tasksTable = resolveTasksTableName(context);
|
|
2754
|
+
const progressQueue = createSerializedQueue();
|
|
2755
|
+
const state = {
|
|
2756
|
+
stdout: "",
|
|
2757
|
+
stderr: "",
|
|
2758
|
+
workerResult: null,
|
|
2759
|
+
hadErrorMessage: false
|
|
2075
2760
|
};
|
|
2076
|
-
|
|
2761
|
+
const writeProgress = (text) => {
|
|
2762
|
+
const trimmed = typeof text === "string" ? text.slice(0, MAX_PROGRESS_TEXT_LEN) : "";
|
|
2763
|
+
if (!trimmed) return;
|
|
2764
|
+
progressQueue.push(async () => {
|
|
2765
|
+
const db = context.db;
|
|
2766
|
+
if (db) {
|
|
2767
|
+
try {
|
|
2768
|
+
await db(tasksTable).where({ id: options.task.id }).update({ progress: trimmed });
|
|
2769
|
+
} catch (error) {
|
|
2770
|
+
context.logger.warn?.(
|
|
2771
|
+
`[tasks] failed to update progress for task=${options.task.id}: ${error?.message ?? String(error)}`
|
|
2772
|
+
);
|
|
2773
|
+
}
|
|
2774
|
+
}
|
|
2775
|
+
if (options.onProgress) {
|
|
2776
|
+
try {
|
|
2777
|
+
await options.onProgress(trimmed);
|
|
2778
|
+
} catch (error) {
|
|
2779
|
+
context.logger.warn?.(
|
|
2780
|
+
`[tasks] reportProgress callback failed for task=${options.task.id}: ${error?.message ?? String(error)}`
|
|
2781
|
+
);
|
|
2782
|
+
}
|
|
2783
|
+
}
|
|
2784
|
+
});
|
|
2785
|
+
};
|
|
2786
|
+
child.stdout?.on("data", (chunk) => {
|
|
2077
2787
|
const text = String(chunk);
|
|
2078
|
-
stdout += text;
|
|
2788
|
+
state.stdout += text;
|
|
2079
2789
|
if (text.trim()) {
|
|
2080
2790
|
context.logger.info?.(`[child:${prefix}] ${text.trimEnd()}`);
|
|
2081
|
-
updateProgress(text.trim().replace(/\s+/g, " ").slice(0, 400));
|
|
2082
2791
|
}
|
|
2083
2792
|
});
|
|
2084
|
-
child.stderr
|
|
2793
|
+
child.stderr?.on("data", (chunk) => {
|
|
2085
2794
|
const text = String(chunk);
|
|
2086
|
-
stderr += text;
|
|
2795
|
+
state.stderr += text;
|
|
2087
2796
|
if (text.trim()) {
|
|
2088
2797
|
context.logger.warn?.(`[child:${prefix}] ${text.trimEnd()}`);
|
|
2089
2798
|
}
|
|
2090
2799
|
});
|
|
2091
2800
|
child.on("message", (message) => {
|
|
2092
2801
|
if (message && typeof message === "object" && "__taskWorkerResult" in message) {
|
|
2093
|
-
workerResult = message.__taskWorkerResult;
|
|
2802
|
+
state.workerResult = message.__taskWorkerResult;
|
|
2094
2803
|
return;
|
|
2095
2804
|
}
|
|
2096
2805
|
if (message && typeof message === "object") {
|
|
2097
2806
|
const level = typeof message.level === "string" ? message.level.toLowerCase() : "";
|
|
2098
2807
|
if (level === "error" || level === "fatal") {
|
|
2099
|
-
hadErrorMessage = true;
|
|
2808
|
+
state.hadErrorMessage = true;
|
|
2100
2809
|
}
|
|
2101
2810
|
}
|
|
2102
|
-
appendTaskIpcLog(context, options.task, message);
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2106
|
-
|
|
2107
|
-
const countNum = Number(String(message.count).trim());
|
|
2108
|
-
const totalNum = Number(message.total);
|
|
2109
|
-
if (Number.isFinite(countNum) && Number.isFinite(totalNum) && totalNum > 0) {
|
|
2110
|
-
context.logger.progress(message.message || "progress", {
|
|
2111
|
-
prefix: message.prefix || prefix,
|
|
2112
|
-
count: countNum,
|
|
2113
|
-
total: totalNum
|
|
2114
|
-
});
|
|
2115
|
-
} else {
|
|
2116
|
-
context.logger.info?.(`[child:${prefix}] ${progressText}`);
|
|
2117
|
-
}
|
|
2118
|
-
} else {
|
|
2119
|
-
context.logger.info?.(`[child:${prefix}] ${progressText}`);
|
|
2120
|
-
}
|
|
2811
|
+
appendTaskIpcLog(context, options.task, message, options.ipcFileLogs);
|
|
2812
|
+
try {
|
|
2813
|
+
options.onChildIpcMessage?.(message);
|
|
2814
|
+
} catch (e) {
|
|
2815
|
+
context.logger.warn?.(`[tasks] onChildIpcMessage failed: ${e?.message ?? String(e)}`);
|
|
2121
2816
|
}
|
|
2817
|
+
if (isProgressPayload(message)) {
|
|
2818
|
+
context.logger.progress(message.message || "progress", {
|
|
2819
|
+
prefix: message.prefix || prefix,
|
|
2820
|
+
count: Number(message.count),
|
|
2821
|
+
total: Number(message.total)
|
|
2822
|
+
});
|
|
2823
|
+
writeProgress(formatProgressText(message, prefix));
|
|
2824
|
+
return;
|
|
2825
|
+
}
|
|
2826
|
+
forwardChildLogToParent(context, prefix, message);
|
|
2122
2827
|
});
|
|
2123
2828
|
return await new Promise((resolve, reject) => {
|
|
2124
2829
|
child.on("error", (error) => reject(error));
|
|
2125
2830
|
child.on("close", (exitCode, signal) => {
|
|
2126
|
-
|
|
2831
|
+
void (async () => {
|
|
2832
|
+
await flushTaskIpcLogs(context);
|
|
2833
|
+
await progressQueue.drain();
|
|
2127
2834
|
resolve({
|
|
2128
2835
|
exitCode,
|
|
2129
2836
|
signal,
|
|
2130
|
-
stdout: stdout.trim(),
|
|
2131
|
-
stderr: stderr.trim(),
|
|
2132
|
-
workerResult,
|
|
2133
|
-
hadErrorMessage
|
|
2837
|
+
stdout: state.stdout.trim(),
|
|
2838
|
+
stderr: state.stderr.trim(),
|
|
2839
|
+
workerResult: state.workerResult,
|
|
2840
|
+
hadErrorMessage: state.hadErrorMessage
|
|
2134
2841
|
});
|
|
2135
|
-
});
|
|
2842
|
+
})();
|
|
2136
2843
|
});
|
|
2137
2844
|
});
|
|
2138
2845
|
}
|
|
2139
2846
|
|
|
2140
|
-
// src/tasks/index.
|
|
2847
|
+
// src/tasks/index.js
|
|
2141
2848
|
var LOCKED_BY_ERROR_MESSAGE = "locked by error";
|
|
2142
2849
|
var defaultTasksRegistry = TasksRegistry.withCoreTasks();
|
|
2143
2850
|
function getDb3(context) {
|
|
@@ -2152,22 +2859,13 @@ function normalizeRegistry(registry) {
|
|
|
2152
2859
|
if (registry instanceof TasksRegistry) return registry;
|
|
2153
2860
|
return new TasksRegistry().addMany(registry);
|
|
2154
2861
|
}
|
|
2155
|
-
function
|
|
2156
|
-
if (!value) return void 0;
|
|
2157
|
-
if (Array.isArray(value)) {
|
|
2158
|
-
const out2 = value.map((v) => String(v).trim()).filter(Boolean);
|
|
2159
|
-
return out2.length ? out2 : void 0;
|
|
2160
|
-
}
|
|
2161
|
-
const out = String(value).split(",").map((v) => v.trim()).filter(Boolean);
|
|
2162
|
-
return out.length ? out : void 0;
|
|
2163
|
-
}
|
|
2164
|
-
async function enqueueStopTask(context, target, queue = "tasks", allowanceMs = 5e3) {
|
|
2862
|
+
async function enqueueStopTask(context, serviceGroup, queueName = "tasks", allowanceMs = 5e3) {
|
|
2165
2863
|
return enqueueTask(context, {
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
task: "stopRunner",
|
|
2864
|
+
queueName,
|
|
2865
|
+
name: "stopRunner",
|
|
2169
2866
|
params: { allowanceMs },
|
|
2170
|
-
priority:
|
|
2867
|
+
priority: 0,
|
|
2868
|
+
serviceGroup
|
|
2171
2869
|
});
|
|
2172
2870
|
}
|
|
2173
2871
|
async function signalRunningTasksStop(context, runningTaskInstances, allowanceMs) {
|
|
@@ -2185,18 +2883,20 @@ async function signalRunningTasksStop(context, runningTaskInstances, allowanceMs
|
|
|
2185
2883
|
}
|
|
2186
2884
|
async function executeClaimedTask(context, tasksTable, historyTable, row, registry, runningTaskInstances) {
|
|
2187
2885
|
const db = getDb3(context);
|
|
2188
|
-
const taskName = row.
|
|
2886
|
+
const taskName = row.name;
|
|
2189
2887
|
const TaskClass = registry.get(taskName);
|
|
2190
|
-
const { paused_at: _pausedAt, ...rowForHistory } = row;
|
|
2191
2888
|
if (!TaskClass) {
|
|
2192
2889
|
const err = { message: `Unknown task "${taskName}"` };
|
|
2193
|
-
await db(historyTable).insert(
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2890
|
+
await db(historyTable).insert(
|
|
2891
|
+
taskHistoryInsertFromQueueRow(row, {
|
|
2892
|
+
completed_at: /* @__PURE__ */ new Date(),
|
|
2893
|
+
success: false,
|
|
2894
|
+
status: "failed",
|
|
2895
|
+
status_changed_at: db.fn.now(),
|
|
2896
|
+
params: toJsonColumn(row.params),
|
|
2897
|
+
results: toJsonColumn(err)
|
|
2898
|
+
})
|
|
2899
|
+
);
|
|
2200
2900
|
if (row.schedule) {
|
|
2201
2901
|
await db(tasksTable).where({ id: row.id }).update({
|
|
2202
2902
|
started_at: null,
|
|
@@ -2204,7 +2904,8 @@ async function executeClaimedTask(context, tasksTable, historyTable, row, regist
|
|
|
2204
2904
|
success: false,
|
|
2205
2905
|
results: toJsonColumn(err),
|
|
2206
2906
|
past_due: null,
|
|
2207
|
-
|
|
2907
|
+
status: "paused",
|
|
2908
|
+
status_changed_at: db.fn.now(),
|
|
2208
2909
|
progress: LOCKED_BY_ERROR_MESSAGE
|
|
2209
2910
|
});
|
|
2210
2911
|
} else {
|
|
@@ -2231,13 +2932,16 @@ async function executeClaimedTask(context, tasksTable, historyTable, row, regist
|
|
|
2231
2932
|
} finally {
|
|
2232
2933
|
runningTaskInstances.delete(row.id);
|
|
2233
2934
|
}
|
|
2234
|
-
await db(historyTable).insert(
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2935
|
+
await db(historyTable).insert(
|
|
2936
|
+
taskHistoryInsertFromQueueRow(row, {
|
|
2937
|
+
completed_at: /* @__PURE__ */ new Date(),
|
|
2938
|
+
success,
|
|
2939
|
+
status: success ? "completed" : "failed",
|
|
2940
|
+
status_changed_at: db.fn.now(),
|
|
2941
|
+
params: toJsonColumn(row.params),
|
|
2942
|
+
results: toJsonColumn(results)
|
|
2943
|
+
})
|
|
2944
|
+
);
|
|
2241
2945
|
if (!success) {
|
|
2242
2946
|
const dbName = String(context?.params?.get?.("dbName") || "local");
|
|
2243
2947
|
const tableName = String(context?.params?.get?.("table") || "tasks");
|
|
@@ -2252,19 +2956,32 @@ async function executeClaimedTask(context, tasksTable, historyTable, row, regist
|
|
|
2252
2956
|
const rerunCommand = results && typeof results === "object" && results.rerunCommand ? results.rerunCommand : fallbackRecoverCommand;
|
|
2253
2957
|
appendTaskIpcLog(context, row, {
|
|
2254
2958
|
level: "error",
|
|
2255
|
-
message: `[tasks] task failed: ${row.
|
|
2959
|
+
message: `[tasks] task failed: ${row.name} id=${row.id}. Once problem is fixed, re-run: ${rerunCommand}`,
|
|
2256
2960
|
details: results
|
|
2257
2961
|
});
|
|
2258
2962
|
}
|
|
2259
2963
|
if (row.schedule) {
|
|
2260
2964
|
if (success) {
|
|
2965
|
+
let nextRunAt = null;
|
|
2966
|
+
try {
|
|
2967
|
+
nextRunAt = nextTimeMatch(row.schedule, /* @__PURE__ */ new Date());
|
|
2968
|
+
} catch (e) {
|
|
2969
|
+
context.logger?.warn?.(`[tasks] nextTimeMatch after success for task ${row.id}: ${e?.message ?? String(e)}`);
|
|
2970
|
+
}
|
|
2261
2971
|
await db(tasksTable).where({ id: row.id }).update({
|
|
2262
2972
|
started_at: null,
|
|
2263
2973
|
completed_at: /* @__PURE__ */ new Date(),
|
|
2264
2974
|
success,
|
|
2265
2975
|
results: toJsonColumn(results),
|
|
2266
2976
|
progress: null,
|
|
2267
|
-
past_due: null
|
|
2977
|
+
past_due: null,
|
|
2978
|
+
status: "idle",
|
|
2979
|
+
status_changed_at: db.fn.now(),
|
|
2980
|
+
next_run_at: nextRunAt,
|
|
2981
|
+
// Claim overwrites these; clear so idle rows stay “any worker” (see claimNextRunnableTask).
|
|
2982
|
+
service_name: null,
|
|
2983
|
+
server_name: null,
|
|
2984
|
+
instance_number: null
|
|
2268
2985
|
});
|
|
2269
2986
|
} else {
|
|
2270
2987
|
await db(tasksTable).where({ id: row.id }).update({
|
|
@@ -2272,7 +2989,8 @@ async function executeClaimedTask(context, tasksTable, historyTable, row, regist
|
|
|
2272
2989
|
completed_at: /* @__PURE__ */ new Date(),
|
|
2273
2990
|
success,
|
|
2274
2991
|
results: toJsonColumn(results),
|
|
2275
|
-
|
|
2992
|
+
status: "paused",
|
|
2993
|
+
status_changed_at: db.fn.now(),
|
|
2276
2994
|
progress: LOCKED_BY_ERROR_MESSAGE,
|
|
2277
2995
|
past_due: null
|
|
2278
2996
|
});
|
|
@@ -2284,58 +3002,93 @@ async function executeClaimedTask(context, tasksTable, historyTable, row, regist
|
|
|
2284
3002
|
const stopAllowanceMs = stopRunnerRequested ? Number(results.allowanceMs ?? 5e3) : 0;
|
|
2285
3003
|
return { stopRunnerRequested, stopAllowanceMs };
|
|
2286
3004
|
}
|
|
2287
|
-
|
|
3005
|
+
function shuffleTaskRowsInPlace(rows) {
|
|
3006
|
+
for (let i = rows.length - 1; i > 0; i--) {
|
|
3007
|
+
const j = Math.floor(Math.random() * (i + 1));
|
|
3008
|
+
const t = rows[i];
|
|
3009
|
+
rows[i] = rows[j];
|
|
3010
|
+
rows[j] = t;
|
|
3011
|
+
}
|
|
3012
|
+
}
|
|
3013
|
+
async function claimNextRunnableTask(context, tasksTable, serviceGroup, registry, scanLimit, taskNames, runnerIdentity) {
|
|
2288
3014
|
const db = getDb3(context);
|
|
2289
|
-
let query = db(tasksTable).
|
|
3015
|
+
let query = db(tasksTable).where({ status: "idle" }).where(function() {
|
|
3016
|
+
this.whereNull("service_group").orWhere({ service_group: serviceGroup });
|
|
3017
|
+
}).orderByRaw("CASE WHEN past_due IS NULL THEN 1 ELSE 0 END ASC").orderBy([{ column: "priority", order: "asc" }]).orderByRaw("CASE WHEN completed_at IS NULL THEN 0 ELSE 1 END ASC").orderBy([{ column: "completed_at", order: "asc" }, { column: "created_at", order: "asc" }]).limit(scanLimit);
|
|
2290
3018
|
if (taskNames && taskNames.length > 0) {
|
|
2291
|
-
query = query.whereIn("
|
|
3019
|
+
query = query.whereIn("name", taskNames);
|
|
3020
|
+
}
|
|
3021
|
+
if (runnerIdentity) {
|
|
3022
|
+
query = query.where(function() {
|
|
3023
|
+
this.whereNull("service_name").orWhere({ service_name: runnerIdentity.service_name });
|
|
3024
|
+
}).where(function() {
|
|
3025
|
+
this.whereNull("instance_number").orWhere({ instance_number: runnerIdentity.instance_number });
|
|
3026
|
+
}).where(function() {
|
|
3027
|
+
this.whereNull("server_name").orWhere({ server_name: runnerIdentity.server_name });
|
|
3028
|
+
});
|
|
3029
|
+
} else {
|
|
3030
|
+
query = query.whereNull("service_name").whereNull("instance_number").whereNull("server_name");
|
|
2292
3031
|
}
|
|
2293
3032
|
const candidates = await query;
|
|
3033
|
+
shuffleTaskRowsInPlace(candidates);
|
|
2294
3034
|
for (const row of candidates) {
|
|
2295
3035
|
if (!row.past_due && row.schedule && !timeMatcher(row.schedule)) {
|
|
2296
3036
|
continue;
|
|
2297
3037
|
}
|
|
2298
|
-
const TaskClass = registry.get(row.
|
|
2299
|
-
if (TaskClass) {
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
3038
|
+
const TaskClass = registry.get(row.name);
|
|
3039
|
+
if (!TaskClass) {
|
|
3040
|
+
continue;
|
|
3041
|
+
}
|
|
3042
|
+
const taskInstance = new TaskClass(context, row);
|
|
3043
|
+
const reason = taskInstance.cantRunReason ? await taskInstance.cantRunReason() : null;
|
|
3044
|
+
if (reason) {
|
|
3045
|
+
if (!row.past_due) {
|
|
3046
|
+
await db(tasksTable).where({ id: row.id }).update({
|
|
3047
|
+
past_due: db.fn.now(),
|
|
3048
|
+
progress: String(reason)
|
|
3049
|
+
});
|
|
2310
3050
|
}
|
|
3051
|
+
continue;
|
|
3052
|
+
}
|
|
3053
|
+
const claimPatch = {
|
|
3054
|
+
started_at: db.fn.now(),
|
|
3055
|
+
status: "running",
|
|
3056
|
+
status_changed_at: db.fn.now()
|
|
3057
|
+
};
|
|
3058
|
+
if (runnerIdentity) {
|
|
3059
|
+
claimPatch.service_name = runnerIdentity.service_name;
|
|
3060
|
+
claimPatch.server_name = runnerIdentity.server_name;
|
|
3061
|
+
claimPatch.instance_number = runnerIdentity.instance_number;
|
|
2311
3062
|
}
|
|
2312
|
-
const updated = await db(tasksTable).where({ id: row.id
|
|
3063
|
+
const updated = await db(tasksTable).where({ id: row.id, status: "idle" }).update(claimPatch).returning("*");
|
|
2313
3064
|
const claimed = Array.isArray(updated) ? updated[0] : null;
|
|
2314
3065
|
if (claimed) return claimed;
|
|
2315
3066
|
}
|
|
2316
3067
|
return null;
|
|
2317
3068
|
}
|
|
2318
3069
|
async function runTasksLoop(context, options) {
|
|
2319
|
-
const
|
|
3070
|
+
const queueName = options.queueName ?? "tasks";
|
|
2320
3071
|
const target = options.target;
|
|
2321
3072
|
const pollMs = options.pollMs ?? 1e3;
|
|
2322
|
-
const
|
|
3073
|
+
const claimJitterMs = options.claimJitterMs ?? 0;
|
|
3074
|
+
const maxParallel = options.maxParallel ?? 32;
|
|
2323
3075
|
const scanLimit = options.scanLimit ?? 100;
|
|
2324
3076
|
const allowedTasks = normalizeAllowedTasks(options.allowedTasks);
|
|
2325
3077
|
const registry = normalizeRegistry(options.registry);
|
|
2326
|
-
const { tasksTable, historyTable } = queueToTableNames(
|
|
3078
|
+
const { tasksTable, historyTable } = queueToTableNames(queueName);
|
|
2327
3079
|
if (!target) throw new Error("runTasksLoop: target is required");
|
|
3080
|
+
context.tasksQueueName = queueName;
|
|
2328
3081
|
const runningPromises = /* @__PURE__ */ new Set();
|
|
2329
3082
|
const runningTaskInstances = /* @__PURE__ */ new Map();
|
|
2330
3083
|
let runningStopControlPromise = null;
|
|
2331
3084
|
let stopRequested = false;
|
|
2332
3085
|
let stopAllowanceMs = 5e3;
|
|
2333
|
-
context.
|
|
3086
|
+
context.tasksRunnerStop = false;
|
|
2334
3087
|
let registryReg = null;
|
|
2335
3088
|
let registryInterval = null;
|
|
3089
|
+
let runnerIdentity = null;
|
|
2336
3090
|
const hbGroup = options.runnerServiceGroup?.trim();
|
|
2337
3091
|
if (hbGroup) {
|
|
2338
|
-
const identityDir = options.runnerIdentityDir ?? "./data/runner-identities";
|
|
2339
3092
|
const hbIntervalMs = options.runnerHeartbeatIntervalMs ?? 1e4;
|
|
2340
3093
|
const staleMs = options.runnerHeartbeatStaleMs ?? 45e3;
|
|
2341
3094
|
const defaultMeta = {
|
|
@@ -2343,16 +3096,21 @@ async function runTasksLoop(context, options) {
|
|
|
2343
3096
|
allowedTasks: allowedTasks?.length ? allowedTasks.join(",") : "all"
|
|
2344
3097
|
};
|
|
2345
3098
|
registryReg = await registerInServicesRegistry(context, {
|
|
2346
|
-
|
|
3099
|
+
queueName,
|
|
2347
3100
|
target,
|
|
2348
3101
|
serviceGroup: hbGroup,
|
|
2349
3102
|
serviceName: options.runnerServiceName,
|
|
2350
|
-
|
|
3103
|
+
instanceNumber: options.runnerInstanceNumber,
|
|
2351
3104
|
staleMs,
|
|
2352
3105
|
groupMaxInstances: options.runnerGroupMaxInstances,
|
|
2353
3106
|
enforceMaxInstances: options.runnerEnforceMaxInstances ?? true,
|
|
2354
3107
|
metadata: options.runnerMetadata ?? defaultMeta
|
|
2355
3108
|
});
|
|
3109
|
+
runnerIdentity = {
|
|
3110
|
+
service_name: registryReg.serviceName,
|
|
3111
|
+
server_name: os3.hostname(),
|
|
3112
|
+
instance_number: registryReg.instanceNumber
|
|
3113
|
+
};
|
|
2356
3114
|
registryInterval = setInterval(() => {
|
|
2357
3115
|
void touchServicesRegistry(context, registryReg).catch((err) => {
|
|
2358
3116
|
context.logger.warn?.(`[services-registry] touch failed: ${err?.message ?? String(err)}`);
|
|
@@ -2360,7 +3118,7 @@ async function runTasksLoop(context, options) {
|
|
|
2360
3118
|
}, hbIntervalMs);
|
|
2361
3119
|
}
|
|
2362
3120
|
try {
|
|
2363
|
-
while (!context.isStop() && !stopRequested &&
|
|
3121
|
+
while (!context.isStop() && !stopRequested && context.tasksRunnerStop !== true) {
|
|
2364
3122
|
if (!runningStopControlPromise) {
|
|
2365
3123
|
const claimedStopTask = await claimNextRunnableTask(
|
|
2366
3124
|
context,
|
|
@@ -2368,7 +3126,8 @@ async function runTasksLoop(context, options) {
|
|
|
2368
3126
|
target,
|
|
2369
3127
|
registry,
|
|
2370
3128
|
10,
|
|
2371
|
-
["stopRunner", "stop"]
|
|
3129
|
+
["stopRunner", "stop"],
|
|
3130
|
+
runnerIdentity
|
|
2372
3131
|
);
|
|
2373
3132
|
if (claimedStopTask) {
|
|
2374
3133
|
runningStopControlPromise = executeClaimedTask(
|
|
@@ -2382,7 +3141,7 @@ async function runTasksLoop(context, options) {
|
|
|
2382
3141
|
if (outcome.stopRunnerRequested && !stopRequested) {
|
|
2383
3142
|
stopRequested = true;
|
|
2384
3143
|
stopAllowanceMs = outcome.stopAllowanceMs || 5e3;
|
|
2385
|
-
context.
|
|
3144
|
+
context.tasksRunnerStop = true;
|
|
2386
3145
|
await signalRunningTasksStop(context, runningTaskInstances, stopAllowanceMs);
|
|
2387
3146
|
}
|
|
2388
3147
|
}).finally(() => {
|
|
@@ -2390,6 +3149,9 @@ async function runTasksLoop(context, options) {
|
|
|
2390
3149
|
});
|
|
2391
3150
|
}
|
|
2392
3151
|
}
|
|
3152
|
+
if (claimJitterMs > 0) {
|
|
3153
|
+
await sleepMs(Math.floor(Math.random() * (claimJitterMs + 1)));
|
|
3154
|
+
}
|
|
2393
3155
|
while (runningPromises.size < maxParallel) {
|
|
2394
3156
|
const claimed = await claimNextRunnableTask(
|
|
2395
3157
|
context,
|
|
@@ -2397,14 +3159,15 @@ async function runTasksLoop(context, options) {
|
|
|
2397
3159
|
target,
|
|
2398
3160
|
registry,
|
|
2399
3161
|
scanLimit,
|
|
2400
|
-
allowedTasks
|
|
3162
|
+
allowedTasks,
|
|
3163
|
+
runnerIdentity
|
|
2401
3164
|
);
|
|
2402
3165
|
if (!claimed) break;
|
|
2403
3166
|
const p = executeClaimedTask(context, tasksTable, historyTable, claimed, registry, runningTaskInstances).then(async (outcome) => {
|
|
2404
3167
|
if (outcome.stopRunnerRequested && !stopRequested) {
|
|
2405
3168
|
stopRequested = true;
|
|
2406
3169
|
stopAllowanceMs = outcome.stopAllowanceMs || 5e3;
|
|
2407
|
-
context.
|
|
3170
|
+
context.tasksRunnerStop = true;
|
|
2408
3171
|
await signalRunningTasksStop(context, runningTaskInstances, stopAllowanceMs);
|
|
2409
3172
|
}
|
|
2410
3173
|
}).finally(() => {
|
|
@@ -2412,7 +3175,16 @@ async function runTasksLoop(context, options) {
|
|
|
2412
3175
|
});
|
|
2413
3176
|
runningPromises.add(p);
|
|
2414
3177
|
}
|
|
2415
|
-
|
|
3178
|
+
const wakePromises = [...runningPromises];
|
|
3179
|
+
if (runningStopControlPromise) {
|
|
3180
|
+
wakePromises.push(runningStopControlPromise);
|
|
3181
|
+
}
|
|
3182
|
+
if (wakePromises.length === 0) {
|
|
3183
|
+
await sleepMs(pollMs);
|
|
3184
|
+
} else {
|
|
3185
|
+
const safe = wakePromises.map((p) => p.catch(() => void 0));
|
|
3186
|
+
await Promise.race([sleepMs(pollMs), Promise.race(safe)]);
|
|
3187
|
+
}
|
|
2416
3188
|
}
|
|
2417
3189
|
if (context.isStop() && !stopRequested) {
|
|
2418
3190
|
await signalRunningTasksStop(context, runningTaskInstances, 5e3);
|
|
@@ -2448,89 +3220,130 @@ async function runTasksLoop(context, options) {
|
|
|
2448
3220
|
}
|
|
2449
3221
|
async function waitForTaskResult(context, taskId, options = {}) {
|
|
2450
3222
|
const db = getDb3(context);
|
|
2451
|
-
const
|
|
3223
|
+
const queueName = options.queueName ?? "tasks";
|
|
2452
3224
|
const timeoutMs = options.timeoutMs ?? 6e4;
|
|
2453
3225
|
const pollMs = options.pollMs ?? 500;
|
|
2454
|
-
const { tasksTable, historyTable } = queueToTableNames(
|
|
3226
|
+
const { tasksTable, historyTable } = queueToTableNames(queueName);
|
|
2455
3227
|
const deadline = Date.now() + timeoutMs;
|
|
3228
|
+
const waitStartedAt = /* @__PURE__ */ new Date();
|
|
3229
|
+
let cachedNameOpid = null;
|
|
3230
|
+
async function historySinceWait(name, opid) {
|
|
3231
|
+
let q = db(historyTable).where({ name }).where("completed_at", ">=", waitStartedAt);
|
|
3232
|
+
if (opid == null || opid === "") {
|
|
3233
|
+
q = q.whereNull("opid");
|
|
3234
|
+
} else {
|
|
3235
|
+
q = q.where({ opid });
|
|
3236
|
+
}
|
|
3237
|
+
return await q.orderBy("completed_at", "desc").first();
|
|
3238
|
+
}
|
|
2456
3239
|
while (Date.now() <= deadline) {
|
|
2457
|
-
const
|
|
2458
|
-
if (
|
|
3240
|
+
const legacy = await db(historyTable).where({ id: taskId }).orderBy("created_at", "desc").first();
|
|
3241
|
+
if (legacy) {
|
|
3242
|
+
return legacy;
|
|
3243
|
+
}
|
|
2459
3244
|
const pending = await db(tasksTable).where({ id: taskId }).first();
|
|
2460
|
-
if (
|
|
2461
|
-
|
|
2462
|
-
|
|
3245
|
+
if (pending) {
|
|
3246
|
+
cachedNameOpid = { name: pending.name, opid: pending.opid };
|
|
3247
|
+
const done = await historySinceWait(pending.name, pending.opid);
|
|
3248
|
+
if (done) {
|
|
3249
|
+
return done;
|
|
3250
|
+
}
|
|
3251
|
+
} else if (cachedNameOpid) {
|
|
3252
|
+
const done = await historySinceWait(cachedNameOpid.name, cachedNameOpid.opid);
|
|
3253
|
+
if (done) {
|
|
3254
|
+
return done;
|
|
3255
|
+
}
|
|
3256
|
+
return null;
|
|
3257
|
+
} else {
|
|
3258
|
+
return null;
|
|
2463
3259
|
}
|
|
2464
3260
|
await sleepMs(pollMs);
|
|
2465
3261
|
}
|
|
2466
3262
|
return null;
|
|
2467
3263
|
}
|
|
2468
3264
|
var TasksManager = class _TasksManager {
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
|
|
2476
|
-
|
|
2477
|
-
|
|
2478
|
-
|
|
2479
|
-
|
|
2480
|
-
|
|
2481
|
-
|
|
2482
|
-
|
|
2483
|
-
|
|
2484
|
-
|
|
2485
|
-
|
|
3265
|
+
/**
|
|
3266
|
+
* @param {object} context
|
|
3267
|
+
* @param {{
|
|
3268
|
+
* queueName?: string,
|
|
3269
|
+
* target?: string,
|
|
3270
|
+
* recreateTaskTables?: boolean,
|
|
3271
|
+
* pollMs?: number,
|
|
3272
|
+
* claimJitterMs?: number,
|
|
3273
|
+
* maxParallel?: number,
|
|
3274
|
+
* scanLimit?: number,
|
|
3275
|
+
* allowedTasks?: string | string[],
|
|
3276
|
+
* registry?: TasksRegistry | Record<string, Function>,
|
|
3277
|
+
* runnerServiceGroup?: string,
|
|
3278
|
+
* runnerServiceName?: string,
|
|
3279
|
+
* runnerInstanceNumber?: number,
|
|
3280
|
+
* runnerHeartbeatIntervalMs?: number,
|
|
3281
|
+
* runnerHeartbeatStaleMs?: number,
|
|
3282
|
+
* runnerGroupMaxInstances?: number,
|
|
3283
|
+
* runnerEnforceMaxInstances?: boolean,
|
|
3284
|
+
* runnerMetadata?: Record<string, unknown>,
|
|
3285
|
+
* }} [options]
|
|
3286
|
+
*/
|
|
2486
3287
|
constructor(context, options = {}) {
|
|
2487
3288
|
this.context = context;
|
|
2488
|
-
this.
|
|
3289
|
+
this.queueName = options.queueName ?? "tasks";
|
|
2489
3290
|
this.target = options.target ?? "localRunner";
|
|
2490
3291
|
this.recreateTaskTables = options.recreateTaskTables ?? false;
|
|
2491
3292
|
this.pollMs = options.pollMs ?? 1e3;
|
|
3293
|
+
this.claimJitterMs = options.claimJitterMs ?? 0;
|
|
2492
3294
|
this.maxParallel = options.maxParallel ?? 1;
|
|
2493
3295
|
this.scanLimit = options.scanLimit ?? 100;
|
|
2494
3296
|
this.allowedTasks = normalizeAllowedTasks(options.allowedTasks);
|
|
2495
3297
|
this.registry = normalizeRegistry(options.registry);
|
|
2496
3298
|
this.runnerServiceGroup = options.runnerServiceGroup;
|
|
2497
3299
|
this.runnerServiceName = options.runnerServiceName;
|
|
2498
|
-
this.
|
|
3300
|
+
this.runnerInstanceNumber = options.runnerInstanceNumber;
|
|
2499
3301
|
this.runnerHeartbeatIntervalMs = options.runnerHeartbeatIntervalMs;
|
|
2500
3302
|
this.runnerHeartbeatStaleMs = options.runnerHeartbeatStaleMs;
|
|
2501
3303
|
this.runnerGroupMaxInstances = options.runnerGroupMaxInstances;
|
|
2502
3304
|
this.runnerEnforceMaxInstances = options.runnerEnforceMaxInstances;
|
|
2503
3305
|
this.runnerMetadata = options.runnerMetadata;
|
|
2504
3306
|
}
|
|
3307
|
+
/**
|
|
3308
|
+
* Preferred factory: reads defaults from `context.params` (module namespace
|
|
3309
|
+
* `"tasks"`), then overlays explicit `options`. Keeps CLI flags, env vars,
|
|
3310
|
+
* and inline options in one consistent resolver.
|
|
3311
|
+
*
|
|
3312
|
+
* @param {object} context
|
|
3313
|
+
* @param {ConstructorParameters<typeof TasksManager>[1]} [options]
|
|
3314
|
+
* @returns {TasksManager}
|
|
3315
|
+
*/
|
|
2505
3316
|
static init(context, options = {}) {
|
|
2506
3317
|
const defs = {
|
|
2507
3318
|
table: "string default tasks",
|
|
2508
3319
|
target: "string default localRunner",
|
|
2509
3320
|
recreateTaskTables: "boolean default false",
|
|
2510
3321
|
pollMs: "number default 1000",
|
|
3322
|
+
claimJitterMs: "number default 0",
|
|
2511
3323
|
maxParallel: "number default 1",
|
|
2512
3324
|
scanLimit: "number default 100",
|
|
2513
3325
|
allowedTasks: "string",
|
|
2514
3326
|
runnerServiceGroup: "string",
|
|
2515
3327
|
runnerServiceName: "string",
|
|
2516
|
-
|
|
3328
|
+
runnerInstanceNumber: "number",
|
|
2517
3329
|
runnerHeartbeatIntervalMs: "number default 10000",
|
|
2518
3330
|
runnerHeartbeatStaleMs: "number default 45000",
|
|
2519
3331
|
runnerGroupMaxInstances: "number",
|
|
2520
3332
|
runnerEnforceMaxInstances: "boolean default true"
|
|
2521
3333
|
};
|
|
2522
|
-
const discovered = context.params.getAllForModule(defs);
|
|
3334
|
+
const discovered = context.params.getAllForModule("tasks", defs);
|
|
2523
3335
|
const resolved = {
|
|
2524
|
-
|
|
3336
|
+
queueName: discovered.table,
|
|
2525
3337
|
target: discovered.target,
|
|
2526
3338
|
recreateTaskTables: discovered.recreateTaskTables,
|
|
2527
3339
|
pollMs: discovered.pollMs,
|
|
3340
|
+
claimJitterMs: discovered.claimJitterMs,
|
|
2528
3341
|
maxParallel: discovered.maxParallel,
|
|
2529
3342
|
scanLimit: discovered.scanLimit,
|
|
2530
3343
|
allowedTasks: discovered.allowedTasks,
|
|
2531
3344
|
runnerServiceGroup: discovered.runnerServiceGroup,
|
|
2532
3345
|
runnerServiceName: discovered.runnerServiceName,
|
|
2533
|
-
|
|
3346
|
+
runnerInstanceNumber: discovered.runnerInstanceNumber,
|
|
2534
3347
|
runnerHeartbeatIntervalMs: discovered.runnerHeartbeatIntervalMs,
|
|
2535
3348
|
runnerHeartbeatStaleMs: discovered.runnerHeartbeatStaleMs,
|
|
2536
3349
|
runnerGroupMaxInstances: discovered.runnerGroupMaxInstances,
|
|
@@ -2539,24 +3352,39 @@ var TasksManager = class _TasksManager {
|
|
|
2539
3352
|
};
|
|
2540
3353
|
return new _TasksManager(context, resolved);
|
|
2541
3354
|
}
|
|
3355
|
+
/**
|
|
3356
|
+
* Idempotently ensure the three backing tables exist for this queue.
|
|
3357
|
+
*
|
|
3358
|
+
* @param {{ recreate?: boolean }} [options]
|
|
3359
|
+
* @returns {Promise<void>}
|
|
3360
|
+
*/
|
|
2542
3361
|
async ensureTaskTables(options = {}) {
|
|
2543
3362
|
await ensureTaskTables(this.context, {
|
|
2544
|
-
|
|
3363
|
+
queueName: this.queueName,
|
|
2545
3364
|
recreate: options.recreate ?? this.recreateTaskTables
|
|
2546
3365
|
});
|
|
2547
3366
|
}
|
|
3367
|
+
/**
|
|
3368
|
+
* Start the runner loop using this manager's resolved config. Per-call
|
|
3369
|
+
* options override the stored defaults, but `runnerMetadata` still falls
|
|
3370
|
+
* through when omitted.
|
|
3371
|
+
*
|
|
3372
|
+
* @param {Partial<ConstructorParameters<typeof TasksManager>[1]>} [options]
|
|
3373
|
+
* @returns {Promise<void>}
|
|
3374
|
+
*/
|
|
2548
3375
|
async runTasksLoop(options = {}) {
|
|
2549
3376
|
await runTasksLoop(this.context, {
|
|
2550
|
-
|
|
3377
|
+
queueName: options.queueName ?? this.queueName,
|
|
2551
3378
|
target: options.target ?? this.target,
|
|
2552
3379
|
pollMs: options.pollMs ?? this.pollMs,
|
|
3380
|
+
claimJitterMs: options.claimJitterMs ?? this.claimJitterMs,
|
|
2553
3381
|
maxParallel: options.maxParallel ?? this.maxParallel,
|
|
2554
3382
|
scanLimit: options.scanLimit ?? this.scanLimit,
|
|
2555
3383
|
allowedTasks: options.allowedTasks ?? this.allowedTasks,
|
|
2556
3384
|
registry: options.registry ?? this.registry,
|
|
2557
3385
|
runnerServiceGroup: options.runnerServiceGroup ?? this.runnerServiceGroup,
|
|
2558
3386
|
runnerServiceName: options.runnerServiceName ?? this.runnerServiceName,
|
|
2559
|
-
|
|
3387
|
+
runnerInstanceNumber: options.runnerInstanceNumber ?? this.runnerInstanceNumber,
|
|
2560
3388
|
runnerHeartbeatIntervalMs: options.runnerHeartbeatIntervalMs ?? this.runnerHeartbeatIntervalMs,
|
|
2561
3389
|
runnerHeartbeatStaleMs: options.runnerHeartbeatStaleMs ?? this.runnerHeartbeatStaleMs,
|
|
2562
3390
|
runnerGroupMaxInstances: options.runnerGroupMaxInstances ?? this.runnerGroupMaxInstances,
|
|
@@ -2566,7 +3394,9 @@ var TasksManager = class _TasksManager {
|
|
|
2566
3394
|
}
|
|
2567
3395
|
};
|
|
2568
3396
|
export {
|
|
2569
|
-
|
|
3397
|
+
AbstractTask,
|
|
3398
|
+
SERVICE_TASK_NAMES,
|
|
3399
|
+
TaskGetLogs,
|
|
2570
3400
|
TaskPing,
|
|
2571
3401
|
TaskSampleProcess,
|
|
2572
3402
|
TaskShellCommand,
|
|
@@ -2576,19 +3406,31 @@ export {
|
|
|
2576
3406
|
TasksManager,
|
|
2577
3407
|
TasksRegistry,
|
|
2578
3408
|
appendTaskIpcLog,
|
|
3409
|
+
convertPattern,
|
|
2579
3410
|
defaultTasksRegistry,
|
|
2580
3411
|
enqueueStopTask,
|
|
2581
3412
|
enqueueTask,
|
|
2582
3413
|
ensureTaskTables,
|
|
3414
|
+
flushTaskIpcLogs,
|
|
3415
|
+
ipcFileLogsTableNameForSourceResource,
|
|
2583
3416
|
listServicesRegistry as listAliveRunnerHeartbeats,
|
|
2584
3417
|
listServicesRegistry,
|
|
3418
|
+
matchesParsedPattern,
|
|
3419
|
+
mergeAllowedTasksWithServiceTasks,
|
|
3420
|
+
nextTimeMatch,
|
|
3421
|
+
normalizeAllowedTasks,
|
|
2585
3422
|
queueToTableNames,
|
|
3423
|
+
readTaskIpcLogsSnapshot,
|
|
2586
3424
|
registerInServicesRegistry,
|
|
2587
3425
|
registerInServicesRegistry as registerRunnerHeartbeat,
|
|
3426
|
+
resolveAsterisks,
|
|
3427
|
+
resolveIpcFileLogsDir,
|
|
3428
|
+
resolveRanges,
|
|
3429
|
+
resolveSteps,
|
|
2588
3430
|
runNodeTaskScript,
|
|
2589
3431
|
runTasksLoop,
|
|
2590
|
-
|
|
2591
|
-
|
|
3432
|
+
taskHistoryInsertFromQueueRow,
|
|
3433
|
+
timeMatcher,
|
|
2592
3434
|
touchServicesRegistry as touchRunnerHeartbeat,
|
|
2593
3435
|
touchServicesRegistry,
|
|
2594
3436
|
unregisterServicesRegistry as unregisterRunnerHeartbeat,
|