@nmakarov/cli-toolkit 0.18.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/dist/cli-runner.cjs +426 -97
- package/dist/cli-runner.cjs.map +1 -1
- package/dist/cli-runner.js +457 -128
- package/dist/cli-runner.js.map +1 -1
- package/dist/db.cjs +1 -1
- package/dist/db.cjs.map +1 -1
- package/dist/db.js +1 -1
- package/dist/db.js.map +1 -1
- package/dist/filedatabase.cjs.map +1 -1
- package/dist/filedatabase.js.map +1 -1
- package/dist/http-client2.cjs +6 -2
- package/dist/http-client2.cjs.map +1 -1
- package/dist/http-client2.js +6 -2
- package/dist/http-client2.js.map +1 -1
- package/dist/index.cjs +482 -96
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +469 -94
- package/dist/index.js.map +1 -1
- package/dist/init.cjs +16 -1
- package/dist/init.cjs.map +1 -1
- package/dist/init.js +16 -1
- package/dist/init.js.map +1 -1
- package/dist/mock-server.cjs.map +1 -1
- package/dist/mock-server.js.map +1 -1
- package/dist/params.cjs +16 -1
- package/dist/params.cjs.map +1 -1
- package/dist/params.js +16 -1
- package/dist/params.js.map +1 -1
- package/dist/tasks.cjs +458 -87
- package/dist/tasks.cjs.map +1 -1
- package/dist/tasks.js +478 -118
- package/dist/tasks.js.map +1 -1
- package/package.json +10 -3
- package/scripts/ssm/parse-cli.ts +35 -0
- package/scripts/ssm/ssm-admin.ts +151 -0
- package/scripts/ssm/ssm-pull.ts +150 -0
package/dist/cli-runner.cjs
CHANGED
|
@@ -924,11 +924,11 @@ function buildBreadcrumb(parts) {
|
|
|
924
924
|
if (parts.length === 1) return parts[0];
|
|
925
925
|
return parts.slice(1).map((part) => `\u2190 ${part}`).join(" ");
|
|
926
926
|
}
|
|
927
|
-
function buildDetailBreadcrumb(
|
|
928
|
-
if (
|
|
929
|
-
return suffix ? `\u2190 ${suffix}` :
|
|
927
|
+
function buildDetailBreadcrumb(path6, suffix = "") {
|
|
928
|
+
if (path6.length <= 1) {
|
|
929
|
+
return suffix ? `\u2190 ${suffix}` : path6[0] || "";
|
|
930
930
|
}
|
|
931
|
-
const breadcrumb = buildBreadcrumb(
|
|
931
|
+
const breadcrumb = buildBreadcrumb(path6);
|
|
932
932
|
return suffix ? `${breadcrumb} ${suffix}` : breadcrumb;
|
|
933
933
|
}
|
|
934
934
|
var init_utils = __esm({
|
|
@@ -1118,7 +1118,7 @@ var init_screen = __esm({
|
|
|
1118
1118
|
});
|
|
1119
1119
|
|
|
1120
1120
|
// src/scripts/cli-runner.ts
|
|
1121
|
-
var
|
|
1121
|
+
var import_node_path2 = __toESM(require("path"), 1);
|
|
1122
1122
|
var import_node_url = require("url");
|
|
1123
1123
|
|
|
1124
1124
|
// src/args/index.ts
|
|
@@ -1944,7 +1944,7 @@ var Params = class _Params {
|
|
|
1944
1944
|
throw new ParamError(`default value "${defValObj.value}" type mismatch`);
|
|
1945
1945
|
}
|
|
1946
1946
|
type = type.default(defValObj.value);
|
|
1947
|
-
} else if (str.match(
|
|
1947
|
+
} else if (str.match(/\s*required\s*/)) {
|
|
1948
1948
|
type = type.required();
|
|
1949
1949
|
} else {
|
|
1950
1950
|
type = type.optional();
|
|
@@ -2020,6 +2020,8 @@ var Params = class _Params {
|
|
|
2020
2020
|
/**
|
|
2021
2021
|
* Get all parameters from definitions (main script).
|
|
2022
2022
|
* Same as getAllForModule("script", defs). Processes left-to-right for cross-parameter references.
|
|
2023
|
+
* Libraries should use {@link getAllForModule} with an explicit module name (or {@link runWithModule}
|
|
2024
|
+
* around {@link get}) so --showUsedParams groups usage correctly.
|
|
2023
2025
|
*/
|
|
2024
2026
|
getAll(defs2) {
|
|
2025
2027
|
return this.getAllForModule("script", defs2);
|
|
@@ -2056,6 +2058,19 @@ var Params = class _Params {
|
|
|
2056
2058
|
this._currentModule = prev;
|
|
2057
2059
|
}
|
|
2058
2060
|
}
|
|
2061
|
+
/**
|
|
2062
|
+
* Run a callback with {@link _currentModule} set so single {@link get} calls are tracked
|
|
2063
|
+
* under the same module (for --showUsedParams / getFiguredByModule).
|
|
2064
|
+
*/
|
|
2065
|
+
runWithModule(moduleName, fn) {
|
|
2066
|
+
const prev = this._currentModule;
|
|
2067
|
+
this._currentModule = moduleName;
|
|
2068
|
+
try {
|
|
2069
|
+
return fn();
|
|
2070
|
+
} finally {
|
|
2071
|
+
this._currentModule = prev;
|
|
2072
|
+
}
|
|
2073
|
+
}
|
|
2059
2074
|
/**
|
|
2060
2075
|
* Infer module name from call stack: first caller outside params/index gives path like .../src/<moduleName>/...
|
|
2061
2076
|
*/
|
|
@@ -2069,9 +2084,9 @@ var Params = class _Params {
|
|
|
2069
2084
|
if (!parenMatch) continue;
|
|
2070
2085
|
const parts = parenMatch[1].split(":");
|
|
2071
2086
|
if (parts.length < 3) continue;
|
|
2072
|
-
const
|
|
2073
|
-
if (!
|
|
2074
|
-
const srcMatch =
|
|
2087
|
+
const path6 = parts.slice(0, -2).join(":").replace(/^file:\/\//, "");
|
|
2088
|
+
if (!path6 || path6.includes(paramsIndexPath)) continue;
|
|
2089
|
+
const srcMatch = path6.match(/[/\\]src[/\\]([^/\\]+)(?:[/\\]|$)/);
|
|
2075
2090
|
if (srcMatch) return srcMatch[1];
|
|
2076
2091
|
}
|
|
2077
2092
|
return "script";
|
|
@@ -2922,7 +2937,7 @@ async function dbFindAndConnect(context, dbNameOrConnectionString) {
|
|
|
2922
2937
|
dbConnectionString: "string",
|
|
2923
2938
|
dbProfile: "boolean default false"
|
|
2924
2939
|
};
|
|
2925
|
-
const paramsConfig = context.params.
|
|
2940
|
+
const paramsConfig = context.params.getAll(defs2);
|
|
2926
2941
|
dbName = paramsConfig.dbName;
|
|
2927
2942
|
dbConnectionString = paramsConfig.dbConnectionString;
|
|
2928
2943
|
dbProfile = paramsConfig.dbProfile;
|
|
@@ -3027,6 +3042,12 @@ function toJsonColumn(value) {
|
|
|
3027
3042
|
return JSON.stringify(value);
|
|
3028
3043
|
}
|
|
3029
3044
|
|
|
3045
|
+
// src/tasks/servicesRegistry.ts
|
|
3046
|
+
var import_node_crypto2 = require("crypto");
|
|
3047
|
+
var import_promises = require("fs/promises");
|
|
3048
|
+
var import_node_os = __toESM(require("os"), 1);
|
|
3049
|
+
var import_node_path = __toESM(require("path"), 1);
|
|
3050
|
+
|
|
3030
3051
|
// src/tasks/taskUtils.ts
|
|
3031
3052
|
var import_node_crypto = require("crypto");
|
|
3032
3053
|
function getDb(context) {
|
|
@@ -3045,6 +3066,12 @@ function queueToTableNames(queue) {
|
|
|
3045
3066
|
historyTable: `${queue}_history`
|
|
3046
3067
|
};
|
|
3047
3068
|
}
|
|
3069
|
+
function servicesRegistryTable(queue) {
|
|
3070
|
+
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(queue)) {
|
|
3071
|
+
throw new Error(`Invalid queue name "${queue}". Use letters, numbers, underscore only.`);
|
|
3072
|
+
}
|
|
3073
|
+
return `${queue}_services_registry`;
|
|
3074
|
+
}
|
|
3048
3075
|
async function ensureTaskTables(context, options = {}) {
|
|
3049
3076
|
const queue = options.queue ?? "tasks";
|
|
3050
3077
|
const recreate = options.recreate ?? false;
|
|
@@ -3081,18 +3108,6 @@ async function ensureTaskTables(context, options = {}) {
|
|
|
3081
3108
|
t.index(["target", "task"], `${tasksTable}_target_task_idx`);
|
|
3082
3109
|
});
|
|
3083
3110
|
}
|
|
3084
|
-
const tasksHasOpid = await db.schema.hasColumn(tasksTable, "opid");
|
|
3085
|
-
if (!tasksHasOpid) {
|
|
3086
|
-
await db.schema.alterTable(tasksTable, (t) => {
|
|
3087
|
-
t.text("opid");
|
|
3088
|
-
});
|
|
3089
|
-
}
|
|
3090
|
-
const tasksHasPausedAt = await db.schema.hasColumn(tasksTable, "paused_at");
|
|
3091
|
-
if (!tasksHasPausedAt) {
|
|
3092
|
-
await db.schema.alterTable(tasksTable, (t) => {
|
|
3093
|
-
t.timestamp("paused_at").defaultTo(null);
|
|
3094
|
-
});
|
|
3095
|
-
}
|
|
3096
3111
|
if (needsHistory) {
|
|
3097
3112
|
await db.schema.createTable(historyTable, (t) => {
|
|
3098
3113
|
t.uuid("id").notNullable();
|
|
@@ -3115,10 +3130,24 @@ async function ensureTaskTables(context, options = {}) {
|
|
|
3115
3130
|
t.index(["task", "created_at"], `${historyTable}_task_created_idx`);
|
|
3116
3131
|
});
|
|
3117
3132
|
}
|
|
3118
|
-
const
|
|
3119
|
-
|
|
3120
|
-
|
|
3121
|
-
|
|
3133
|
+
const registryTable = servicesRegistryTable(queue);
|
|
3134
|
+
const needsRegistry = !await db.tableExists(registryTable);
|
|
3135
|
+
if (needsRegistry) {
|
|
3136
|
+
await db.schema.createTable(registryTable, (t) => {
|
|
3137
|
+
t.uuid("id").primary().defaultTo(db.raw("uuid_generate_v4()"));
|
|
3138
|
+
t.uuid("instance_id").notNullable().unique();
|
|
3139
|
+
t.text("queue").notNullable();
|
|
3140
|
+
t.text("service_group").notNullable();
|
|
3141
|
+
t.text("service_name").notNullable();
|
|
3142
|
+
t.text("target").notNullable();
|
|
3143
|
+
t.text("hostname");
|
|
3144
|
+
t.integer("pid");
|
|
3145
|
+
t.json("metadata");
|
|
3146
|
+
t.timestamp("created_at").notNullable().defaultTo(db.fn.now());
|
|
3147
|
+
t.timestamp("last_seen_at").notNullable().defaultTo(db.fn.now());
|
|
3148
|
+
t.unique(["queue", "service_name"], `${registryTable}_queue_service_name_uniq`);
|
|
3149
|
+
t.index(["queue", "service_group", "last_seen_at"], `${registryTable}_queue_group_seen_idx`);
|
|
3150
|
+
t.index(["queue", "last_seen_at"], `${registryTable}_queue_seen_idx`);
|
|
3122
3151
|
});
|
|
3123
3152
|
}
|
|
3124
3153
|
}
|
|
@@ -3129,6 +3158,225 @@ async function updateTaskProgress(context, tasksTable, taskId, progress) {
|
|
|
3129
3158
|
});
|
|
3130
3159
|
}
|
|
3131
3160
|
|
|
3161
|
+
// src/tasks/servicesRegistry.ts
|
|
3162
|
+
function getDb2(context) {
|
|
3163
|
+
const db = context.db;
|
|
3164
|
+
if (!db) {
|
|
3165
|
+
throw new Error("Services registry requires context.db");
|
|
3166
|
+
}
|
|
3167
|
+
return db;
|
|
3168
|
+
}
|
|
3169
|
+
var DEFAULT_GROUP_MAX_INSTANCES = {
|
|
3170
|
+
intake: 1,
|
|
3171
|
+
harvest: 1,
|
|
3172
|
+
loader: 0,
|
|
3173
|
+
photos: 0,
|
|
3174
|
+
photosprocessor: 0,
|
|
3175
|
+
ingest: 0
|
|
3176
|
+
};
|
|
3177
|
+
function sanitizeNamePart(raw) {
|
|
3178
|
+
const s = String(raw || "").trim().replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
|
|
3179
|
+
return s.slice(0, 80) || "runner";
|
|
3180
|
+
}
|
|
3181
|
+
function identityFilePath(identityDir, queue, serviceGroup) {
|
|
3182
|
+
const safeQ = sanitizeNamePart(queue);
|
|
3183
|
+
const safeG = sanitizeNamePart(serviceGroup);
|
|
3184
|
+
return import_node_path.default.join(identityDir, `${safeQ}_${safeG}.json`);
|
|
3185
|
+
}
|
|
3186
|
+
async function readIdentityFile(filePath) {
|
|
3187
|
+
try {
|
|
3188
|
+
const text = await (0, import_promises.readFile)(filePath, "utf8");
|
|
3189
|
+
const parsed = JSON.parse(text);
|
|
3190
|
+
return parsed && typeof parsed === "object" ? parsed : {};
|
|
3191
|
+
} catch {
|
|
3192
|
+
return {};
|
|
3193
|
+
}
|
|
3194
|
+
}
|
|
3195
|
+
async function writeIdentityFile(filePath, data) {
|
|
3196
|
+
await (0, import_promises.mkdir)(import_node_path.default.dirname(filePath), { recursive: true });
|
|
3197
|
+
await (0, import_promises.writeFile)(filePath, `${JSON.stringify(data, null, 2)}
|
|
3198
|
+
`, "utf8");
|
|
3199
|
+
}
|
|
3200
|
+
function resolveMaxInstances(serviceGroup, override) {
|
|
3201
|
+
if (override !== void 0 && Number.isFinite(override)) {
|
|
3202
|
+
return Math.max(0, Math.floor(Number(override)));
|
|
3203
|
+
}
|
|
3204
|
+
const g = serviceGroup.trim().toLowerCase();
|
|
3205
|
+
return DEFAULT_GROUP_MAX_INSTANCES[g] ?? 0;
|
|
3206
|
+
}
|
|
3207
|
+
async function countAliveInGroup(db, registryTable, queue, serviceGroup, staleMs, excludeInstanceId) {
|
|
3208
|
+
const cutoff = new Date(Date.now() - staleMs);
|
|
3209
|
+
let q = db(registryTable).where({ queue, service_group: serviceGroup }).where("last_seen_at", ">", cutoff);
|
|
3210
|
+
if (excludeInstanceId) {
|
|
3211
|
+
q = q.whereNot("instance_id", excludeInstanceId);
|
|
3212
|
+
}
|
|
3213
|
+
const row = await q.count("id as count").first();
|
|
3214
|
+
return Number(row?.count ?? 0);
|
|
3215
|
+
}
|
|
3216
|
+
function isUniqueViolation(error) {
|
|
3217
|
+
const code = error?.code ?? error?.errno;
|
|
3218
|
+
return code === "23505" || String(error?.message || "").includes("duplicate key");
|
|
3219
|
+
}
|
|
3220
|
+
async function registerInServicesRegistry(context, options) {
|
|
3221
|
+
const db = getDb2(context);
|
|
3222
|
+
const registryTable = servicesRegistryTable(options.queue);
|
|
3223
|
+
const serviceGroup = options.serviceGroup.trim();
|
|
3224
|
+
if (!serviceGroup) {
|
|
3225
|
+
throw new Error("registerInServicesRegistry: serviceGroup is required");
|
|
3226
|
+
}
|
|
3227
|
+
const identityPath = identityFilePath(options.identityDir, options.queue, serviceGroup);
|
|
3228
|
+
let identity = await readIdentityFile(identityPath);
|
|
3229
|
+
let instanceId = typeof identity.instanceId === "string" && identity.instanceId.trim() ? identity.instanceId.trim() : (0, import_node_crypto2.randomUUID)();
|
|
3230
|
+
identity.instanceId = instanceId;
|
|
3231
|
+
await writeIdentityFile(identityPath, identity);
|
|
3232
|
+
const hostname = import_node_os.default.hostname();
|
|
3233
|
+
const pid = typeof process.pid === "number" ? process.pid : null;
|
|
3234
|
+
const meta = toJsonColumn(options.metadata ?? null);
|
|
3235
|
+
const existing = await db(registryTable).where({ instance_id: instanceId }).first();
|
|
3236
|
+
if (existing) {
|
|
3237
|
+
await db(registryTable).where({ instance_id: instanceId }).update({
|
|
3238
|
+
target: options.target,
|
|
3239
|
+
hostname,
|
|
3240
|
+
pid,
|
|
3241
|
+
metadata: meta,
|
|
3242
|
+
last_seen_at: db.fn.now()
|
|
3243
|
+
});
|
|
3244
|
+
const serviceName = String(existing.service_name);
|
|
3245
|
+
identity.serviceName = serviceName;
|
|
3246
|
+
await writeIdentityFile(identityPath, identity);
|
|
3247
|
+
const reg = {
|
|
3248
|
+
instanceId,
|
|
3249
|
+
serviceName,
|
|
3250
|
+
serviceGroup,
|
|
3251
|
+
queue: options.queue,
|
|
3252
|
+
target: options.target,
|
|
3253
|
+
rowId: String(existing.id)
|
|
3254
|
+
};
|
|
3255
|
+
context.servicesRegistry = reg;
|
|
3256
|
+
context.runnerHeartbeat = reg;
|
|
3257
|
+
context.logger.info?.(
|
|
3258
|
+
`[services-registry] resumed instance_id=${instanceId} name=${serviceName} group=${serviceGroup} queue=${options.queue}`
|
|
3259
|
+
);
|
|
3260
|
+
return {
|
|
3261
|
+
instanceId,
|
|
3262
|
+
serviceName,
|
|
3263
|
+
serviceGroup,
|
|
3264
|
+
queue: options.queue,
|
|
3265
|
+
target: options.target,
|
|
3266
|
+
rowId: String(existing.id),
|
|
3267
|
+
registryTable
|
|
3268
|
+
};
|
|
3269
|
+
}
|
|
3270
|
+
const maxAllowed = resolveMaxInstances(serviceGroup, options.groupMaxInstances);
|
|
3271
|
+
const aliveOthers = await countAliveInGroup(
|
|
3272
|
+
db,
|
|
3273
|
+
registryTable,
|
|
3274
|
+
options.queue,
|
|
3275
|
+
serviceGroup,
|
|
3276
|
+
options.staleMs,
|
|
3277
|
+
instanceId
|
|
3278
|
+
);
|
|
3279
|
+
if (maxAllowed > 0 && aliveOthers >= maxAllowed) {
|
|
3280
|
+
const msg = `[services-registry] group limit reached for "${serviceGroup}": ${aliveOthers} alive (max ${maxAllowed}, queue=${options.queue}).`;
|
|
3281
|
+
if (options.enforceMaxInstances) {
|
|
3282
|
+
throw new Error(msg);
|
|
3283
|
+
}
|
|
3284
|
+
context.logger.warn?.(`${msg} Starting anyway (runnerEnforceMaxInstances=false).`);
|
|
3285
|
+
}
|
|
3286
|
+
const explicitName = options.serviceName?.trim();
|
|
3287
|
+
const fromFile = typeof identity.serviceName === "string" ? identity.serviceName.trim() : "";
|
|
3288
|
+
const hostBase = sanitizeNamePart(hostname);
|
|
3289
|
+
const groupBase = sanitizeNamePart(serviceGroup);
|
|
3290
|
+
const baseCandidates = [];
|
|
3291
|
+
if (explicitName) baseCandidates.push(sanitizeNamePart(explicitName));
|
|
3292
|
+
if (fromFile) baseCandidates.push(sanitizeNamePart(fromFile));
|
|
3293
|
+
baseCandidates.push(`${groupBase}-${hostBase}`);
|
|
3294
|
+
baseCandidates.push(groupBase);
|
|
3295
|
+
function* eachServiceNameCandidate(bases) {
|
|
3296
|
+
const seen = /* @__PURE__ */ new Set();
|
|
3297
|
+
for (const rawBase of bases) {
|
|
3298
|
+
const base = sanitizeNamePart(rawBase);
|
|
3299
|
+
if (!base) continue;
|
|
3300
|
+
const seq = [base];
|
|
3301
|
+
for (let n = 2; n <= 500; n++) seq.push(`${base}-${n}`);
|
|
3302
|
+
for (const c of seq) {
|
|
3303
|
+
if (seen.has(c)) continue;
|
|
3304
|
+
seen.add(c);
|
|
3305
|
+
yield c;
|
|
3306
|
+
}
|
|
3307
|
+
}
|
|
3308
|
+
}
|
|
3309
|
+
let inserted;
|
|
3310
|
+
for (const candidate of eachServiceNameCandidate(baseCandidates)) {
|
|
3311
|
+
try {
|
|
3312
|
+
const rows = await db(registryTable).insert({
|
|
3313
|
+
instance_id: instanceId,
|
|
3314
|
+
queue: options.queue,
|
|
3315
|
+
service_group: serviceGroup,
|
|
3316
|
+
service_name: candidate,
|
|
3317
|
+
target: options.target,
|
|
3318
|
+
hostname,
|
|
3319
|
+
pid,
|
|
3320
|
+
metadata: meta,
|
|
3321
|
+
last_seen_at: db.fn.now()
|
|
3322
|
+
}).returning(["id", "service_name"]);
|
|
3323
|
+
const row = Array.isArray(rows) ? rows[0] : rows;
|
|
3324
|
+
if (row) {
|
|
3325
|
+
inserted = { id: String(row.id), service_name: String(row.service_name) };
|
|
3326
|
+
break;
|
|
3327
|
+
}
|
|
3328
|
+
} catch (error) {
|
|
3329
|
+
if (!isUniqueViolation(error)) {
|
|
3330
|
+
throw error;
|
|
3331
|
+
}
|
|
3332
|
+
}
|
|
3333
|
+
}
|
|
3334
|
+
if (!inserted) {
|
|
3335
|
+
throw new Error(
|
|
3336
|
+
`[services-registry] could not allocate a unique service_name for group=${serviceGroup} queue=${options.queue} (too many collisions).`
|
|
3337
|
+
);
|
|
3338
|
+
}
|
|
3339
|
+
identity.serviceName = inserted.service_name;
|
|
3340
|
+
await writeIdentityFile(identityPath, identity);
|
|
3341
|
+
const regNew = {
|
|
3342
|
+
instanceId,
|
|
3343
|
+
serviceName: inserted.service_name,
|
|
3344
|
+
serviceGroup,
|
|
3345
|
+
queue: options.queue,
|
|
3346
|
+
target: options.target,
|
|
3347
|
+
rowId: inserted.id
|
|
3348
|
+
};
|
|
3349
|
+
context.servicesRegistry = regNew;
|
|
3350
|
+
context.runnerHeartbeat = regNew;
|
|
3351
|
+
context.logger.info?.(
|
|
3352
|
+
`[services-registry] registered instance_id=${instanceId} name=${inserted.service_name} group=${serviceGroup} queue=${options.queue} target=${options.target}`
|
|
3353
|
+
);
|
|
3354
|
+
return {
|
|
3355
|
+
instanceId,
|
|
3356
|
+
serviceName: inserted.service_name,
|
|
3357
|
+
serviceGroup,
|
|
3358
|
+
queue: options.queue,
|
|
3359
|
+
target: options.target,
|
|
3360
|
+
rowId: inserted.id,
|
|
3361
|
+
registryTable
|
|
3362
|
+
};
|
|
3363
|
+
}
|
|
3364
|
+
async function touchServicesRegistry(context, registration) {
|
|
3365
|
+
const db = getDb2(context);
|
|
3366
|
+
const hostname = import_node_os.default.hostname();
|
|
3367
|
+
const pid = typeof process.pid === "number" ? process.pid : null;
|
|
3368
|
+
await db(registration.registryTable).where({ instance_id: registration.instanceId }).update({
|
|
3369
|
+
last_seen_at: db.fn.now(),
|
|
3370
|
+
hostname,
|
|
3371
|
+
pid
|
|
3372
|
+
});
|
|
3373
|
+
}
|
|
3374
|
+
async function unregisterServicesRegistry(context, registration) {
|
|
3375
|
+
const db = getDb2(context);
|
|
3376
|
+
await db(registration.registryTable).where({ instance_id: registration.instanceId }).delete();
|
|
3377
|
+
context.logger.info?.(`[services-registry] unregistered name=${registration.serviceName} instance_id=${registration.instanceId}`);
|
|
3378
|
+
}
|
|
3379
|
+
|
|
3132
3380
|
// src/filedatabase/index.ts
|
|
3133
3381
|
var import_fs4 = __toESM(require("fs"), 1);
|
|
3134
3382
|
var import_path4 = __toESM(require("path"), 1);
|
|
@@ -4497,8 +4745,8 @@ var TaskShellCommand = class extends TaskMaster {
|
|
|
4497
4745
|
};
|
|
4498
4746
|
|
|
4499
4747
|
// src/tasks/coreTasks/TaskSystemInfo.ts
|
|
4500
|
-
var
|
|
4501
|
-
var
|
|
4748
|
+
var import_node_os2 = __toESM(require("os"), 1);
|
|
4749
|
+
var import_promises2 = __toESM(require("fs/promises"), 1);
|
|
4502
4750
|
function toGb(valueBytes) {
|
|
4503
4751
|
return `${(valueBytes / 1024 ** 3).toFixed(2)} GB`;
|
|
4504
4752
|
}
|
|
@@ -4506,7 +4754,7 @@ function toMb(valueBytes) {
|
|
|
4506
4754
|
return `${(valueBytes / 1024 ** 2).toFixed(2)} MB`;
|
|
4507
4755
|
}
|
|
4508
4756
|
async function getDiskStats() {
|
|
4509
|
-
const stats = await
|
|
4757
|
+
const stats = await import_promises2.default.statfs("/");
|
|
4510
4758
|
const total = Number(stats.bsize) * Number(stats.blocks);
|
|
4511
4759
|
const free = Number(stats.bsize) * Number(stats.bavail);
|
|
4512
4760
|
const used = total - free;
|
|
@@ -4519,10 +4767,10 @@ async function getDiskStats() {
|
|
|
4519
4767
|
var TaskSystemInfo = class extends TaskMaster {
|
|
4520
4768
|
async run() {
|
|
4521
4769
|
try {
|
|
4522
|
-
const totalMemory =
|
|
4523
|
-
const freeMemory =
|
|
4770
|
+
const totalMemory = import_node_os2.default.totalmem();
|
|
4771
|
+
const freeMemory = import_node_os2.default.freemem();
|
|
4524
4772
|
const usedMemory = totalMemory - freeMemory;
|
|
4525
|
-
const cpus =
|
|
4773
|
+
const cpus = import_node_os2.default.cpus();
|
|
4526
4774
|
const cpuUtilization = cpus.map((cpu) => {
|
|
4527
4775
|
const total = Object.values(cpu.times).reduce((acc, time) => acc + time, 0);
|
|
4528
4776
|
const usage = (total - cpu.times.idle) / total * 100;
|
|
@@ -4548,10 +4796,10 @@ var TaskSystemInfo = class extends TaskMaster {
|
|
|
4548
4796
|
utilization: cpuUtilization
|
|
4549
4797
|
},
|
|
4550
4798
|
runtime: {
|
|
4551
|
-
platform:
|
|
4552
|
-
arch:
|
|
4553
|
-
uptimeSec:
|
|
4554
|
-
hostname:
|
|
4799
|
+
platform: import_node_os2.default.platform(),
|
|
4800
|
+
arch: import_node_os2.default.arch(),
|
|
4801
|
+
uptimeSec: import_node_os2.default.uptime(),
|
|
4802
|
+
hostname: import_node_os2.default.hostname()
|
|
4555
4803
|
}
|
|
4556
4804
|
};
|
|
4557
4805
|
this.context.logger.info?.(`[TaskSystemInfo] collected system metrics (${this.task.id})`);
|
|
@@ -4654,7 +4902,7 @@ var import_node_child_process2 = require("child_process");
|
|
|
4654
4902
|
// src/tasks/index.ts
|
|
4655
4903
|
var LOCKED_BY_ERROR_MESSAGE = "locked by error";
|
|
4656
4904
|
var defaultTasksRegistry = TasksRegistry.withCoreTasks();
|
|
4657
|
-
function
|
|
4905
|
+
function getDb3(context) {
|
|
4658
4906
|
const db = context.db;
|
|
4659
4907
|
if (!db) {
|
|
4660
4908
|
throw new Error("Tasks component requires context.db. Initialize DB first and attach to context.");
|
|
@@ -4689,7 +4937,7 @@ async function signalRunningTasksStop(context, runningTaskInstances, allowanceMs
|
|
|
4689
4937
|
context.emitter.emit("stop", allowanceMs);
|
|
4690
4938
|
}
|
|
4691
4939
|
async function executeClaimedTask(context, tasksTable, historyTable, row, registry, runningTaskInstances) {
|
|
4692
|
-
const db =
|
|
4940
|
+
const db = getDb3(context);
|
|
4693
4941
|
const taskName = row.task;
|
|
4694
4942
|
const TaskClass = registry.get(taskName);
|
|
4695
4943
|
const { paused_at: _pausedAt, ...rowForHistory } = row;
|
|
@@ -4790,7 +5038,7 @@ async function executeClaimedTask(context, tasksTable, historyTable, row, regist
|
|
|
4790
5038
|
return { stopRunnerRequested, stopAllowanceMs };
|
|
4791
5039
|
}
|
|
4792
5040
|
async function claimNextRunnableTask(context, tasksTable, target, registry, scanLimit, taskNames) {
|
|
4793
|
-
const db =
|
|
5041
|
+
const db = getDb3(context);
|
|
4794
5042
|
let query = db(tasksTable).whereNull("started_at").whereNull("paused_at").where({ target }).orderByRaw("CASE WHEN past_due IS NULL THEN 1 ELSE 0 END ASC").orderBy([{ column: "priority", order: "desc" }]).orderByRaw("CASE WHEN completed_at IS NULL THEN 0 ELSE 1 END ASC").orderBy([{ column: "completed_at", order: "asc" }, { column: "created_at", order: "asc" }]).limit(scanLimit);
|
|
4795
5043
|
if (taskNames && taskNames.length > 0) {
|
|
4796
5044
|
query = query.whereIn("task", taskNames);
|
|
@@ -4836,25 +5084,76 @@ async function runTasksLoop(context, options) {
|
|
|
4836
5084
|
let stopRequested = false;
|
|
4837
5085
|
let stopAllowanceMs = 5e3;
|
|
4838
5086
|
context.__tasksRunnerStop = false;
|
|
4839
|
-
|
|
4840
|
-
|
|
4841
|
-
|
|
4842
|
-
|
|
4843
|
-
|
|
4844
|
-
|
|
4845
|
-
|
|
4846
|
-
|
|
4847
|
-
|
|
4848
|
-
)
|
|
4849
|
-
|
|
4850
|
-
|
|
5087
|
+
let registryReg = null;
|
|
5088
|
+
let registryInterval = null;
|
|
5089
|
+
const hbGroup = options.runnerServiceGroup?.trim();
|
|
5090
|
+
if (hbGroup) {
|
|
5091
|
+
const identityDir = options.runnerIdentityDir ?? "./data/runner-identities";
|
|
5092
|
+
const hbIntervalMs = options.runnerHeartbeatIntervalMs ?? 1e4;
|
|
5093
|
+
const staleMs = options.runnerHeartbeatStaleMs ?? 45e3;
|
|
5094
|
+
const defaultMeta = {
|
|
5095
|
+
component: "tasks-runner",
|
|
5096
|
+
allowedTasks: allowedTasks?.length ? allowedTasks.join(",") : "all"
|
|
5097
|
+
};
|
|
5098
|
+
registryReg = await registerInServicesRegistry(context, {
|
|
5099
|
+
queue,
|
|
5100
|
+
target,
|
|
5101
|
+
serviceGroup: hbGroup,
|
|
5102
|
+
serviceName: options.runnerServiceName,
|
|
5103
|
+
identityDir,
|
|
5104
|
+
staleMs,
|
|
5105
|
+
groupMaxInstances: options.runnerGroupMaxInstances,
|
|
5106
|
+
enforceMaxInstances: options.runnerEnforceMaxInstances ?? true,
|
|
5107
|
+
metadata: options.runnerMetadata ?? defaultMeta
|
|
5108
|
+
});
|
|
5109
|
+
registryInterval = setInterval(() => {
|
|
5110
|
+
void touchServicesRegistry(context, registryReg).catch((err) => {
|
|
5111
|
+
context.logger.warn?.(`[services-registry] touch failed: ${err?.message ?? String(err)}`);
|
|
5112
|
+
});
|
|
5113
|
+
}, hbIntervalMs);
|
|
5114
|
+
}
|
|
5115
|
+
try {
|
|
5116
|
+
while (!context.isStop() && !stopRequested && !context.__tasksRunnerStop) {
|
|
5117
|
+
if (!runningStopControlPromise) {
|
|
5118
|
+
const claimedStopTask = await claimNextRunnableTask(
|
|
5119
|
+
context,
|
|
5120
|
+
tasksTable,
|
|
5121
|
+
target,
|
|
5122
|
+
registry,
|
|
5123
|
+
10,
|
|
5124
|
+
["stopRunner", "stop"]
|
|
5125
|
+
);
|
|
5126
|
+
if (claimedStopTask) {
|
|
5127
|
+
runningStopControlPromise = executeClaimedTask(
|
|
5128
|
+
context,
|
|
5129
|
+
tasksTable,
|
|
5130
|
+
historyTable,
|
|
5131
|
+
claimedStopTask,
|
|
5132
|
+
registry,
|
|
5133
|
+
runningTaskInstances
|
|
5134
|
+
).then(async (outcome) => {
|
|
5135
|
+
if (outcome.stopRunnerRequested && !stopRequested) {
|
|
5136
|
+
stopRequested = true;
|
|
5137
|
+
stopAllowanceMs = outcome.stopAllowanceMs || 5e3;
|
|
5138
|
+
context.__tasksRunnerStop = true;
|
|
5139
|
+
await signalRunningTasksStop(context, runningTaskInstances, stopAllowanceMs);
|
|
5140
|
+
}
|
|
5141
|
+
}).finally(() => {
|
|
5142
|
+
runningStopControlPromise = null;
|
|
5143
|
+
});
|
|
5144
|
+
}
|
|
5145
|
+
}
|
|
5146
|
+
while (runningPromises.size < maxParallel) {
|
|
5147
|
+
const claimed = await claimNextRunnableTask(
|
|
4851
5148
|
context,
|
|
4852
5149
|
tasksTable,
|
|
4853
|
-
|
|
4854
|
-
claimedStopTask,
|
|
5150
|
+
target,
|
|
4855
5151
|
registry,
|
|
4856
|
-
|
|
4857
|
-
|
|
5152
|
+
scanLimit,
|
|
5153
|
+
allowedTasks
|
|
5154
|
+
);
|
|
5155
|
+
if (!claimed) break;
|
|
5156
|
+
const p = executeClaimedTask(context, tasksTable, historyTable, claimed, registry, runningTaskInstances).then(async (outcome) => {
|
|
4858
5157
|
if (outcome.stopRunnerRequested && !stopRequested) {
|
|
4859
5158
|
stopRequested = true;
|
|
4860
5159
|
stopAllowanceMs = outcome.stopAllowanceMs || 5e3;
|
|
@@ -4862,49 +5161,41 @@ async function runTasksLoop(context, options) {
|
|
|
4862
5161
|
await signalRunningTasksStop(context, runningTaskInstances, stopAllowanceMs);
|
|
4863
5162
|
}
|
|
4864
5163
|
}).finally(() => {
|
|
4865
|
-
|
|
5164
|
+
runningPromises.delete(p);
|
|
4866
5165
|
});
|
|
5166
|
+
runningPromises.add(p);
|
|
5167
|
+
}
|
|
5168
|
+
await sleepMs(pollMs);
|
|
5169
|
+
}
|
|
5170
|
+
if (context.isStop() && !stopRequested) {
|
|
5171
|
+
await signalRunningTasksStop(context, runningTaskInstances, 5e3);
|
|
5172
|
+
}
|
|
5173
|
+
if (runningPromises.size > 0) {
|
|
5174
|
+
if (stopRequested) {
|
|
5175
|
+
await Promise.race([
|
|
5176
|
+
Promise.allSettled(Array.from(runningPromises)),
|
|
5177
|
+
sleepMs(stopAllowanceMs).then(() => {
|
|
5178
|
+
context.logger.warn?.(
|
|
5179
|
+
`[tasks] stop allowance (${stopAllowanceMs}ms) reached; ${runningPromises.size} task(s) still running`
|
|
5180
|
+
);
|
|
5181
|
+
})
|
|
5182
|
+
]);
|
|
5183
|
+
} else {
|
|
5184
|
+
await Promise.allSettled(Array.from(runningPromises));
|
|
4867
5185
|
}
|
|
4868
5186
|
}
|
|
4869
|
-
|
|
4870
|
-
|
|
4871
|
-
|
|
4872
|
-
|
|
4873
|
-
target,
|
|
4874
|
-
registry,
|
|
4875
|
-
scanLimit,
|
|
4876
|
-
allowedTasks
|
|
4877
|
-
);
|
|
4878
|
-
if (!claimed) break;
|
|
4879
|
-
const p = executeClaimedTask(context, tasksTable, historyTable, claimed, registry, runningTaskInstances).then(async (outcome) => {
|
|
4880
|
-
if (outcome.stopRunnerRequested && !stopRequested) {
|
|
4881
|
-
stopRequested = true;
|
|
4882
|
-
stopAllowanceMs = outcome.stopAllowanceMs || 5e3;
|
|
4883
|
-
context.__tasksRunnerStop = true;
|
|
4884
|
-
await signalRunningTasksStop(context, runningTaskInstances, stopAllowanceMs);
|
|
4885
|
-
}
|
|
4886
|
-
}).finally(() => {
|
|
4887
|
-
runningPromises.delete(p);
|
|
4888
|
-
});
|
|
4889
|
-
runningPromises.add(p);
|
|
5187
|
+
} finally {
|
|
5188
|
+
if (registryInterval) {
|
|
5189
|
+
clearInterval(registryInterval);
|
|
5190
|
+
registryInterval = null;
|
|
4890
5191
|
}
|
|
4891
|
-
|
|
4892
|
-
|
|
4893
|
-
|
|
4894
|
-
|
|
4895
|
-
|
|
4896
|
-
|
|
4897
|
-
|
|
4898
|
-
await Promise.race([
|
|
4899
|
-
Promise.allSettled(Array.from(runningPromises)),
|
|
4900
|
-
sleepMs(stopAllowanceMs).then(() => {
|
|
4901
|
-
context.logger.warn?.(
|
|
4902
|
-
`[tasks] stop allowance (${stopAllowanceMs}ms) reached; ${runningPromises.size} task(s) still running`
|
|
4903
|
-
);
|
|
4904
|
-
})
|
|
4905
|
-
]);
|
|
4906
|
-
} else {
|
|
4907
|
-
await Promise.allSettled(Array.from(runningPromises));
|
|
5192
|
+
if (registryReg) {
|
|
5193
|
+
await unregisterServicesRegistry(context, registryReg).catch((err) => {
|
|
5194
|
+
context.logger.warn?.(`[services-registry] unregister failed: ${err?.message ?? String(err)}`);
|
|
5195
|
+
});
|
|
5196
|
+
registryReg = null;
|
|
5197
|
+
delete context.servicesRegistry;
|
|
5198
|
+
delete context.runnerHeartbeat;
|
|
4908
5199
|
}
|
|
4909
5200
|
}
|
|
4910
5201
|
}
|
|
@@ -4918,6 +5209,14 @@ var TasksManager = class _TasksManager {
|
|
|
4918
5209
|
scanLimit;
|
|
4919
5210
|
allowedTasks;
|
|
4920
5211
|
registry;
|
|
5212
|
+
runnerServiceGroup;
|
|
5213
|
+
runnerServiceName;
|
|
5214
|
+
runnerIdentityDir;
|
|
5215
|
+
runnerHeartbeatIntervalMs;
|
|
5216
|
+
runnerHeartbeatStaleMs;
|
|
5217
|
+
runnerGroupMaxInstances;
|
|
5218
|
+
runnerEnforceMaxInstances;
|
|
5219
|
+
runnerMetadata;
|
|
4921
5220
|
constructor(context, options = {}) {
|
|
4922
5221
|
this.context = context;
|
|
4923
5222
|
this.queue = options.queue ?? "tasks";
|
|
@@ -4928,6 +5227,14 @@ var TasksManager = class _TasksManager {
|
|
|
4928
5227
|
this.scanLimit = options.scanLimit ?? 100;
|
|
4929
5228
|
this.allowedTasks = normalizeAllowedTasks(options.allowedTasks);
|
|
4930
5229
|
this.registry = normalizeRegistry(options.registry);
|
|
5230
|
+
this.runnerServiceGroup = options.runnerServiceGroup;
|
|
5231
|
+
this.runnerServiceName = options.runnerServiceName;
|
|
5232
|
+
this.runnerIdentityDir = options.runnerIdentityDir;
|
|
5233
|
+
this.runnerHeartbeatIntervalMs = options.runnerHeartbeatIntervalMs;
|
|
5234
|
+
this.runnerHeartbeatStaleMs = options.runnerHeartbeatStaleMs;
|
|
5235
|
+
this.runnerGroupMaxInstances = options.runnerGroupMaxInstances;
|
|
5236
|
+
this.runnerEnforceMaxInstances = options.runnerEnforceMaxInstances;
|
|
5237
|
+
this.runnerMetadata = options.runnerMetadata;
|
|
4931
5238
|
}
|
|
4932
5239
|
static init(context, options = {}) {
|
|
4933
5240
|
const defs2 = {
|
|
@@ -4937,7 +5244,14 @@ var TasksManager = class _TasksManager {
|
|
|
4937
5244
|
pollMs: "number default 1000",
|
|
4938
5245
|
maxParallel: "number default 1",
|
|
4939
5246
|
scanLimit: "number default 100",
|
|
4940
|
-
allowedTasks: "string"
|
|
5247
|
+
allowedTasks: "string",
|
|
5248
|
+
runnerServiceGroup: "string",
|
|
5249
|
+
runnerServiceName: "string",
|
|
5250
|
+
runnerIdentityDir: "string default ./data/runner-identities",
|
|
5251
|
+
runnerHeartbeatIntervalMs: "number default 10000",
|
|
5252
|
+
runnerHeartbeatStaleMs: "number default 45000",
|
|
5253
|
+
runnerGroupMaxInstances: "number",
|
|
5254
|
+
runnerEnforceMaxInstances: "boolean default true"
|
|
4941
5255
|
};
|
|
4942
5256
|
const discovered = context.params.getAllForModule(defs2);
|
|
4943
5257
|
const resolved = {
|
|
@@ -4948,6 +5262,13 @@ var TasksManager = class _TasksManager {
|
|
|
4948
5262
|
maxParallel: discovered.maxParallel,
|
|
4949
5263
|
scanLimit: discovered.scanLimit,
|
|
4950
5264
|
allowedTasks: discovered.allowedTasks,
|
|
5265
|
+
runnerServiceGroup: discovered.runnerServiceGroup,
|
|
5266
|
+
runnerServiceName: discovered.runnerServiceName,
|
|
5267
|
+
runnerIdentityDir: discovered.runnerIdentityDir,
|
|
5268
|
+
runnerHeartbeatIntervalMs: discovered.runnerHeartbeatIntervalMs,
|
|
5269
|
+
runnerHeartbeatStaleMs: discovered.runnerHeartbeatStaleMs,
|
|
5270
|
+
runnerGroupMaxInstances: discovered.runnerGroupMaxInstances,
|
|
5271
|
+
runnerEnforceMaxInstances: discovered.runnerEnforceMaxInstances,
|
|
4951
5272
|
...options
|
|
4952
5273
|
};
|
|
4953
5274
|
return new _TasksManager(context, resolved);
|
|
@@ -4966,7 +5287,15 @@ var TasksManager = class _TasksManager {
|
|
|
4966
5287
|
maxParallel: options.maxParallel ?? this.maxParallel,
|
|
4967
5288
|
scanLimit: options.scanLimit ?? this.scanLimit,
|
|
4968
5289
|
allowedTasks: options.allowedTasks ?? this.allowedTasks,
|
|
4969
|
-
registry: options.registry ?? this.registry
|
|
5290
|
+
registry: options.registry ?? this.registry,
|
|
5291
|
+
runnerServiceGroup: options.runnerServiceGroup ?? this.runnerServiceGroup,
|
|
5292
|
+
runnerServiceName: options.runnerServiceName ?? this.runnerServiceName,
|
|
5293
|
+
runnerIdentityDir: options.runnerIdentityDir ?? this.runnerIdentityDir,
|
|
5294
|
+
runnerHeartbeatIntervalMs: options.runnerHeartbeatIntervalMs ?? this.runnerHeartbeatIntervalMs,
|
|
5295
|
+
runnerHeartbeatStaleMs: options.runnerHeartbeatStaleMs ?? this.runnerHeartbeatStaleMs,
|
|
5296
|
+
runnerGroupMaxInstances: options.runnerGroupMaxInstances ?? this.runnerGroupMaxInstances,
|
|
5297
|
+
runnerEnforceMaxInstances: options.runnerEnforceMaxInstances ?? this.runnerEnforceMaxInstances,
|
|
5298
|
+
runnerMetadata: options.runnerMetadata ?? this.runnerMetadata
|
|
4970
5299
|
});
|
|
4971
5300
|
}
|
|
4972
5301
|
};
|
|
@@ -4977,7 +5306,7 @@ var defs = {
|
|
|
4977
5306
|
tasksModule: "string"
|
|
4978
5307
|
};
|
|
4979
5308
|
async function loadTasksModule(modulePath) {
|
|
4980
|
-
const absolute =
|
|
5309
|
+
const absolute = import_node_path2.default.isAbsolute(modulePath) ? modulePath : import_node_path2.default.resolve(process.cwd(), modulePath);
|
|
4981
5310
|
const imported = await import((0, import_node_url.pathToFileURL)(absolute).href);
|
|
4982
5311
|
if (!imported.tasksRegistry || typeof imported.tasksRegistry !== "object") {
|
|
4983
5312
|
throw new Error(`tasksModule "${modulePath}" must export "tasksRegistry" object`);
|