opencode-supertask 0.1.26 → 0.1.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +41 -17
- package/dist/cli/index.js +3464 -1089
- package/dist/cli/index.js.map +1 -1
- package/dist/daemon/pm2-supervisor.d.ts +3 -0
- package/dist/daemon/pm2-supervisor.js +145 -0
- package/dist/daemon/pm2-supervisor.js.map +1 -0
- package/dist/gateway/index.d.ts +8 -1
- package/dist/gateway/index.js +2131 -681
- package/dist/gateway/index.js.map +1 -1
- package/dist/plugin/supertask.d.ts +2 -1
- package/dist/plugin/supertask.js +1352 -345
- package/dist/plugin/supertask.js.map +1 -1
- package/dist/web/index.d.ts +1 -0
- package/dist/web/index.js +1042 -264
- package/dist/web/index.js.map +1 -1
- package/dist/worker/index.d.ts +17 -3
- package/dist/worker/index.js +1204 -247
- package/dist/worker/index.js.map +1 -1
- package/dist/worker/launcher.d.ts +9 -0
- package/dist/worker/launcher.js +164 -0
- package/dist/worker/launcher.js.map +1 -0
- package/drizzle/0005_natural_ma_gnuci.sql +1 -0
- package/drizzle/0006_worried_agent_zero.sql +2 -0
- package/drizzle/0007_lean_titanium_man.sql +1 -0
- package/drizzle/meta/0005_snapshot.json +559 -0
- package/drizzle/meta/0006_snapshot.json +575 -0
- package/drizzle/meta/0007_snapshot.json +585 -0
- package/drizzle/meta/_journal.json +21 -0
- package/package.json +1 -1
package/dist/gateway/index.js
CHANGED
|
@@ -1204,7 +1204,7 @@ var init_sql = __esm({
|
|
|
1204
1204
|
return new SQL([new StringChunk(str)]);
|
|
1205
1205
|
}
|
|
1206
1206
|
sql2.raw = raw2;
|
|
1207
|
-
function
|
|
1207
|
+
function join5(chunks, separator) {
|
|
1208
1208
|
const result = [];
|
|
1209
1209
|
for (const [i, chunk] of chunks.entries()) {
|
|
1210
1210
|
if (i > 0 && separator !== void 0) {
|
|
@@ -1214,7 +1214,7 @@ var init_sql = __esm({
|
|
|
1214
1214
|
}
|
|
1215
1215
|
return new SQL(result);
|
|
1216
1216
|
}
|
|
1217
|
-
sql2.join =
|
|
1217
|
+
sql2.join = join5;
|
|
1218
1218
|
function identifier(value) {
|
|
1219
1219
|
return new Name(value);
|
|
1220
1220
|
}
|
|
@@ -3830,7 +3830,7 @@ var init_select2 = __esm({
|
|
|
3830
3830
|
return (table, on) => {
|
|
3831
3831
|
const baseTableName = this.tableName;
|
|
3832
3832
|
const tableName = getTableLikeName(table);
|
|
3833
|
-
if (typeof tableName === "string" && this.config.joins?.some((
|
|
3833
|
+
if (typeof tableName === "string" && this.config.joins?.some((join5) => join5.alias === tableName)) {
|
|
3834
3834
|
throw new Error(`Alias "${tableName}" is already used in this query`);
|
|
3835
3835
|
}
|
|
3836
3836
|
if (!this.isPartialSelect) {
|
|
@@ -4672,7 +4672,7 @@ var init_update = __esm({
|
|
|
4672
4672
|
createJoin(joinType) {
|
|
4673
4673
|
return (table, on) => {
|
|
4674
4674
|
const tableName = getTableLikeName(table);
|
|
4675
|
-
if (typeof tableName === "string" && this.config.joins.some((
|
|
4675
|
+
if (typeof tableName === "string" && this.config.joins.some((join5) => join5.alias === tableName)) {
|
|
4676
4676
|
throw new Error(`Alias "${tableName}" is already used in this query`);
|
|
4677
4677
|
}
|
|
4678
4678
|
if (typeof on === "function") {
|
|
@@ -6033,7 +6033,9 @@ var init_schema = __esm({
|
|
|
6033
6033
|
}, (table) => [
|
|
6034
6034
|
index("tasks_queue_idx").on(table.status, table.retryAfter, table.urgency, table.importance, table.createdAt, table.id),
|
|
6035
6035
|
index("tasks_batch_status_idx").on(table.batchId, table.status),
|
|
6036
|
-
index("tasks_template_status_idx").on(table.templateId, table.status)
|
|
6036
|
+
index("tasks_template_status_idx").on(table.templateId, table.status),
|
|
6037
|
+
index("tasks_depends_on_status_idx").on(table.dependsOn, table.status),
|
|
6038
|
+
index("tasks_cleanup_idx").on(table.finishedAt, table.id, table.status)
|
|
6037
6039
|
]);
|
|
6038
6040
|
TASK_CATEGORIES = [
|
|
6039
6041
|
"translate",
|
|
@@ -6056,7 +6058,8 @@ var init_schema = __esm({
|
|
|
6056
6058
|
lockedBy: text("locked_by"),
|
|
6057
6059
|
heartbeatAt: integer("heartbeat_at"),
|
|
6058
6060
|
workerPid: integer("worker_pid"),
|
|
6059
|
-
childPid: integer("child_pid")
|
|
6061
|
+
childPid: integer("child_pid"),
|
|
6062
|
+
launchProtocol: text("launch_protocol")
|
|
6060
6063
|
}, (table) => [
|
|
6061
6064
|
index("task_runs_task_started_idx").on(table.taskId, table.startedAt, table.id),
|
|
6062
6065
|
index("task_runs_status_heartbeat_idx").on(table.status, table.heartbeatAt)
|
|
@@ -6086,7 +6089,13 @@ var init_schema = __esm({
|
|
|
6086
6089
|
createdAt: integer("created_at").default(0),
|
|
6087
6090
|
updatedAt: integer("updated_at").default(0)
|
|
6088
6091
|
}, (table) => [
|
|
6089
|
-
index("task_templates_due_idx").on(table.enabled, table.nextRunAt, table.id)
|
|
6092
|
+
index("task_templates_due_idx").on(table.enabled, table.nextRunAt, table.id),
|
|
6093
|
+
index("task_templates_retention_idx").on(
|
|
6094
|
+
table.scheduleType,
|
|
6095
|
+
table.enabled,
|
|
6096
|
+
table.lastRunAt,
|
|
6097
|
+
table.id
|
|
6098
|
+
)
|
|
6090
6099
|
]);
|
|
6091
6100
|
}
|
|
6092
6101
|
});
|
|
@@ -6109,36 +6118,47 @@ function getMigrationsFolder() {
|
|
|
6109
6118
|
}
|
|
6110
6119
|
return join(__dirname, "../../drizzle");
|
|
6111
6120
|
}
|
|
6112
|
-
function
|
|
6113
|
-
|
|
6114
|
-
if (!existsSync(dataDir)) {
|
|
6115
|
-
mkdirSync(dataDir, { recursive: true });
|
|
6116
|
-
}
|
|
6117
|
-
_sqlite = new Database2(DB_FILE_PATH);
|
|
6118
|
-
_sqlite.exec("PRAGMA journal_mode = WAL;");
|
|
6119
|
-
_sqlite.exec("PRAGMA busy_timeout = 5000;");
|
|
6120
|
-
_sqlite.exec(`
|
|
6121
|
+
function ensureGatewayLock(sqliteDb) {
|
|
6122
|
+
sqliteDb.exec(`
|
|
6121
6123
|
CREATE TABLE IF NOT EXISTS gateway_lock (
|
|
6122
6124
|
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
6123
6125
|
pid INTEGER NOT NULL,
|
|
6124
6126
|
acquired_at INTEGER NOT NULL,
|
|
6125
6127
|
heartbeat_at INTEGER NOT NULL,
|
|
6126
|
-
ready_at INTEGER
|
|
6128
|
+
ready_at INTEGER,
|
|
6129
|
+
version TEXT
|
|
6127
6130
|
);
|
|
6128
6131
|
`);
|
|
6129
|
-
const
|
|
6130
|
-
if (!
|
|
6131
|
-
|
|
6132
|
+
const columns = sqliteDb.query("PRAGMA table_info(gateway_lock)").all();
|
|
6133
|
+
if (!columns.some((column) => column.name === "ready_at")) {
|
|
6134
|
+
sqliteDb.exec("ALTER TABLE gateway_lock ADD COLUMN ready_at INTEGER;");
|
|
6135
|
+
}
|
|
6136
|
+
if (!columns.some((column) => column.name === "version")) {
|
|
6137
|
+
sqliteDb.exec("ALTER TABLE gateway_lock ADD COLUMN version TEXT;");
|
|
6138
|
+
}
|
|
6139
|
+
}
|
|
6140
|
+
function migrateSqliteDatabase(sqliteDb) {
|
|
6141
|
+
ensureGatewayLock(sqliteDb);
|
|
6142
|
+
const drizzleDb = drizzle(sqliteDb, { schema: schema_exports });
|
|
6143
|
+
migrate(drizzleDb, { migrationsFolder: getMigrationsFolder() });
|
|
6144
|
+
sqliteDb.exec("PRAGMA foreign_keys = ON;");
|
|
6145
|
+
const violations = sqliteDb.query("PRAGMA foreign_key_check;").all();
|
|
6146
|
+
if (violations.length > 0) {
|
|
6147
|
+
throw new Error(`\u68C0\u6D4B\u5230 ${violations.length} \u6761\u5B64\u7ACB\u5173\u8054\u8BB0\u5F55\uFF0C\u8BF7\u5148\u4FEE\u590D\u6570\u636E\u518D\u542F\u52A8`);
|
|
6148
|
+
}
|
|
6149
|
+
return drizzleDb;
|
|
6150
|
+
}
|
|
6151
|
+
function initDb() {
|
|
6152
|
+
const dataDir = dirname(DB_FILE_PATH);
|
|
6153
|
+
if (!existsSync(dataDir)) {
|
|
6154
|
+
mkdirSync(dataDir, { recursive: true });
|
|
6132
6155
|
}
|
|
6133
|
-
|
|
6156
|
+
_sqlite = new Database2(DB_FILE_PATH);
|
|
6157
|
+
_sqlite.exec("PRAGMA journal_mode = WAL;");
|
|
6158
|
+
_sqlite.exec("PRAGMA busy_timeout = 5000;");
|
|
6134
6159
|
if (!_migrationRan) {
|
|
6135
6160
|
try {
|
|
6136
|
-
|
|
6137
|
-
_sqlite.exec("PRAGMA foreign_keys = ON;");
|
|
6138
|
-
const violations = _sqlite.query("PRAGMA foreign_key_check;").all();
|
|
6139
|
-
if (violations.length > 0) {
|
|
6140
|
-
throw new Error(`\u68C0\u6D4B\u5230 ${violations.length} \u6761\u5B64\u7ACB\u5173\u8054\u8BB0\u5F55\uFF0C\u8BF7\u5148\u4FEE\u590D\u6570\u636E\u518D\u542F\u52A8`);
|
|
6141
|
-
}
|
|
6161
|
+
_db = migrateSqliteDatabase(_sqlite);
|
|
6142
6162
|
_migrationRan = true;
|
|
6143
6163
|
} catch (err) {
|
|
6144
6164
|
const msg = err instanceof Error ? err.message : String(err);
|
|
@@ -6149,6 +6169,8 @@ function initDb() {
|
|
|
6149
6169
|
console.error(`[supertask] migration failed: ${msg}`);
|
|
6150
6170
|
throw new Error(`[supertask] DB migration failed: ${msg}`);
|
|
6151
6171
|
}
|
|
6172
|
+
} else {
|
|
6173
|
+
_db = drizzle(_sqlite, { schema: schema_exports });
|
|
6152
6174
|
}
|
|
6153
6175
|
return _db;
|
|
6154
6176
|
}
|
|
@@ -6370,7 +6392,52 @@ var init_backoff = __esm({
|
|
|
6370
6392
|
});
|
|
6371
6393
|
|
|
6372
6394
|
// src/core/services/task.service.ts
|
|
6373
|
-
|
|
6395
|
+
function hasNoExecutableDependents() {
|
|
6396
|
+
return sql`NOT EXISTS (
|
|
6397
|
+
SELECT 1 FROM tasks AS dependent_task
|
|
6398
|
+
WHERE dependent_task.depends_on = ${tasks2.id}
|
|
6399
|
+
AND dependent_task.status IN ('pending', 'running', 'failed', 'dead_letter')
|
|
6400
|
+
)`;
|
|
6401
|
+
}
|
|
6402
|
+
function hasViableDependency() {
|
|
6403
|
+
return or(
|
|
6404
|
+
isNull(tasks2.dependsOn),
|
|
6405
|
+
sql`EXISTS (
|
|
6406
|
+
SELECT 1 FROM tasks AS dependency_task
|
|
6407
|
+
WHERE dependency_task.id = ${tasks2.dependsOn}
|
|
6408
|
+
AND dependency_task.cwd IS ${tasks2.cwd}
|
|
6409
|
+
AND (
|
|
6410
|
+
dependency_task.status IN ('pending', 'running', 'done')
|
|
6411
|
+
OR (
|
|
6412
|
+
dependency_task.status = 'failed'
|
|
6413
|
+
AND dependency_task.retry_count <= dependency_task.max_retries
|
|
6414
|
+
)
|
|
6415
|
+
)
|
|
6416
|
+
)`
|
|
6417
|
+
);
|
|
6418
|
+
}
|
|
6419
|
+
function blockedDependentsOf(prerequisiteId) {
|
|
6420
|
+
return sql`${tasks2.id} IN (
|
|
6421
|
+
WITH RECURSIVE blocked_task(id) AS (
|
|
6422
|
+
SELECT direct_dependent.id
|
|
6423
|
+
FROM tasks AS direct_dependent
|
|
6424
|
+
WHERE direct_dependent.depends_on = ${prerequisiteId}
|
|
6425
|
+
AND direct_dependent.status IN ('pending', 'failed')
|
|
6426
|
+
AND EXISTS (
|
|
6427
|
+
SELECT 1 FROM tasks AS prerequisite
|
|
6428
|
+
WHERE prerequisite.id = ${prerequisiteId}
|
|
6429
|
+
AND prerequisite.status IN ('dead_letter', 'cancelled')
|
|
6430
|
+
)
|
|
6431
|
+
UNION
|
|
6432
|
+
SELECT descendant.id
|
|
6433
|
+
FROM tasks AS descendant
|
|
6434
|
+
INNER JOIN blocked_task ON descendant.depends_on = blocked_task.id
|
|
6435
|
+
WHERE descendant.status IN ('pending', 'failed')
|
|
6436
|
+
)
|
|
6437
|
+
SELECT id FROM blocked_task
|
|
6438
|
+
)`;
|
|
6439
|
+
}
|
|
6440
|
+
var tasks2, taskRuns2, cleanupInvocation, TaskDeletionConflictError, TaskService;
|
|
6374
6441
|
var init_task_service = __esm({
|
|
6375
6442
|
"src/core/services/task.service.ts"() {
|
|
6376
6443
|
"use strict";
|
|
@@ -6378,6 +6445,13 @@ var init_task_service = __esm({
|
|
|
6378
6445
|
init_drizzle_orm();
|
|
6379
6446
|
init_backoff();
|
|
6380
6447
|
({ tasks: tasks2, taskRuns: taskRuns2 } = schema_exports);
|
|
6448
|
+
cleanupInvocation = 0;
|
|
6449
|
+
TaskDeletionConflictError = class extends Error {
|
|
6450
|
+
constructor(message) {
|
|
6451
|
+
super(message);
|
|
6452
|
+
this.name = "TaskDeletionConflictError";
|
|
6453
|
+
}
|
|
6454
|
+
};
|
|
6381
6455
|
TaskService = class {
|
|
6382
6456
|
static buildScopeWhere(scope) {
|
|
6383
6457
|
const conditions = [];
|
|
@@ -6388,8 +6462,28 @@ var init_task_service = __esm({
|
|
|
6388
6462
|
}
|
|
6389
6463
|
static async add(data) {
|
|
6390
6464
|
this.validateNewTask(data);
|
|
6391
|
-
|
|
6392
|
-
|
|
6465
|
+
return db.transaction((tx) => {
|
|
6466
|
+
if (data.dependsOn != null) {
|
|
6467
|
+
const dependency = tx.select({
|
|
6468
|
+
id: tasks2.id,
|
|
6469
|
+
cwd: tasks2.cwd,
|
|
6470
|
+
status: tasks2.status,
|
|
6471
|
+
retryCount: tasks2.retryCount,
|
|
6472
|
+
maxRetries: tasks2.maxRetries
|
|
6473
|
+
}).from(tasks2).where(eq(tasks2.id, data.dependsOn)).get();
|
|
6474
|
+
if (!dependency) {
|
|
6475
|
+
throw new Error(`dependsOn \u6307\u5411\u7684\u4EFB\u52A1 #${data.dependsOn} \u4E0D\u5B58\u5728`);
|
|
6476
|
+
}
|
|
6477
|
+
if ((dependency.cwd ?? null) !== (data.cwd ?? null)) {
|
|
6478
|
+
throw new Error("dependsOn \u5FC5\u987B\u6307\u5411\u540C\u4E00 cwd \u7684\u4EFB\u52A1");
|
|
6479
|
+
}
|
|
6480
|
+
const dependencyIsRecoverable = dependency.status === "pending" || dependency.status === "running" || dependency.status === "done" || dependency.status === "failed" && (dependency.retryCount ?? 0) <= (dependency.maxRetries ?? 3);
|
|
6481
|
+
if (!dependencyIsRecoverable) {
|
|
6482
|
+
throw new Error(`dependsOn \u6307\u5411\u7684\u4EFB\u52A1 #${data.dependsOn} \u5DF2\u8FDB\u5165\u4E0D\u53EF\u6062\u590D\u7EC8\u6001`);
|
|
6483
|
+
}
|
|
6484
|
+
}
|
|
6485
|
+
return tx.insert(tasks2).values(data).returning().get();
|
|
6486
|
+
}, { behavior: "immediate" });
|
|
6393
6487
|
}
|
|
6394
6488
|
static validateNewTask(data) {
|
|
6395
6489
|
if (!data.name.trim()) throw new Error("name \u4E0D\u80FD\u4E3A\u7A7A");
|
|
@@ -6441,23 +6535,54 @@ var init_task_service = __esm({
|
|
|
6441
6535
|
if (batchFilter) {
|
|
6442
6536
|
conditions.push(batchFilter);
|
|
6443
6537
|
}
|
|
6444
|
-
const
|
|
6538
|
+
const result = await db.select().from(tasks2).where(and(
|
|
6539
|
+
...conditions,
|
|
6540
|
+
or(
|
|
6541
|
+
isNull(tasks2.dependsOn),
|
|
6542
|
+
sql`EXISTS (
|
|
6543
|
+
SELECT 1 FROM tasks AS dependency_task
|
|
6544
|
+
WHERE dependency_task.id = ${tasks2.dependsOn}
|
|
6545
|
+
AND dependency_task.status = 'done'
|
|
6546
|
+
AND dependency_task.cwd IS ${tasks2.cwd}
|
|
6547
|
+
)`
|
|
6548
|
+
),
|
|
6549
|
+
or(
|
|
6550
|
+
isNull(tasks2.batchId),
|
|
6551
|
+
sql`NOT EXISTS (
|
|
6552
|
+
SELECT 1 FROM tasks AS running_batch_task
|
|
6553
|
+
WHERE running_batch_task.batch_id = ${tasks2.batchId}
|
|
6554
|
+
AND (
|
|
6555
|
+
running_batch_task.status = 'running'
|
|
6556
|
+
OR EXISTS (
|
|
6557
|
+
SELECT 1 FROM task_runs AS running_batch_run
|
|
6558
|
+
WHERE running_batch_run.task_id = running_batch_task.id
|
|
6559
|
+
AND running_batch_run.status = 'running'
|
|
6560
|
+
)
|
|
6561
|
+
)
|
|
6562
|
+
)`
|
|
6563
|
+
),
|
|
6564
|
+
sql`NOT EXISTS (
|
|
6565
|
+
SELECT 1 FROM task_runs AS candidate_active_run
|
|
6566
|
+
WHERE candidate_active_run.task_id = ${tasks2.id}
|
|
6567
|
+
AND candidate_active_run.status = 'running'
|
|
6568
|
+
)`
|
|
6569
|
+
)).orderBy(
|
|
6445
6570
|
desc(tasks2.urgency),
|
|
6446
6571
|
desc(tasks2.importance),
|
|
6447
6572
|
asc(tasks2.createdAt),
|
|
6448
6573
|
asc(tasks2.id)
|
|
6449
|
-
);
|
|
6450
|
-
|
|
6451
|
-
|
|
6452
|
-
|
|
6453
|
-
|
|
6454
|
-
|
|
6455
|
-
|
|
6456
|
-
|
|
6457
|
-
|
|
6458
|
-
|
|
6459
|
-
|
|
6460
|
-
|
|
6574
|
+
).limit(1);
|
|
6575
|
+
return result[0] ?? null;
|
|
6576
|
+
}
|
|
6577
|
+
static async countRunning() {
|
|
6578
|
+
return db.transaction((tx) => {
|
|
6579
|
+
const runningTasks = tx.select({ count: sql`count(*)` }).from(tasks2).where(eq(tasks2.status, "running")).get();
|
|
6580
|
+
const runsWithoutRunningTask = tx.select({ count: sql`count(DISTINCT ${taskRuns2.taskId})` }).from(taskRuns2).innerJoin(tasks2, eq(tasks2.id, taskRuns2.taskId)).where(and(
|
|
6581
|
+
eq(taskRuns2.status, "running"),
|
|
6582
|
+
sql`${tasks2.status} <> 'running'`
|
|
6583
|
+
)).get();
|
|
6584
|
+
return Number(runningTasks?.count ?? 0) + Number(runsWithoutRunningTask?.count ?? 0);
|
|
6585
|
+
}, { behavior: "deferred" });
|
|
6461
6586
|
}
|
|
6462
6587
|
static async start(id, scope = {}) {
|
|
6463
6588
|
const conditions = [
|
|
@@ -6493,27 +6618,202 @@ var init_task_service = __esm({
|
|
|
6493
6618
|
return result[0] || null;
|
|
6494
6619
|
}
|
|
6495
6620
|
static async fail(id, log, scope = {}, options) {
|
|
6496
|
-
|
|
6497
|
-
|
|
6498
|
-
|
|
6499
|
-
|
|
6500
|
-
|
|
6501
|
-
|
|
6502
|
-
|
|
6503
|
-
|
|
6504
|
-
|
|
6505
|
-
|
|
6621
|
+
return db.transaction((tx) => {
|
|
6622
|
+
const current = tx.select().from(tasks2).where(and(
|
|
6623
|
+
eq(tasks2.id, id),
|
|
6624
|
+
eq(tasks2.status, "running"),
|
|
6625
|
+
...this.buildScopeWhere(scope)
|
|
6626
|
+
)).get();
|
|
6627
|
+
if (!current) return null;
|
|
6628
|
+
const newRetryCount = (current.retryCount ?? 0) + 1;
|
|
6629
|
+
const maxRetries = current.maxRetries ?? 3;
|
|
6630
|
+
const isDeadLetter = options?.setDeadLetter ?? newRetryCount > maxRetries;
|
|
6631
|
+
const failed = tx.update(tasks2).set({
|
|
6632
|
+
status: isDeadLetter ? "dead_letter" : "failed",
|
|
6633
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
6634
|
+
resultLog: log,
|
|
6635
|
+
retryCount: newRetryCount,
|
|
6636
|
+
retryAfter: isDeadLetter ? null : options?.retryAfterMs ?? Date.now() + computeBackoff(
|
|
6637
|
+
newRetryCount,
|
|
6638
|
+
current.retryBackoffMs ?? 3e4
|
|
6639
|
+
)
|
|
6640
|
+
}).where(and(eq(tasks2.id, id), eq(tasks2.status, "running"))).returning().get();
|
|
6641
|
+
if (failed?.status === "dead_letter") {
|
|
6642
|
+
tx.update(tasks2).set({
|
|
6643
|
+
status: "dead_letter",
|
|
6644
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
6645
|
+
retryAfter: null,
|
|
6646
|
+
resultLog: `\u4F9D\u8D56\u4EFB\u52A1 #${id} \u5DF2\u8FDB\u5165\u4E0D\u53EF\u6062\u590D\u7EC8\u6001`
|
|
6647
|
+
}).where(blockedDependentsOf(id)).run();
|
|
6648
|
+
}
|
|
6649
|
+
return failed ?? null;
|
|
6650
|
+
}, { behavior: "immediate" });
|
|
6651
|
+
}
|
|
6652
|
+
static async completeRun(taskId, runId, log) {
|
|
6653
|
+
return db.transaction((tx) => {
|
|
6654
|
+
const currentTask = tx.select().from(tasks2).where(and(eq(tasks2.id, taskId), eq(tasks2.status, "running"))).get();
|
|
6655
|
+
const currentRun = tx.select({ id: taskRuns2.id }).from(taskRuns2).where(and(
|
|
6656
|
+
eq(taskRuns2.id, runId),
|
|
6657
|
+
eq(taskRuns2.taskId, taskId),
|
|
6658
|
+
eq(taskRuns2.status, "running")
|
|
6659
|
+
)).get();
|
|
6660
|
+
if (!currentTask || !currentRun) return null;
|
|
6661
|
+
const finishedAt = /* @__PURE__ */ new Date();
|
|
6662
|
+
const completed = tx.update(tasks2).set({
|
|
6663
|
+
status: "done",
|
|
6664
|
+
finishedAt,
|
|
6665
|
+
resultLog: log,
|
|
6666
|
+
retryAfter: null
|
|
6667
|
+
}).where(and(eq(tasks2.id, taskId), eq(tasks2.status, "running"))).returning().get();
|
|
6668
|
+
if (!completed) return null;
|
|
6669
|
+
tx.update(taskRuns2).set({ status: "done", finishedAt, log }).where(and(eq(taskRuns2.id, runId), eq(taskRuns2.status, "running"))).run();
|
|
6670
|
+
return completed;
|
|
6671
|
+
}, { behavior: "immediate" });
|
|
6672
|
+
}
|
|
6673
|
+
static async failRun(taskId, runId, log, options) {
|
|
6674
|
+
const failed = db.transaction((tx) => {
|
|
6675
|
+
const currentTask = tx.select().from(tasks2).where(and(eq(tasks2.id, taskId), eq(tasks2.status, "running"))).get();
|
|
6676
|
+
const currentRun = tx.select({ id: taskRuns2.id }).from(taskRuns2).where(and(
|
|
6677
|
+
eq(taskRuns2.id, runId),
|
|
6678
|
+
eq(taskRuns2.taskId, taskId),
|
|
6679
|
+
eq(taskRuns2.status, "running")
|
|
6680
|
+
)).get();
|
|
6681
|
+
if (!currentTask || !currentRun) return null;
|
|
6682
|
+
const newRetryCount = (currentTask.retryCount ?? 0) + 1;
|
|
6683
|
+
const maxRetries = currentTask.maxRetries ?? 3;
|
|
6684
|
+
const isDeadLetter = options?.setDeadLetter ?? newRetryCount > maxRetries;
|
|
6685
|
+
const finishedAt = /* @__PURE__ */ new Date();
|
|
6686
|
+
const retryAfter = isDeadLetter ? null : options?.retryAfterMs ?? Date.now() + computeBackoff(
|
|
6687
|
+
newRetryCount,
|
|
6688
|
+
currentTask.retryBackoffMs ?? 3e4
|
|
6689
|
+
);
|
|
6690
|
+
const failed2 = tx.update(tasks2).set({
|
|
6691
|
+
status: isDeadLetter ? "dead_letter" : "failed",
|
|
6692
|
+
finishedAt,
|
|
6693
|
+
resultLog: log,
|
|
6694
|
+
retryCount: newRetryCount,
|
|
6695
|
+
retryAfter
|
|
6696
|
+
}).where(and(eq(tasks2.id, taskId), eq(tasks2.status, "running"))).returning().get();
|
|
6697
|
+
if (!failed2) return null;
|
|
6698
|
+
tx.update(taskRuns2).set({ status: "failed", finishedAt, log }).where(and(eq(taskRuns2.id, runId), eq(taskRuns2.status, "running"))).run();
|
|
6699
|
+
if (failed2.status === "dead_letter") {
|
|
6700
|
+
tx.update(tasks2).set({
|
|
6701
|
+
status: "dead_letter",
|
|
6702
|
+
finishedAt,
|
|
6703
|
+
retryAfter: null,
|
|
6704
|
+
resultLog: `\u4F9D\u8D56\u4EFB\u52A1 #${taskId} \u5DF2\u8FDB\u5165\u4E0D\u53EF\u6062\u590D\u7EC8\u6001`
|
|
6705
|
+
}).where(blockedDependentsOf(taskId)).run();
|
|
6706
|
+
}
|
|
6707
|
+
return failed2;
|
|
6708
|
+
}, { behavior: "immediate" });
|
|
6709
|
+
return failed;
|
|
6710
|
+
}
|
|
6711
|
+
static async recoverRun(taskId, runId, log) {
|
|
6712
|
+
const recovery = db.transaction((tx) => {
|
|
6713
|
+
const currentTask = tx.select().from(tasks2).where(and(eq(tasks2.id, taskId), eq(tasks2.status, "running"))).get();
|
|
6714
|
+
const currentRun = tx.select({ id: taskRuns2.id }).from(taskRuns2).where(and(
|
|
6715
|
+
eq(taskRuns2.id, runId),
|
|
6716
|
+
eq(taskRuns2.taskId, taskId),
|
|
6717
|
+
eq(taskRuns2.status, "running")
|
|
6718
|
+
)).get();
|
|
6719
|
+
if (!currentRun) return null;
|
|
6720
|
+
const finishedAt = /* @__PURE__ */ new Date();
|
|
6721
|
+
tx.update(taskRuns2).set({ status: "failed", finishedAt, log }).where(and(eq(taskRuns2.id, runId), eq(taskRuns2.status, "running"))).run();
|
|
6722
|
+
if (!currentTask) return null;
|
|
6723
|
+
const retryCount = (currentTask.retryCount ?? 0) + 1;
|
|
6724
|
+
const maxRetries = currentTask.maxRetries ?? 3;
|
|
6725
|
+
const isDeadLetter = retryCount > maxRetries;
|
|
6726
|
+
const recoveryStatus = isDeadLetter ? "dead_letter" : "pending";
|
|
6727
|
+
const retryAfterMs = isDeadLetter ? null : Date.now() + computeBackoff(
|
|
6728
|
+
retryCount,
|
|
6729
|
+
currentTask.retryBackoffMs ?? 3e4
|
|
6730
|
+
);
|
|
6731
|
+
tx.update(tasks2).set({
|
|
6732
|
+
status: recoveryStatus,
|
|
6733
|
+
startedAt: null,
|
|
6734
|
+
finishedAt: isDeadLetter ? finishedAt : null,
|
|
6735
|
+
retryCount,
|
|
6736
|
+
retryAfter: retryAfterMs,
|
|
6737
|
+
resultLog: log
|
|
6738
|
+
}).where(and(eq(tasks2.id, taskId), eq(tasks2.status, "running"))).run();
|
|
6739
|
+
if (recoveryStatus === "dead_letter") {
|
|
6740
|
+
tx.update(tasks2).set({
|
|
6741
|
+
status: "dead_letter",
|
|
6742
|
+
finishedAt,
|
|
6743
|
+
retryAfter: null,
|
|
6744
|
+
resultLog: `\u4F9D\u8D56\u4EFB\u52A1 #${taskId} \u5DF2\u8FDB\u5165\u4E0D\u53EF\u6062\u590D\u7EC8\u6001`
|
|
6745
|
+
}).where(blockedDependentsOf(taskId)).run();
|
|
6746
|
+
}
|
|
6747
|
+
return {
|
|
6748
|
+
status: recoveryStatus,
|
|
6749
|
+
retryCount,
|
|
6750
|
+
retryAfterMs
|
|
6751
|
+
};
|
|
6752
|
+
}, { behavior: "immediate" });
|
|
6753
|
+
return recovery;
|
|
6754
|
+
}
|
|
6755
|
+
static async interruptRun(taskId, runId, log) {
|
|
6756
|
+
return db.transaction((tx) => {
|
|
6757
|
+
const currentRun = tx.select({ id: taskRuns2.id }).from(taskRuns2).where(and(
|
|
6758
|
+
eq(taskRuns2.id, runId),
|
|
6759
|
+
eq(taskRuns2.taskId, taskId),
|
|
6760
|
+
eq(taskRuns2.status, "running")
|
|
6761
|
+
)).get();
|
|
6762
|
+
if (!currentRun) return false;
|
|
6763
|
+
const updatedRun = tx.update(taskRuns2).set({ status: "failed", finishedAt: /* @__PURE__ */ new Date(), log }).where(and(eq(taskRuns2.id, runId), eq(taskRuns2.status, "running"))).returning({ id: taskRuns2.id }).get();
|
|
6764
|
+
tx.update(tasks2).set({
|
|
6765
|
+
status: "pending",
|
|
6766
|
+
startedAt: null,
|
|
6767
|
+
finishedAt: null,
|
|
6768
|
+
retryAfter: null,
|
|
6769
|
+
resultLog: log
|
|
6770
|
+
}).where(and(eq(tasks2.id, taskId), eq(tasks2.status, "running"))).run();
|
|
6771
|
+
return updatedRun != null;
|
|
6772
|
+
}, { behavior: "immediate" });
|
|
6773
|
+
}
|
|
6774
|
+
static async resolveBlockedDependencies() {
|
|
6775
|
+
const finishedAt = /* @__PURE__ */ new Date();
|
|
6776
|
+
const result = await db.update(tasks2).set({
|
|
6777
|
+
status: "dead_letter",
|
|
6778
|
+
finishedAt,
|
|
6779
|
+
retryAfter: null,
|
|
6780
|
+
resultLog: "\u4F9D\u8D56\u4EFB\u52A1\u4E0D\u5B58\u5728\u3001\u8DE8\u9879\u76EE\u6216\u5DF2\u8FDB\u5165\u4E0D\u53EF\u6062\u590D\u7EC8\u6001"
|
|
6781
|
+
}).where(sql`${tasks2.id} IN (
|
|
6782
|
+
WITH RECURSIVE blocked_task(id) AS (
|
|
6783
|
+
SELECT candidate.id
|
|
6784
|
+
FROM tasks AS candidate
|
|
6785
|
+
WHERE candidate.status IN ('pending', 'failed')
|
|
6786
|
+
AND candidate.depends_on IS NOT NULL
|
|
6787
|
+
AND NOT EXISTS (
|
|
6788
|
+
SELECT 1 FROM tasks AS viable_dependency
|
|
6789
|
+
WHERE viable_dependency.id = candidate.depends_on
|
|
6790
|
+
AND viable_dependency.cwd IS candidate.cwd
|
|
6791
|
+
AND (
|
|
6792
|
+
viable_dependency.status IN ('pending', 'running', 'done')
|
|
6793
|
+
OR (
|
|
6794
|
+
viable_dependency.status = 'failed'
|
|
6795
|
+
AND viable_dependency.retry_count <= viable_dependency.max_retries
|
|
6796
|
+
)
|
|
6797
|
+
)
|
|
6798
|
+
)
|
|
6799
|
+
UNION
|
|
6800
|
+
SELECT descendant.id
|
|
6801
|
+
FROM tasks AS descendant
|
|
6802
|
+
INNER JOIN blocked_task ON descendant.depends_on = blocked_task.id
|
|
6803
|
+
WHERE descendant.status IN ('pending', 'failed')
|
|
6804
|
+
)
|
|
6805
|
+
SELECT id FROM blocked_task
|
|
6806
|
+
)`).returning();
|
|
6807
|
+
return result.length;
|
|
6808
|
+
}
|
|
6809
|
+
static async rejectBlockedDependents(prerequisiteId) {
|
|
6506
6810
|
const result = await db.update(tasks2).set({
|
|
6507
|
-
status:
|
|
6811
|
+
status: "dead_letter",
|
|
6508
6812
|
finishedAt: /* @__PURE__ */ new Date(),
|
|
6509
|
-
|
|
6510
|
-
|
|
6511
|
-
|
|
6512
|
-
|
|
6513
|
-
current.retryBackoffMs ?? 3e4
|
|
6514
|
-
)
|
|
6515
|
-
}).where(and(...conditions)).returning();
|
|
6516
|
-
return result[0] || null;
|
|
6813
|
+
retryAfter: null,
|
|
6814
|
+
resultLog: `\u4F9D\u8D56\u4EFB\u52A1 #${prerequisiteId} \u5DF2\u8FDB\u5165\u4E0D\u53EF\u6062\u590D\u7EC8\u6001`
|
|
6815
|
+
}).where(blockedDependentsOf(prerequisiteId)).returning({ id: tasks2.id });
|
|
6816
|
+
return result.length;
|
|
6517
6817
|
}
|
|
6518
6818
|
static async markPendingForRetry(id, retryAfterMs, retryCount) {
|
|
6519
6819
|
const result = await db.update(tasks2).set({
|
|
@@ -6526,8 +6826,18 @@ var init_task_service = __esm({
|
|
|
6526
6826
|
return result[0] || null;
|
|
6527
6827
|
}
|
|
6528
6828
|
static async markDeadLetter(id, retryCount) {
|
|
6529
|
-
|
|
6530
|
-
|
|
6829
|
+
return db.transaction((tx) => {
|
|
6830
|
+
const finishedAt = /* @__PURE__ */ new Date();
|
|
6831
|
+
const task = tx.update(tasks2).set({ status: "dead_letter", finishedAt, retryCount }).where(eq(tasks2.id, id)).returning().get();
|
|
6832
|
+
if (!task) return null;
|
|
6833
|
+
tx.update(tasks2).set({
|
|
6834
|
+
status: "dead_letter",
|
|
6835
|
+
finishedAt,
|
|
6836
|
+
retryAfter: null,
|
|
6837
|
+
resultLog: `\u4F9D\u8D56\u4EFB\u52A1 #${id} \u5DF2\u8FDB\u5165\u4E0D\u53EF\u6062\u590D\u7EC8\u6001`
|
|
6838
|
+
}).where(blockedDependentsOf(id)).run();
|
|
6839
|
+
return task;
|
|
6840
|
+
}, { behavior: "immediate" });
|
|
6531
6841
|
}
|
|
6532
6842
|
static async resetRunningToPending(ids) {
|
|
6533
6843
|
if (ids.length === 0) return 0;
|
|
@@ -6570,41 +6880,91 @@ var init_task_service = __esm({
|
|
|
6570
6880
|
),
|
|
6571
6881
|
...this.buildScopeWhere(scope)
|
|
6572
6882
|
];
|
|
6573
|
-
|
|
6574
|
-
|
|
6575
|
-
|
|
6576
|
-
|
|
6577
|
-
|
|
6578
|
-
|
|
6883
|
+
return db.transaction((tx) => {
|
|
6884
|
+
const finishedAt = /* @__PURE__ */ new Date();
|
|
6885
|
+
const task = tx.update(tasks2).set({
|
|
6886
|
+
status: "cancelled",
|
|
6887
|
+
finishedAt,
|
|
6888
|
+
retryAfter: null
|
|
6889
|
+
}).where(and(...conditions)).returning().get();
|
|
6890
|
+
if (!task) return null;
|
|
6891
|
+
tx.update(tasks2).set({
|
|
6892
|
+
status: "dead_letter",
|
|
6893
|
+
finishedAt,
|
|
6894
|
+
retryAfter: null,
|
|
6895
|
+
resultLog: `\u4F9D\u8D56\u4EFB\u52A1 #${id} \u5DF2\u8FDB\u5165\u4E0D\u53EF\u6062\u590D\u7EC8\u6001`
|
|
6896
|
+
}).where(blockedDependentsOf(id)).run();
|
|
6897
|
+
return task;
|
|
6898
|
+
}, { behavior: "immediate" });
|
|
6579
6899
|
}
|
|
6580
6900
|
static async retry(id, scope = {}) {
|
|
6581
|
-
const current = await this.getById(id, scope);
|
|
6582
|
-
if (!current) return null;
|
|
6583
|
-
if (current.status !== "failed" && current.status !== "dead_letter") return null;
|
|
6584
|
-
const conditions = [eq(tasks2.id, id), ...this.buildScopeWhere(scope)];
|
|
6585
|
-
const result = await db.update(tasks2).set({
|
|
6586
|
-
status: "pending",
|
|
6587
|
-
startedAt: null,
|
|
6588
|
-
finishedAt: null,
|
|
6589
|
-
retryAfter: null,
|
|
6590
|
-
retryCount: 0
|
|
6591
|
-
}).where(and(...conditions)).returning();
|
|
6592
|
-
return result[0] || null;
|
|
6593
|
-
}
|
|
6594
|
-
static async retryBatch(batchId, scope = {}) {
|
|
6595
6901
|
const conditions = [
|
|
6596
|
-
eq(tasks2.
|
|
6902
|
+
eq(tasks2.id, id),
|
|
6597
6903
|
or(eq(tasks2.status, "failed"), eq(tasks2.status, "dead_letter")),
|
|
6598
6904
|
...this.buildScopeWhere(scope)
|
|
6599
6905
|
];
|
|
6600
|
-
|
|
6906
|
+
return db.transaction((tx) => tx.update(tasks2).set({
|
|
6601
6907
|
status: "pending",
|
|
6602
6908
|
startedAt: null,
|
|
6603
6909
|
finishedAt: null,
|
|
6604
6910
|
retryAfter: null,
|
|
6605
6911
|
retryCount: 0
|
|
6606
|
-
}).where(and(...conditions)).returning();
|
|
6607
|
-
|
|
6912
|
+
}).where(and(...conditions, hasViableDependency())).returning().get() ?? null, { behavior: "immediate" });
|
|
6913
|
+
}
|
|
6914
|
+
static async retryBatch(batchId, scope = {}) {
|
|
6915
|
+
const sqlite2 = getSqlite();
|
|
6916
|
+
const scopeFilter = scope.cwd === void 0 ? "" : "AND candidate.cwd = ?";
|
|
6917
|
+
const parameters = scope.cwd === void 0 ? [batchId] : [batchId, scope.cwd];
|
|
6918
|
+
return db.transaction(() => sqlite2.query(`
|
|
6919
|
+
WITH RECURSIVE
|
|
6920
|
+
candidate(id, cwd, depends_on) AS MATERIALIZED (
|
|
6921
|
+
SELECT candidate.id, candidate.cwd, candidate.depends_on
|
|
6922
|
+
FROM tasks AS candidate
|
|
6923
|
+
WHERE candidate.batch_id = ?
|
|
6924
|
+
AND candidate.status IN ('failed', 'dead_letter')
|
|
6925
|
+
${scopeFilter}
|
|
6926
|
+
),
|
|
6927
|
+
retryable(id, cwd, depends_on) AS (
|
|
6928
|
+
SELECT candidate.id,
|
|
6929
|
+
candidate.cwd,
|
|
6930
|
+
candidate.depends_on
|
|
6931
|
+
FROM candidate
|
|
6932
|
+
WHERE candidate.depends_on IS NULL
|
|
6933
|
+
OR (
|
|
6934
|
+
NOT EXISTS (
|
|
6935
|
+
SELECT 1 FROM candidate AS internal_parent
|
|
6936
|
+
WHERE internal_parent.id = candidate.depends_on
|
|
6937
|
+
)
|
|
6938
|
+
AND EXISTS (
|
|
6939
|
+
SELECT 1 FROM tasks AS external_parent
|
|
6940
|
+
WHERE external_parent.id = candidate.depends_on
|
|
6941
|
+
AND external_parent.cwd IS candidate.cwd
|
|
6942
|
+
AND (
|
|
6943
|
+
external_parent.status IN ('pending', 'running', 'done')
|
|
6944
|
+
OR (
|
|
6945
|
+
external_parent.status = 'failed'
|
|
6946
|
+
AND external_parent.retry_count <= external_parent.max_retries
|
|
6947
|
+
)
|
|
6948
|
+
)
|
|
6949
|
+
)
|
|
6950
|
+
)
|
|
6951
|
+
UNION
|
|
6952
|
+
SELECT child.id,
|
|
6953
|
+
child.cwd,
|
|
6954
|
+
child.depends_on
|
|
6955
|
+
FROM candidate AS child
|
|
6956
|
+
INNER JOIN retryable AS parent
|
|
6957
|
+
ON child.depends_on = parent.id
|
|
6958
|
+
AND child.cwd IS parent.cwd
|
|
6959
|
+
)
|
|
6960
|
+
UPDATE tasks
|
|
6961
|
+
SET status = 'pending',
|
|
6962
|
+
started_at = NULL,
|
|
6963
|
+
finished_at = NULL,
|
|
6964
|
+
retry_after = NULL,
|
|
6965
|
+
retry_count = 0
|
|
6966
|
+
WHERE id IN (SELECT id FROM retryable)
|
|
6967
|
+
`).run(...parameters).changes, { behavior: "immediate" });
|
|
6608
6968
|
}
|
|
6609
6969
|
static async getById(id, scope = {}) {
|
|
6610
6970
|
const conditions = [eq(tasks2.id, id), ...this.buildScopeWhere(scope)];
|
|
@@ -6669,36 +7029,360 @@ var init_task_service = __esm({
|
|
|
6669
7029
|
return stats;
|
|
6670
7030
|
}
|
|
6671
7031
|
static async delete(id, scope = {}) {
|
|
6672
|
-
const conditions = [
|
|
6673
|
-
|
|
6674
|
-
|
|
6675
|
-
|
|
7032
|
+
const conditions = [
|
|
7033
|
+
eq(tasks2.id, id),
|
|
7034
|
+
sql`${tasks2.status} <> 'running'`,
|
|
7035
|
+
sql`NOT EXISTS (
|
|
7036
|
+
SELECT 1 FROM ${taskRuns2}
|
|
7037
|
+
WHERE ${taskRuns2.taskId} = ${tasks2.id}
|
|
7038
|
+
AND ${taskRuns2.status} = 'running'
|
|
7039
|
+
)`,
|
|
7040
|
+
hasNoExecutableDependents(),
|
|
7041
|
+
...this.buildScopeWhere(scope)
|
|
7042
|
+
];
|
|
6676
7043
|
const result = await db.delete(tasks2).where(and(...conditions)).returning();
|
|
6677
|
-
|
|
7044
|
+
if (result.length > 0) {
|
|
7045
|
+
await db.delete(taskRuns2).where(eq(taskRuns2.taskId, id));
|
|
7046
|
+
return true;
|
|
7047
|
+
}
|
|
7048
|
+
if (!await this.getById(id, scope)) return false;
|
|
7049
|
+
const dependent = await db.select({ id: tasks2.id }).from(tasks2).where(and(
|
|
7050
|
+
eq(tasks2.dependsOn, id),
|
|
7051
|
+
sql`${tasks2.status} IN ('pending', 'running', 'failed', 'dead_letter')`
|
|
7052
|
+
)).orderBy(asc(tasks2.id)).limit(1);
|
|
7053
|
+
if (dependent[0]) {
|
|
7054
|
+
throw new TaskDeletionConflictError(
|
|
7055
|
+
`\u4EFB\u52A1 #${id} \u4ECD\u88AB\u53EF\u6267\u884C\u4EFB\u52A1 #${dependent[0].id} \u4F9D\u8D56\uFF0C\u8BF7\u5148\u5904\u7406\u4F9D\u8D56\u4EFB\u52A1`
|
|
7056
|
+
);
|
|
7057
|
+
}
|
|
7058
|
+
throw new TaskDeletionConflictError(
|
|
7059
|
+
`\u4EFB\u52A1 #${id} \u6B63\u5728\u8FD0\u884C\uFF0C\u8BF7\u5148\u53D6\u6D88\u4EFB\u52A1\u5E76\u7B49\u5F85\u6267\u884C\u8FDB\u7A0B\u9000\u51FA`
|
|
7060
|
+
);
|
|
6678
7061
|
}
|
|
6679
|
-
static async deleteOlderThan(retentionDays) {
|
|
7062
|
+
static async deleteOlderThan(retentionDays, shouldStop = () => false) {
|
|
6680
7063
|
const cutoffSec = Math.floor(Date.now() / 1e3) - retentionDays * 86400;
|
|
6681
|
-
const
|
|
6682
|
-
|
|
6683
|
-
|
|
6684
|
-
|
|
6685
|
-
|
|
6686
|
-
|
|
6687
|
-
|
|
6688
|
-
|
|
7064
|
+
const batchSize = 500;
|
|
7065
|
+
const sqlite2 = getSqlite();
|
|
7066
|
+
cleanupInvocation += 1;
|
|
7067
|
+
const candidateTable = `cleanup_candidates_${process.pid}_${cleanupInvocation}`;
|
|
7068
|
+
let deletedTotal = 0;
|
|
7069
|
+
sqlite2.exec(`
|
|
7070
|
+
CREATE TEMP TABLE ${candidateTable} (
|
|
7071
|
+
id INTEGER NOT NULL PRIMARY KEY
|
|
7072
|
+
) WITHOUT ROWID;
|
|
7073
|
+
`);
|
|
7074
|
+
try {
|
|
7075
|
+
let ceilingId = null;
|
|
7076
|
+
while (true) {
|
|
7077
|
+
if (shouldStop()) return deletedTotal;
|
|
7078
|
+
const batch = db.transaction(() => {
|
|
7079
|
+
sqlite2.query(`DELETE FROM ${candidateTable}`).run();
|
|
7080
|
+
const ceilingPredicate = ceilingId == null ? "" : "AND candidate.id < ?";
|
|
7081
|
+
const rawCandidateStatement = sqlite2.query(`
|
|
7082
|
+
INSERT INTO ${candidateTable}(id)
|
|
7083
|
+
SELECT candidate.id
|
|
7084
|
+
FROM tasks AS candidate NOT INDEXED
|
|
7085
|
+
WHERE candidate.status IN ('done', 'dead_letter', 'cancelled')
|
|
7086
|
+
AND candidate.finished_at IS NOT NULL
|
|
7087
|
+
AND candidate.finished_at < ?
|
|
7088
|
+
${ceilingPredicate}
|
|
7089
|
+
AND NOT EXISTS (
|
|
7090
|
+
SELECT 1 FROM task_runs AS active_run
|
|
7091
|
+
WHERE active_run.task_id = candidate.id
|
|
7092
|
+
AND active_run.status = 'running'
|
|
7093
|
+
)
|
|
7094
|
+
ORDER BY candidate.id DESC
|
|
7095
|
+
LIMIT ?
|
|
7096
|
+
`);
|
|
7097
|
+
const rawCount = ceilingId == null ? rawCandidateStatement.run(cutoffSec, batchSize).changes : rawCandidateStatement.run(cutoffSec, ceilingId, batchSize).changes;
|
|
7098
|
+
if (rawCount === 0) return { deleted: 0, nextCeilingId: null };
|
|
7099
|
+
const rawPage = sqlite2.query(`
|
|
7100
|
+
SELECT min(id) AS nextCeilingId FROM ${candidateTable}
|
|
7101
|
+
`).get();
|
|
7102
|
+
sqlite2.query(`
|
|
7103
|
+
DELETE FROM ${candidateTable}
|
|
7104
|
+
WHERE EXISTS (
|
|
7105
|
+
SELECT 1 FROM tasks AS anomalous
|
|
7106
|
+
WHERE anomalous.id = ${candidateTable}.id
|
|
7107
|
+
AND anomalous.depends_on IS NOT NULL
|
|
7108
|
+
AND anomalous.depends_on >= anomalous.id
|
|
7109
|
+
)
|
|
7110
|
+
`).run();
|
|
7111
|
+
while (true) {
|
|
7112
|
+
const pruned = sqlite2.query(`
|
|
7113
|
+
DELETE FROM ${candidateTable}
|
|
7114
|
+
WHERE EXISTS (
|
|
7115
|
+
SELECT 1 FROM tasks AS dependent_task
|
|
7116
|
+
WHERE dependent_task.depends_on = ${candidateTable}.id
|
|
7117
|
+
AND NOT EXISTS (
|
|
7118
|
+
SELECT 1 FROM ${candidateTable} AS selected_dependent
|
|
7119
|
+
WHERE selected_dependent.id = dependent_task.id
|
|
7120
|
+
)
|
|
7121
|
+
)
|
|
7122
|
+
`).run().changes;
|
|
7123
|
+
if (pruned === 0) break;
|
|
7124
|
+
}
|
|
7125
|
+
const selected = sqlite2.query(`
|
|
7126
|
+
SELECT count(*) AS count FROM ${candidateTable}
|
|
7127
|
+
`).get();
|
|
7128
|
+
if (selected.count === 0) {
|
|
7129
|
+
return { deleted: 0, nextCeilingId: rawPage.nextCeilingId };
|
|
7130
|
+
}
|
|
7131
|
+
sqlite2.query(`
|
|
7132
|
+
DELETE FROM tasks
|
|
7133
|
+
WHERE id IN (SELECT id FROM ${candidateTable})
|
|
7134
|
+
`).run();
|
|
7135
|
+
const remaining = sqlite2.query(`
|
|
7136
|
+
SELECT count(*) AS count
|
|
7137
|
+
FROM tasks
|
|
7138
|
+
WHERE id IN (SELECT id FROM ${candidateTable})
|
|
7139
|
+
`).get();
|
|
7140
|
+
if (remaining.count !== 0) {
|
|
7141
|
+
throw new Error("\u5386\u53F2\u6E05\u7406\u5019\u9009\u5728\u540C\u4E00\u5199\u4E8B\u52A1\u5185\u53D1\u751F\u6F02\u79FB\uFF0C\u5DF2\u56DE\u6EDA\u672C\u6279\u5220\u9664");
|
|
7142
|
+
}
|
|
7143
|
+
return { deleted: selected.count, nextCeilingId: rawPage.nextCeilingId };
|
|
7144
|
+
}, { behavior: "immediate" });
|
|
7145
|
+
if (batch.nextCeilingId == null) break;
|
|
7146
|
+
ceilingId = batch.nextCeilingId;
|
|
7147
|
+
deletedTotal += batch.deleted;
|
|
7148
|
+
await Bun.sleep(0);
|
|
7149
|
+
if (shouldStop()) return deletedTotal;
|
|
7150
|
+
}
|
|
7151
|
+
return deletedTotal;
|
|
7152
|
+
} finally {
|
|
7153
|
+
sqlite2.exec(`DROP TABLE IF EXISTS ${candidateTable}`);
|
|
7154
|
+
}
|
|
6689
7155
|
}
|
|
6690
7156
|
};
|
|
6691
7157
|
}
|
|
6692
7158
|
});
|
|
6693
7159
|
|
|
7160
|
+
// src/core/launch-protocol.ts
|
|
7161
|
+
function isLaunchIdentity(value) {
|
|
7162
|
+
return value != null && /^gateway-[1-9]\d*:launch:[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(value);
|
|
7163
|
+
}
|
|
7164
|
+
function isMatchingDrainProof(message, launchIdentity) {
|
|
7165
|
+
if (typeof message !== "object" || message == null) return false;
|
|
7166
|
+
const candidate = message;
|
|
7167
|
+
return candidate.type === DRAIN_PROOF_MESSAGE_TYPE && candidate.identity === launchIdentity;
|
|
7168
|
+
}
|
|
7169
|
+
var LEGACY_GUARDIAN_LAUNCH_PROTOCOL, TOKEN_GUARDIAN_LAUNCH_PROTOCOL, LAUNCH_IDENTITY_ARGUMENT, DRAIN_PROOF_MESSAGE_TYPE, MANAGED_RUN_ENV, MANAGED_RUN_ENV_VALUE;
|
|
7170
|
+
var init_launch_protocol = __esm({
|
|
7171
|
+
"src/core/launch-protocol.ts"() {
|
|
7172
|
+
"use strict";
|
|
7173
|
+
LEGACY_GUARDIAN_LAUNCH_PROTOCOL = "gated-v2-guardian";
|
|
7174
|
+
TOKEN_GUARDIAN_LAUNCH_PROTOCOL = "gated-v3-token-guardian";
|
|
7175
|
+
LAUNCH_IDENTITY_ARGUMENT = "--supertask-launch-identity";
|
|
7176
|
+
DRAIN_PROOF_MESSAGE_TYPE = "supertask-drained";
|
|
7177
|
+
MANAGED_RUN_ENV = "SUPERTASK_MANAGED_RUN";
|
|
7178
|
+
MANAGED_RUN_ENV_VALUE = "1";
|
|
7179
|
+
}
|
|
7180
|
+
});
|
|
7181
|
+
|
|
7182
|
+
// src/core/process-control.ts
|
|
7183
|
+
import { spawnSync } from "child_process";
|
|
7184
|
+
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
7185
|
+
import { basename, dirname as dirname2, resolve } from "path";
|
|
7186
|
+
function isSafePid(pid) {
|
|
7187
|
+
return Number.isInteger(pid) && pid > 1 && pid !== process.pid;
|
|
7188
|
+
}
|
|
7189
|
+
function absentLeaderResult(pid) {
|
|
7190
|
+
return inspectSpawnedProcessTreePresence(pid) === "not-running" ? "not-running" : "inspect-failed";
|
|
7191
|
+
}
|
|
7192
|
+
function isProcessAlive(pid) {
|
|
7193
|
+
if (!Number.isInteger(pid) || pid <= 0) return false;
|
|
7194
|
+
try {
|
|
7195
|
+
process.kill(pid, 0);
|
|
7196
|
+
return true;
|
|
7197
|
+
} catch (error) {
|
|
7198
|
+
return error instanceof Error && "code" in error && error.code === "EPERM";
|
|
7199
|
+
}
|
|
7200
|
+
}
|
|
7201
|
+
function isSpawnedProcessTreeAlive(pid) {
|
|
7202
|
+
return inspectSpawnedProcessTreePresence(pid) === "running";
|
|
7203
|
+
}
|
|
7204
|
+
function inspectSpawnedProcessTreePresence(pid) {
|
|
7205
|
+
if (!Number.isInteger(pid) || pid <= 0) return "unknown";
|
|
7206
|
+
if (process.platform === "win32") {
|
|
7207
|
+
const processIds = inspectWindowsProcessTree(pid);
|
|
7208
|
+
if (processIds == null) return "unknown";
|
|
7209
|
+
return processIds.length > 0 ? "running" : "not-running";
|
|
7210
|
+
}
|
|
7211
|
+
try {
|
|
7212
|
+
process.kill(-pid, 0);
|
|
7213
|
+
return "running";
|
|
7214
|
+
} catch (error) {
|
|
7215
|
+
if (!(error instanceof Error) || !("code" in error)) return "unknown";
|
|
7216
|
+
if (error.code === "EPERM") return "running";
|
|
7217
|
+
if (error.code === "ESRCH") return "not-running";
|
|
7218
|
+
return "unknown";
|
|
7219
|
+
}
|
|
7220
|
+
}
|
|
7221
|
+
function inspectUnixProcess(pid) {
|
|
7222
|
+
const result = spawnSync("ps", ["-o", "pgid=", "-o", "command=", "-p", String(pid)], {
|
|
7223
|
+
encoding: "utf8",
|
|
7224
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
7225
|
+
timeout: OS_COMMAND_TIMEOUT_MS,
|
|
7226
|
+
killSignal: "SIGKILL"
|
|
7227
|
+
});
|
|
7228
|
+
if (result.status !== 0) return null;
|
|
7229
|
+
const match2 = result.stdout.trim().match(/^(\d+)\s+(.+)$/s);
|
|
7230
|
+
if (!match2) return null;
|
|
7231
|
+
return { processGroupId: Number(match2[1]), command: match2[2] };
|
|
7232
|
+
}
|
|
7233
|
+
function inspectWindowsCommand(pid) {
|
|
7234
|
+
const script = `(Get-CimInstance Win32_Process -Filter "ProcessId = ${pid}").CommandLine`;
|
|
7235
|
+
const result = spawnSync(
|
|
7236
|
+
"powershell.exe",
|
|
7237
|
+
["-NoProfile", "-NonInteractive", "-Command", script],
|
|
7238
|
+
{
|
|
7239
|
+
encoding: "utf8",
|
|
7240
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
7241
|
+
timeout: OS_COMMAND_TIMEOUT_MS,
|
|
7242
|
+
killSignal: "SIGKILL"
|
|
7243
|
+
}
|
|
7244
|
+
);
|
|
7245
|
+
if (result.status !== 0) return null;
|
|
7246
|
+
return result.stdout.trim() || null;
|
|
7247
|
+
}
|
|
7248
|
+
function inspectWindowsProcessTree(rootPid) {
|
|
7249
|
+
const script = `$root=${rootPid}; $all=@(Get-CimInstance Win32_Process | Select-Object ProcessId,ParentProcessId); $ids=New-Object 'System.Collections.Generic.HashSet[int]'; [void]$ids.Add($root); do { $added=$false; foreach($p in $all) { if($ids.Contains([int]$p.ParentProcessId) -and $ids.Add([int]$p.ProcessId)) { $added=$true } } } while($added); $all | Where-Object { $ids.Contains([int]$_.ProcessId) } | Select-Object -ExpandProperty ProcessId`;
|
|
7250
|
+
const result = spawnSync(
|
|
7251
|
+
"powershell.exe",
|
|
7252
|
+
["-NoProfile", "-NonInteractive", "-Command", script],
|
|
7253
|
+
{
|
|
7254
|
+
encoding: "utf8",
|
|
7255
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
7256
|
+
timeout: OS_COMMAND_TIMEOUT_MS,
|
|
7257
|
+
killSignal: "SIGKILL"
|
|
7258
|
+
}
|
|
7259
|
+
);
|
|
7260
|
+
if (result.status !== 0) return null;
|
|
7261
|
+
return result.stdout.split(/\s+/).filter(Boolean).map(Number).filter((pid) => Number.isInteger(pid) && pid > 0);
|
|
7262
|
+
}
|
|
7263
|
+
function executableName(value) {
|
|
7264
|
+
return basename(value).trim().toLowerCase().replace(/\.(?:exe|cmd|bat)$/i, "");
|
|
7265
|
+
}
|
|
7266
|
+
function commandTokens(command) {
|
|
7267
|
+
return command.trim().split(/\s+/).map((token) => token.replace(/^["']|["']$/g, ""));
|
|
7268
|
+
}
|
|
7269
|
+
function openCodeArgsMatch(args) {
|
|
7270
|
+
const agentIndex = args.indexOf("--agent");
|
|
7271
|
+
const formatIndex = args.indexOf("--format");
|
|
7272
|
+
return args[0] === "run" && agentIndex >= 0 && Boolean(args[agentIndex + 1]) && formatIndex >= 0 && args[formatIndex + 1] === "json";
|
|
7273
|
+
}
|
|
7274
|
+
function guardianLauncherPath() {
|
|
7275
|
+
const extension = import.meta.url.endsWith(".ts") ? "ts" : "js";
|
|
7276
|
+
return resolve(dirname2(fileURLToPath2(import.meta.url)), `../worker/launcher.${extension}`).replaceAll("\\", "/");
|
|
7277
|
+
}
|
|
7278
|
+
function commandMatches(command, expectedExecutable, launchProtocol, expectedLaunchIdentity) {
|
|
7279
|
+
const expectedName = executableName(expectedExecutable);
|
|
7280
|
+
if (!expectedName) return false;
|
|
7281
|
+
const tokens = commandTokens(command);
|
|
7282
|
+
if (launchProtocol === TOKEN_GUARDIAN_LAUNCH_PROTOCOL) {
|
|
7283
|
+
if (!isLaunchIdentity(expectedLaunchIdentity)) return false;
|
|
7284
|
+
const expectedLauncher = guardianLauncherPath();
|
|
7285
|
+
const launcherIndex = tokens.findIndex((token, index2) => index2 > 0 && index2 <= 3 && token.replaceAll("\\", "/") === expectedLauncher);
|
|
7286
|
+
if (launcherIndex < 0) return false;
|
|
7287
|
+
const runtimeName = executableName(process.execPath);
|
|
7288
|
+
const hasRuntime = tokens.slice(0, launcherIndex).some((token) => executableName(token) === runtimeName || executableName(token) === "bun");
|
|
7289
|
+
const identityArgumentIndex = launcherIndex + 1;
|
|
7290
|
+
const launchIdentityIndex = launcherIndex + 2;
|
|
7291
|
+
const executableIndex2 = launcherIndex + 3;
|
|
7292
|
+
return hasRuntime && tokens[identityArgumentIndex] === LAUNCH_IDENTITY_ARGUMENT && tokens[launchIdentityIndex] === expectedLaunchIdentity && executableName(tokens[executableIndex2] ?? "") === expectedName && openCodeArgsMatch(tokens.slice(executableIndex2 + 1));
|
|
7293
|
+
}
|
|
7294
|
+
if (launchProtocol != null) return false;
|
|
7295
|
+
const executableIndex = tokens.findIndex((token, index2) => index2 <= 3 && executableName(token) === expectedName);
|
|
7296
|
+
if (executableIndex < 0) return false;
|
|
7297
|
+
return openCodeArgsMatch(tokens.slice(executableIndex + 1, executableIndex + 12));
|
|
7298
|
+
}
|
|
7299
|
+
function identifyGatewayProcess(pid) {
|
|
7300
|
+
if (!isProcessAlive(pid)) return "not-running";
|
|
7301
|
+
const command = process.platform === "win32" ? inspectWindowsCommand(pid) : inspectUnixProcess(pid)?.command ?? null;
|
|
7302
|
+
if (!command) return "unknown";
|
|
7303
|
+
const tokens = command.trim().split(/\s+/).map((token) => token.replace(/^['"]|['"]$/g, "").replaceAll("\\", "/"));
|
|
7304
|
+
const runtimeName = executableName(process.execPath);
|
|
7305
|
+
const hasRuntime = tokens.slice(0, 4).some((token) => executableName(token) === runtimeName || executableName(token) === "bun");
|
|
7306
|
+
const hasGatewayEntry = tokens.some((token) => /(?:^|\/)gateway\/index\.(?:js|ts)$/.test(token));
|
|
7307
|
+
const cliEntryIndex = tokens.findIndex((token) => /(?:^|\/)cli\/index\.(?:js|ts)$/.test(token));
|
|
7308
|
+
const hasCliGatewayCommand = cliEntryIndex >= 0 && tokens.slice(cliEntryIndex + 1).includes("gateway");
|
|
7309
|
+
const supertaskIndex = tokens.findIndex((token, index2) => index2 <= 3 && executableName(token) === "supertask");
|
|
7310
|
+
const hasBinGatewayCommand = supertaskIndex >= 0 && tokens[supertaskIndex + 1] === "gateway";
|
|
7311
|
+
return hasRuntime && (hasGatewayEntry || hasCliGatewayCommand || hasBinGatewayCommand) ? "match" : "mismatch";
|
|
7312
|
+
}
|
|
7313
|
+
function signalRecordedProcessTreeWithResult(pid, signal, expectedExecutable, launchProtocol = null, expectedLaunchIdentity = null) {
|
|
7314
|
+
if (!isSafePid(pid)) return "identity-mismatch";
|
|
7315
|
+
if (!isProcessAlive(pid)) return absentLeaderResult(pid);
|
|
7316
|
+
if (process.platform === "win32") {
|
|
7317
|
+
const command = inspectWindowsCommand(pid);
|
|
7318
|
+
if (!command) return isProcessAlive(pid) ? "inspect-failed" : absentLeaderResult(pid);
|
|
7319
|
+
if (!commandMatches(
|
|
7320
|
+
command,
|
|
7321
|
+
expectedExecutable,
|
|
7322
|
+
launchProtocol,
|
|
7323
|
+
expectedLaunchIdentity
|
|
7324
|
+
)) {
|
|
7325
|
+
return "identity-mismatch";
|
|
7326
|
+
}
|
|
7327
|
+
const args = ["/PID", String(pid), "/T"];
|
|
7328
|
+
if (signal === "SIGKILL") args.push("/F");
|
|
7329
|
+
const status = spawnSync("taskkill", args, {
|
|
7330
|
+
stdio: "ignore",
|
|
7331
|
+
timeout: OS_COMMAND_TIMEOUT_MS,
|
|
7332
|
+
killSignal: "SIGKILL"
|
|
7333
|
+
}).status;
|
|
7334
|
+
if (status === 0) return "signalled";
|
|
7335
|
+
return isProcessAlive(pid) ? "signal-failed" : absentLeaderResult(pid);
|
|
7336
|
+
}
|
|
7337
|
+
const info = inspectUnixProcess(pid);
|
|
7338
|
+
if (!info) return isProcessAlive(pid) ? "inspect-failed" : absentLeaderResult(pid);
|
|
7339
|
+
if (!commandMatches(
|
|
7340
|
+
info.command,
|
|
7341
|
+
expectedExecutable,
|
|
7342
|
+
launchProtocol,
|
|
7343
|
+
expectedLaunchIdentity
|
|
7344
|
+
)) {
|
|
7345
|
+
return "identity-mismatch";
|
|
7346
|
+
}
|
|
7347
|
+
try {
|
|
7348
|
+
if (info.processGroupId === pid) process.kill(-pid, signal);
|
|
7349
|
+
else process.kill(pid, signal);
|
|
7350
|
+
return "signalled";
|
|
7351
|
+
} catch {
|
|
7352
|
+
return isProcessAlive(pid) ? "signal-failed" : absentLeaderResult(pid);
|
|
7353
|
+
}
|
|
7354
|
+
}
|
|
7355
|
+
async function waitForSpawnedProcessTreeExit(pid, timeoutMs = 5e3) {
|
|
7356
|
+
const deadline = Date.now() + timeoutMs;
|
|
7357
|
+
while (inspectSpawnedProcessTreePresence(pid) !== "not-running" && Date.now() < deadline) {
|
|
7358
|
+
await Bun.sleep(Math.min(50, deadline - Date.now()));
|
|
7359
|
+
}
|
|
7360
|
+
return inspectSpawnedProcessTreePresence(pid) === "not-running";
|
|
7361
|
+
}
|
|
7362
|
+
var OS_COMMAND_TIMEOUT_MS;
|
|
7363
|
+
var init_process_control = __esm({
|
|
7364
|
+
"src/core/process-control.ts"() {
|
|
7365
|
+
"use strict";
|
|
7366
|
+
init_launch_protocol();
|
|
7367
|
+
OS_COMMAND_TIMEOUT_MS = 2e3;
|
|
7368
|
+
}
|
|
7369
|
+
});
|
|
7370
|
+
|
|
6694
7371
|
// src/core/services/task-run.service.ts
|
|
6695
|
-
var taskRuns3, TaskRunService;
|
|
7372
|
+
var tasks3, taskRuns3, LegacyRunAbandonConflictError, TaskRunService;
|
|
6696
7373
|
var init_task_run_service = __esm({
|
|
6697
7374
|
"src/core/services/task-run.service.ts"() {
|
|
6698
7375
|
"use strict";
|
|
6699
7376
|
init_db2();
|
|
6700
7377
|
init_drizzle_orm();
|
|
6701
|
-
(
|
|
7378
|
+
init_process_control();
|
|
7379
|
+
({ tasks: tasks3, taskRuns: taskRuns3 } = schema_exports);
|
|
7380
|
+
LegacyRunAbandonConflictError = class extends Error {
|
|
7381
|
+
constructor(message) {
|
|
7382
|
+
super(message);
|
|
7383
|
+
this.name = "LegacyRunAbandonConflictError";
|
|
7384
|
+
}
|
|
7385
|
+
};
|
|
6702
7386
|
TaskRunService = class {
|
|
6703
7387
|
static async create(data) {
|
|
6704
7388
|
const result = await db.insert(taskRuns3).values(data).returning();
|
|
@@ -6728,12 +7412,12 @@ var init_task_run_service = __esm({
|
|
|
6728
7412
|
const result = await db.update(taskRuns3).set({ heartbeatAt: Date.now() }).where(and(eq(taskRuns3.id, id), eq(taskRuns3.status, "running"))).returning();
|
|
6729
7413
|
return result[0] || null;
|
|
6730
7414
|
}
|
|
6731
|
-
static async updatePid(id, workerPid, childPid) {
|
|
7415
|
+
static async updatePid(id, workerPid, childPid, lockedBy = `gateway-${process.pid}`) {
|
|
6732
7416
|
const result = await db.update(taskRuns3).set({
|
|
6733
7417
|
workerPid,
|
|
6734
7418
|
childPid,
|
|
6735
7419
|
lockedAt: Date.now(),
|
|
6736
|
-
lockedBy
|
|
7420
|
+
lockedBy
|
|
6737
7421
|
}).where(and(eq(taskRuns3.id, id), eq(taskRuns3.status, "running"))).returning();
|
|
6738
7422
|
return result[0] || null;
|
|
6739
7423
|
}
|
|
@@ -6761,39 +7445,108 @@ var init_task_run_service = __esm({
|
|
|
6761
7445
|
}
|
|
6762
7446
|
static async getStaleRuns(heartbeatTimeoutMs) {
|
|
6763
7447
|
const cutoffMs = Date.now() - heartbeatTimeoutMs;
|
|
6764
|
-
const cutoffSec = Math.floor(cutoffMs / 1e3);
|
|
6765
7448
|
const { tasks: tasksTable } = schema_exports;
|
|
6766
7449
|
const result = await db.select({
|
|
6767
7450
|
runId: taskRuns3.id,
|
|
6768
7451
|
taskId: taskRuns3.taskId,
|
|
6769
7452
|
childPid: taskRuns3.childPid,
|
|
7453
|
+
workerPid: taskRuns3.workerPid,
|
|
7454
|
+
launchProtocol: taskRuns3.launchProtocol,
|
|
7455
|
+
lockedBy: taskRuns3.lockedBy,
|
|
7456
|
+
startedAt: taskRuns3.startedAt,
|
|
7457
|
+
heartbeatAt: taskRuns3.heartbeatAt,
|
|
6770
7458
|
taskRetryCount: tasksTable.retryCount,
|
|
6771
7459
|
taskMaxRetries: tasksTable.maxRetries,
|
|
6772
|
-
taskRetryBackoffMs: tasksTable.retryBackoffMs
|
|
6773
|
-
|
|
6774
|
-
|
|
6775
|
-
|
|
6776
|
-
|
|
6777
|
-
|
|
6778
|
-
|
|
6779
|
-
|
|
6780
|
-
|
|
6781
|
-
and(
|
|
6782
|
-
sql`${taskRuns3.heartbeatAt} IS NOT NULL`,
|
|
6783
|
-
sql`${taskRuns3.heartbeatAt} < ${cutoffMs}`
|
|
6784
|
-
)
|
|
6785
|
-
)
|
|
6786
|
-
)
|
|
6787
|
-
);
|
|
6788
|
-
return result.map((row) => ({
|
|
7460
|
+
taskRetryBackoffMs: tasksTable.retryBackoffMs,
|
|
7461
|
+
taskStatus: tasksTable.status,
|
|
7462
|
+
taskCwd: tasksTable.cwd
|
|
7463
|
+
}).from(taskRuns3).innerJoin(tasksTable, eq(taskRuns3.taskId, tasksTable.id)).where(eq(taskRuns3.status, "running"));
|
|
7464
|
+
return result.filter((row) => {
|
|
7465
|
+
const heartbeatExpired = row.heartbeatAt == null ? row.startedAt == null || row.startedAt.getTime() < cutoffMs : row.heartbeatAt < cutoffMs;
|
|
7466
|
+
const ownerExited = row.workerPid != null && row.workerPid > 0 && !isProcessAlive(row.workerPid);
|
|
7467
|
+
return heartbeatExpired || ownerExited;
|
|
7468
|
+
}).map((row) => ({
|
|
6789
7469
|
runId: row.runId,
|
|
6790
7470
|
taskId: row.taskId,
|
|
6791
7471
|
childPid: row.childPid,
|
|
7472
|
+
workerPid: row.workerPid,
|
|
7473
|
+
launchProtocol: row.launchProtocol,
|
|
7474
|
+
lockedBy: row.lockedBy,
|
|
6792
7475
|
taskRetryCount: row.taskRetryCount ?? 0,
|
|
6793
7476
|
taskMaxRetries: row.taskMaxRetries ?? 3,
|
|
6794
|
-
taskRetryBackoffMs: row.taskRetryBackoffMs ?? 3e4
|
|
7477
|
+
taskRetryBackoffMs: row.taskRetryBackoffMs ?? 3e4,
|
|
7478
|
+
taskStatus: row.taskStatus,
|
|
7479
|
+
taskCwd: row.taskCwd,
|
|
7480
|
+
ownerAlive: row.workerPid != null && row.workerPid > 0 && isProcessAlive(row.workerPid)
|
|
7481
|
+
}));
|
|
7482
|
+
}
|
|
7483
|
+
static async listLegacyQuarantinedRuns(heartbeatTimeoutMs = 0) {
|
|
7484
|
+
const staleRuns = await this.getStaleRuns(heartbeatTimeoutMs);
|
|
7485
|
+
return staleRuns.filter(
|
|
7486
|
+
(row) => row.launchProtocol == null && row.childPid == null
|
|
7487
|
+
).map((row) => ({
|
|
7488
|
+
runId: row.runId,
|
|
7489
|
+
taskId: row.taskId,
|
|
7490
|
+
taskStatus: row.taskStatus,
|
|
7491
|
+
taskCwd: row.taskCwd,
|
|
7492
|
+
workerPid: row.workerPid,
|
|
7493
|
+
ownerAlive: row.ownerAlive
|
|
6795
7494
|
}));
|
|
6796
7495
|
}
|
|
7496
|
+
static async abandonLegacyRun(runId) {
|
|
7497
|
+
return db.transaction((tx) => {
|
|
7498
|
+
const current = tx.select({
|
|
7499
|
+
runId: taskRuns3.id,
|
|
7500
|
+
taskId: taskRuns3.taskId,
|
|
7501
|
+
runStatus: taskRuns3.status,
|
|
7502
|
+
taskStatus: tasks3.status,
|
|
7503
|
+
workerPid: taskRuns3.workerPid,
|
|
7504
|
+
childPid: taskRuns3.childPid,
|
|
7505
|
+
launchProtocol: taskRuns3.launchProtocol,
|
|
7506
|
+
log: taskRuns3.log
|
|
7507
|
+
}).from(taskRuns3).innerJoin(tasks3, eq(taskRuns3.taskId, tasks3.id)).where(eq(taskRuns3.id, runId)).get();
|
|
7508
|
+
if (!current) return null;
|
|
7509
|
+
if (current.runStatus !== "running") {
|
|
7510
|
+
throw new LegacyRunAbandonConflictError(`run #${runId} \u5DF2\u4E0D\u662F running \u72B6\u6001`);
|
|
7511
|
+
}
|
|
7512
|
+
if (current.launchProtocol != null) {
|
|
7513
|
+
throw new LegacyRunAbandonConflictError(
|
|
7514
|
+
`run #${runId} \u4F7F\u7528\u672A\u77E5\u6216\u53D7\u7BA1\u534F\u8BAE ${current.launchProtocol}\uFF0C\u7981\u6B62\u4EBA\u5DE5 abandon`
|
|
7515
|
+
);
|
|
7516
|
+
}
|
|
7517
|
+
if (current.childPid != null) {
|
|
7518
|
+
throw new LegacyRunAbandonConflictError(
|
|
7519
|
+
`run #${runId} \u5DF2\u8BB0\u5F55 child PID ${current.childPid}\uFF0C\u5FC5\u987B\u7531 Worker/Watchdog \u786E\u8BA4\u8FDB\u7A0B\u6811\u9000\u51FA`
|
|
7520
|
+
);
|
|
7521
|
+
}
|
|
7522
|
+
if (current.workerPid != null && isProcessAlive(current.workerPid)) {
|
|
7523
|
+
throw new LegacyRunAbandonConflictError(
|
|
7524
|
+
`run #${runId} \u7684 owner PID ${current.workerPid} \u4ECD\u5B58\u6D3B`
|
|
7525
|
+
);
|
|
7526
|
+
}
|
|
7527
|
+
if (current.taskStatus !== "cancelled") {
|
|
7528
|
+
throw new LegacyRunAbandonConflictError(
|
|
7529
|
+
`\u4EFB\u52A1 #${current.taskId} \u5FC5\u987B\u5148\u53D6\u6D88\u5E76\u4FDD\u6301 cancelled \u72B6\u6001`
|
|
7530
|
+
);
|
|
7531
|
+
}
|
|
7532
|
+
const note = "\u64CD\u4F5C\u5458\u5DF2\u786E\u8BA4\u65E7\u7248\u65E0 PID \u6267\u884C\u4E0D\u5B58\u5728\uFF0C\u5E76\u901A\u8FC7 run abandon \u5173\u95ED\u9694\u79BB\u8BB0\u5F55";
|
|
7533
|
+
const updated = tx.update(taskRuns3).set({
|
|
7534
|
+
status: "failed",
|
|
7535
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
7536
|
+
log: current.log ? `${current.log}
|
|
7537
|
+
${note}` : note
|
|
7538
|
+
}).where(and(eq(taskRuns3.id, runId), eq(taskRuns3.status, "running"))).returning({ id: taskRuns3.id }).get();
|
|
7539
|
+
if (!updated) {
|
|
7540
|
+
throw new LegacyRunAbandonConflictError(`run #${runId} \u72B6\u6001\u5DF2\u88AB\u5176\u4ED6\u64CD\u4F5C\u6539\u53D8`);
|
|
7541
|
+
}
|
|
7542
|
+
return {
|
|
7543
|
+
runId,
|
|
7544
|
+
taskId: current.taskId,
|
|
7545
|
+
runStatus: "failed",
|
|
7546
|
+
taskStatus: "cancelled"
|
|
7547
|
+
};
|
|
7548
|
+
}, { behavior: "immediate" });
|
|
7549
|
+
}
|
|
6797
7550
|
static async getRunningRunByTaskId(taskId) {
|
|
6798
7551
|
const result = await db.select().from(taskRuns3).where(and(eq(taskRuns3.taskId, taskId), eq(taskRuns3.status, "running"))).orderBy(desc(taskRuns3.startedAt), desc(taskRuns3.id)).limit(1);
|
|
6799
7552
|
return result[0] || null;
|
|
@@ -6819,23 +7572,63 @@ function initializeGatewayHealth(config) {
|
|
|
6819
7572
|
lastActivityAt: {
|
|
6820
7573
|
worker: now,
|
|
6821
7574
|
scheduler: now,
|
|
6822
|
-
watchdog: now
|
|
7575
|
+
watchdog: now,
|
|
7576
|
+
watchdogCleanup: now
|
|
7577
|
+
},
|
|
7578
|
+
lastSuccessAt: {
|
|
7579
|
+
worker: now,
|
|
7580
|
+
scheduler: now,
|
|
7581
|
+
watchdog: now,
|
|
7582
|
+
watchdogCleanup: now
|
|
7583
|
+
},
|
|
7584
|
+
consecutiveFailures: {
|
|
7585
|
+
worker: 0,
|
|
7586
|
+
scheduler: 0,
|
|
7587
|
+
watchdog: 0,
|
|
7588
|
+
watchdogCleanup: 0
|
|
7589
|
+
},
|
|
7590
|
+
lastError: {
|
|
7591
|
+
worker: null,
|
|
7592
|
+
scheduler: null,
|
|
7593
|
+
watchdog: null,
|
|
7594
|
+
watchdogCleanup: null
|
|
6823
7595
|
}
|
|
6824
7596
|
};
|
|
6825
7597
|
}
|
|
6826
7598
|
function markGatewayActivity(component) {
|
|
6827
7599
|
if (state) state.lastActivityAt[component] = Date.now();
|
|
6828
7600
|
}
|
|
7601
|
+
function markGatewaySuccess(component) {
|
|
7602
|
+
if (!state) return;
|
|
7603
|
+
const now = Date.now();
|
|
7604
|
+
state.lastActivityAt[component] = now;
|
|
7605
|
+
state.lastSuccessAt[component] = now;
|
|
7606
|
+
state.consecutiveFailures[component] = 0;
|
|
7607
|
+
}
|
|
7608
|
+
function markGatewayFailure(component, error) {
|
|
7609
|
+
if (!state) return;
|
|
7610
|
+
const now = Date.now();
|
|
7611
|
+
state.lastActivityAt[component] = now;
|
|
7612
|
+
state.consecutiveFailures[component] += 1;
|
|
7613
|
+
state.lastError[component] = {
|
|
7614
|
+
at: now,
|
|
7615
|
+
message: error instanceof Error ? error.message : String(error)
|
|
7616
|
+
};
|
|
7617
|
+
}
|
|
6829
7618
|
function resetGatewayHealth() {
|
|
6830
7619
|
state = null;
|
|
6831
7620
|
}
|
|
6832
7621
|
function componentStatus(component, enabled, maxAgeMs, now) {
|
|
6833
7622
|
const lastActivityAt = state?.lastActivityAt[component] ?? null;
|
|
6834
7623
|
const ageMs = lastActivityAt == null ? null : Math.max(0, now - lastActivityAt);
|
|
7624
|
+
const consecutiveFailures = state?.consecutiveFailures[component] ?? 0;
|
|
6835
7625
|
return {
|
|
6836
7626
|
enabled,
|
|
6837
|
-
healthy: !enabled || ageMs != null && ageMs <= maxAgeMs,
|
|
7627
|
+
healthy: !enabled || ageMs != null && ageMs <= maxAgeMs && consecutiveFailures === 0,
|
|
6838
7628
|
lastActivityAt,
|
|
7629
|
+
lastSuccessAt: state?.lastSuccessAt[component] ?? null,
|
|
7630
|
+
consecutiveFailures,
|
|
7631
|
+
lastError: state?.lastError[component] ?? null,
|
|
6839
7632
|
ageMs,
|
|
6840
7633
|
maxAgeMs
|
|
6841
7634
|
};
|
|
@@ -6859,6 +7652,12 @@ function getGatewayHealth(now = Date.now()) {
|
|
|
6859
7652
|
Math.max((state?.config.watchdogCheckIntervalMs ?? 6e4) * 3, 5e3),
|
|
6860
7653
|
now
|
|
6861
7654
|
);
|
|
7655
|
+
const watchdogCleanup = componentStatus(
|
|
7656
|
+
"watchdogCleanup",
|
|
7657
|
+
true,
|
|
7658
|
+
Math.max((state?.config.watchdogCleanupIntervalMs ?? 864e5) * 2, 6e4),
|
|
7659
|
+
now
|
|
7660
|
+
);
|
|
6862
7661
|
let lock = { pid: null, heartbeatAt: null, readyAt: null, ageMs: null, healthy: false };
|
|
6863
7662
|
try {
|
|
6864
7663
|
const row = sqlite.prepare(
|
|
@@ -6876,13 +7675,13 @@ function getGatewayHealth(now = Date.now()) {
|
|
|
6876
7675
|
}
|
|
6877
7676
|
} catch {
|
|
6878
7677
|
}
|
|
6879
|
-
const healthy = state != null && lock.healthy && worker.healthy && scheduler.healthy && watchdog.healthy;
|
|
7678
|
+
const healthy = state != null && lock.healthy && worker.healthy && scheduler.healthy && watchdog.healthy && watchdogCleanup.healthy;
|
|
6880
7679
|
return {
|
|
6881
7680
|
status: healthy ? "ok" : "degraded",
|
|
6882
7681
|
pid: process.pid,
|
|
6883
7682
|
uptimeMs: state ? Math.max(0, now - state.startedAt) : 0,
|
|
6884
7683
|
lock,
|
|
6885
|
-
components: { worker, scheduler, watchdog }
|
|
7684
|
+
components: { worker, scheduler, watchdog, watchdogCleanup }
|
|
6886
7685
|
};
|
|
6887
7686
|
}
|
|
6888
7687
|
var LOCK_STALE_MS, state;
|
|
@@ -6895,80 +7694,6 @@ var init_health = __esm({
|
|
|
6895
7694
|
}
|
|
6896
7695
|
});
|
|
6897
7696
|
|
|
6898
|
-
// src/core/process-control.ts
|
|
6899
|
-
import { spawnSync } from "child_process";
|
|
6900
|
-
import { basename } from "path";
|
|
6901
|
-
function isSafePid(pid) {
|
|
6902
|
-
return Number.isInteger(pid) && pid > 1 && pid !== process.pid;
|
|
6903
|
-
}
|
|
6904
|
-
function isProcessAlive(pid) {
|
|
6905
|
-
if (!Number.isInteger(pid) || pid <= 0) return false;
|
|
6906
|
-
try {
|
|
6907
|
-
process.kill(pid, 0);
|
|
6908
|
-
return true;
|
|
6909
|
-
} catch (error) {
|
|
6910
|
-
return error instanceof Error && "code" in error && error.code === "EPERM";
|
|
6911
|
-
}
|
|
6912
|
-
}
|
|
6913
|
-
function inspectUnixProcess(pid) {
|
|
6914
|
-
const result = spawnSync("ps", ["-o", "pgid=", "-o", "command=", "-p", String(pid)], {
|
|
6915
|
-
encoding: "utf8",
|
|
6916
|
-
stdio: ["ignore", "pipe", "ignore"]
|
|
6917
|
-
});
|
|
6918
|
-
if (result.status !== 0) return null;
|
|
6919
|
-
const match2 = result.stdout.trim().match(/^(\d+)\s+(.+)$/s);
|
|
6920
|
-
if (!match2) return null;
|
|
6921
|
-
return { processGroupId: Number(match2[1]), command: match2[2] };
|
|
6922
|
-
}
|
|
6923
|
-
function commandMatches(command, expectedExecutable) {
|
|
6924
|
-
const expectedName = basename(expectedExecutable).trim().toLowerCase();
|
|
6925
|
-
if (!expectedName) return false;
|
|
6926
|
-
return command.toLowerCase().includes(expectedName);
|
|
6927
|
-
}
|
|
6928
|
-
function signalSpawnedProcessTree(pid, signal) {
|
|
6929
|
-
if (!isSafePid(pid)) return false;
|
|
6930
|
-
if (process.platform !== "win32") {
|
|
6931
|
-
try {
|
|
6932
|
-
process.kill(-pid, signal);
|
|
6933
|
-
return true;
|
|
6934
|
-
} catch {
|
|
6935
|
-
}
|
|
6936
|
-
}
|
|
6937
|
-
try {
|
|
6938
|
-
process.kill(pid, signal);
|
|
6939
|
-
return true;
|
|
6940
|
-
} catch {
|
|
6941
|
-
return false;
|
|
6942
|
-
}
|
|
6943
|
-
}
|
|
6944
|
-
function signalRecordedProcessTree(pid, signal, expectedExecutable) {
|
|
6945
|
-
if (!isSafePid(pid)) return false;
|
|
6946
|
-
if (process.platform === "win32") {
|
|
6947
|
-
const list = spawnSync("tasklist", ["/FI", `PID eq ${pid}`, "/FO", "CSV", "/NH"], {
|
|
6948
|
-
encoding: "utf8",
|
|
6949
|
-
stdio: ["ignore", "pipe", "ignore"]
|
|
6950
|
-
});
|
|
6951
|
-
if (list.status !== 0 || !commandMatches(list.stdout, expectedExecutable)) return false;
|
|
6952
|
-
const args = ["/PID", String(pid), "/T"];
|
|
6953
|
-
if (signal === "SIGKILL") args.push("/F");
|
|
6954
|
-
return spawnSync("taskkill", args, { stdio: "ignore" }).status === 0;
|
|
6955
|
-
}
|
|
6956
|
-
const info = inspectUnixProcess(pid);
|
|
6957
|
-
if (!info || !commandMatches(info.command, expectedExecutable)) return false;
|
|
6958
|
-
try {
|
|
6959
|
-
if (info.processGroupId === pid) process.kill(-pid, signal);
|
|
6960
|
-
else process.kill(pid, signal);
|
|
6961
|
-
return true;
|
|
6962
|
-
} catch {
|
|
6963
|
-
return false;
|
|
6964
|
-
}
|
|
6965
|
-
}
|
|
6966
|
-
var init_process_control = __esm({
|
|
6967
|
-
"src/core/process-control.ts"() {
|
|
6968
|
-
"use strict";
|
|
6969
|
-
}
|
|
6970
|
-
});
|
|
6971
|
-
|
|
6972
7697
|
// node_modules/cron-parser/dist/fields/types.js
|
|
6973
7698
|
var require_types = __commonJS({
|
|
6974
7699
|
"node_modules/cron-parser/dist/fields/types.js"(exports) {
|
|
@@ -10686,12 +11411,12 @@ var require_luxon = __commonJS({
|
|
|
10686
11411
|
if (!this.loc.equals(other.loc)) {
|
|
10687
11412
|
return false;
|
|
10688
11413
|
}
|
|
10689
|
-
function
|
|
11414
|
+
function eq2(v1, v2) {
|
|
10690
11415
|
if (v1 === void 0 || v1 === 0) return v2 === void 0 || v2 === 0;
|
|
10691
11416
|
return v1 === v2;
|
|
10692
11417
|
}
|
|
10693
11418
|
for (const u of orderedUnits$1) {
|
|
10694
|
-
if (!
|
|
11419
|
+
if (!eq2(this.values[u], other.values[u])) {
|
|
10695
11420
|
return false;
|
|
10696
11421
|
}
|
|
10697
11422
|
}
|
|
@@ -16035,8 +16760,8 @@ var require_CronFileParser = __commonJS({
|
|
|
16035
16760
|
* @throws If file cannot be read
|
|
16036
16761
|
*/
|
|
16037
16762
|
static parseFileSync(filePath) {
|
|
16038
|
-
const { readFileSync:
|
|
16039
|
-
const data =
|
|
16763
|
+
const { readFileSync: readFileSync4 } = __require("fs");
|
|
16764
|
+
const data = readFileSync4(filePath, "utf8");
|
|
16040
16765
|
return _CronFileParser.#parseContent(data);
|
|
16041
16766
|
}
|
|
16042
16767
|
/**
|
|
@@ -16174,19 +16899,17 @@ var init_task_template_service = __esm({
|
|
|
16174
16899
|
static async create(data) {
|
|
16175
16900
|
this.validate(data);
|
|
16176
16901
|
const now = Date.now();
|
|
16177
|
-
const
|
|
16178
|
-
|
|
16179
|
-
|
|
16180
|
-
|
|
16181
|
-
|
|
16182
|
-
|
|
16183
|
-
|
|
16184
|
-
|
|
16185
|
-
|
|
16186
|
-
|
|
16187
|
-
|
|
16188
|
-
}
|
|
16189
|
-
return tmpl;
|
|
16902
|
+
const nextRunAt = data.nextRunAt ?? this.calculateNextRunAt(
|
|
16903
|
+
data.scheduleType,
|
|
16904
|
+
{
|
|
16905
|
+
cronExpr: data.cronExpr ?? null,
|
|
16906
|
+
intervalMs: data.intervalMs ?? null,
|
|
16907
|
+
runAt: data.runAt ?? null
|
|
16908
|
+
},
|
|
16909
|
+
now
|
|
16910
|
+
);
|
|
16911
|
+
const result = await db.insert(taskTemplates2).values({ ...data, nextRunAt, createdAt: now, updatedAt: now }).returning();
|
|
16912
|
+
return result[0];
|
|
16190
16913
|
}
|
|
16191
16914
|
static validate(data) {
|
|
16192
16915
|
if (!data.name.trim()) throw new Error("name \u4E0D\u80FD\u4E3A\u7A7A");
|
|
@@ -16226,8 +16949,18 @@ var init_task_template_service = __esm({
|
|
|
16226
16949
|
return result[0] || null;
|
|
16227
16950
|
}
|
|
16228
16951
|
static async enable(id) {
|
|
16229
|
-
|
|
16230
|
-
|
|
16952
|
+
return db.transaction((tx) => {
|
|
16953
|
+
const template = tx.select().from(taskTemplates2).where(eq(taskTemplates2.id, id)).limit(1).get();
|
|
16954
|
+
if (!template) return null;
|
|
16955
|
+
const nextRunAt = template.nextRunAt ?? this.calculateNextRunAt(
|
|
16956
|
+
template.scheduleType,
|
|
16957
|
+
template
|
|
16958
|
+
);
|
|
16959
|
+
if (nextRunAt == null) {
|
|
16960
|
+
throw new Error(`\u6A21\u677F #${id} \u65E0\u6CD5\u8BA1\u7B97\u4E0B\u4E00\u6B21\u6267\u884C\u65F6\u95F4\uFF0C\u5DF2\u4FDD\u6301\u7981\u7528`);
|
|
16961
|
+
}
|
|
16962
|
+
return tx.update(taskTemplates2).set({ enabled: true, nextRunAt, updatedAt: Date.now() }).where(eq(taskTemplates2.id, id)).returning().get() ?? null;
|
|
16963
|
+
}, { behavior: "immediate" });
|
|
16231
16964
|
}
|
|
16232
16965
|
static async disable(id) {
|
|
16233
16966
|
const result = await db.update(taskTemplates2).set({ enabled: false, updatedAt: Date.now() }).where(eq(taskTemplates2.id, id)).returning();
|
|
@@ -16237,6 +16970,47 @@ var init_task_template_service = __esm({
|
|
|
16237
16970
|
const result = await db.delete(taskTemplates2).where(eq(taskTemplates2.id, id)).returning();
|
|
16238
16971
|
return result.length > 0;
|
|
16239
16972
|
}
|
|
16973
|
+
static async deleteExpiredDelayed(retentionDays, shouldStop = () => false) {
|
|
16974
|
+
const cutoffMs = Date.now() - retentionDays * 864e5;
|
|
16975
|
+
const batchSize = 500;
|
|
16976
|
+
let deletedTotal = 0;
|
|
16977
|
+
while (!shouldStop()) {
|
|
16978
|
+
const deleted = db.transaction((tx) => tx.delete(taskTemplates2).where(and(
|
|
16979
|
+
sql`${taskTemplates2.id} IN (
|
|
16980
|
+
SELECT candidate.id
|
|
16981
|
+
FROM task_templates AS candidate
|
|
16982
|
+
WHERE candidate.schedule_type = 'delayed'
|
|
16983
|
+
AND candidate.enabled = 0
|
|
16984
|
+
AND candidate.last_run_at IS NOT NULL
|
|
16985
|
+
AND candidate.last_run_at < ${cutoffMs}
|
|
16986
|
+
AND NOT EXISTS (
|
|
16987
|
+
SELECT 1 FROM tasks AS active_task
|
|
16988
|
+
WHERE active_task.template_id = candidate.id
|
|
16989
|
+
AND (
|
|
16990
|
+
active_task.status IN ('pending', 'running')
|
|
16991
|
+
OR (
|
|
16992
|
+
active_task.status = 'failed'
|
|
16993
|
+
AND active_task.retry_count <= active_task.max_retries
|
|
16994
|
+
)
|
|
16995
|
+
)
|
|
16996
|
+
)
|
|
16997
|
+
AND NOT EXISTS (
|
|
16998
|
+
SELECT 1
|
|
16999
|
+
FROM task_runs AS active_run
|
|
17000
|
+
INNER JOIN tasks AS run_task ON run_task.id = active_run.task_id
|
|
17001
|
+
WHERE run_task.template_id = candidate.id
|
|
17002
|
+
AND active_run.status = 'running'
|
|
17003
|
+
)
|
|
17004
|
+
ORDER BY candidate.last_run_at, candidate.id
|
|
17005
|
+
LIMIT ${batchSize}
|
|
17006
|
+
)`
|
|
17007
|
+
)).returning({ id: taskTemplates2.id }).all().length, { behavior: "immediate" });
|
|
17008
|
+
if (deleted === 0) break;
|
|
17009
|
+
deletedTotal += deleted;
|
|
17010
|
+
await Bun.sleep(0);
|
|
17011
|
+
}
|
|
17012
|
+
return deletedTotal;
|
|
17013
|
+
}
|
|
16240
17014
|
static calculateNextRunAt(scheduleType, template, afterMs) {
|
|
16241
17015
|
const base = afterMs ?? Date.now();
|
|
16242
17016
|
switch (scheduleType) {
|
|
@@ -16259,6 +17033,134 @@ var init_task_template_service = __esm({
|
|
|
16259
17033
|
}
|
|
16260
17034
|
});
|
|
16261
17035
|
|
|
17036
|
+
// src/gateway/scheduler/job-templates.ts
|
|
17037
|
+
async function cloneTaskFromTemplate(templateId) {
|
|
17038
|
+
return createTaskFromTemplate(templateId, { advanceSchedule: true });
|
|
17039
|
+
}
|
|
17040
|
+
async function triggerTaskFromTemplate(templateId) {
|
|
17041
|
+
return createTaskFromTemplate(templateId, {
|
|
17042
|
+
advanceSchedule: false,
|
|
17043
|
+
namePrefix: "[\u624B\u52A8\u89E6\u53D1] "
|
|
17044
|
+
});
|
|
17045
|
+
}
|
|
17046
|
+
function createTaskFromTemplate(templateId, options) {
|
|
17047
|
+
const nowMs = Date.now();
|
|
17048
|
+
return db.transaction((tx) => {
|
|
17049
|
+
const tmpl = tx.select().from(taskTemplates3).where(eq(taskTemplates3.id, templateId)).limit(1).get();
|
|
17050
|
+
if (!tmpl || options.advanceSchedule && !tmpl.enabled) return null;
|
|
17051
|
+
const activeTasks = tx.select({ count: sql`count(*)` }).from(schema_exports.tasks).where(and(
|
|
17052
|
+
eq(schema_exports.tasks.templateId, templateId),
|
|
17053
|
+
sql`${schema_exports.tasks.status} IN ('pending', 'running')`
|
|
17054
|
+
)).get();
|
|
17055
|
+
const retryableTasks = tx.select({ count: sql`count(*)` }).from(schema_exports.tasks).where(and(
|
|
17056
|
+
eq(schema_exports.tasks.templateId, templateId),
|
|
17057
|
+
eq(schema_exports.tasks.status, "failed"),
|
|
17058
|
+
sql`${schema_exports.tasks.retryCount} <= ${schema_exports.tasks.maxRetries}`
|
|
17059
|
+
)).get();
|
|
17060
|
+
const detachedRuns = tx.select({ count: sql`count(DISTINCT ${schema_exports.taskRuns.taskId})` }).from(schema_exports.taskRuns).innerJoin(schema_exports.tasks, eq(schema_exports.tasks.id, schema_exports.taskRuns.taskId)).where(and(
|
|
17061
|
+
eq(schema_exports.taskRuns.status, "running"),
|
|
17062
|
+
eq(schema_exports.tasks.templateId, templateId),
|
|
17063
|
+
sql`NOT (
|
|
17064
|
+
${schema_exports.tasks.status} IN ('pending', 'running')
|
|
17065
|
+
OR (
|
|
17066
|
+
${schema_exports.tasks.status} = 'failed'
|
|
17067
|
+
AND ${schema_exports.tasks.retryCount} <= ${schema_exports.tasks.maxRetries}
|
|
17068
|
+
)
|
|
17069
|
+
)`
|
|
17070
|
+
)).get();
|
|
17071
|
+
const activeCount = Number(activeTasks?.count ?? 0) + Number(retryableTasks?.count ?? 0) + Number(detachedRuns?.count ?? 0);
|
|
17072
|
+
if (activeCount >= (tmpl.maxInstances ?? 1)) {
|
|
17073
|
+
if (options.advanceSchedule && tmpl.scheduleType !== "delayed") {
|
|
17074
|
+
const nextRunAt2 = TaskTemplateService.calculateNextRunAt(
|
|
17075
|
+
tmpl.scheduleType,
|
|
17076
|
+
tmpl,
|
|
17077
|
+
nowMs
|
|
17078
|
+
);
|
|
17079
|
+
tx.update(taskTemplates3).set({ nextRunAt: nextRunAt2, updatedAt: nowMs }).where(and(eq(taskTemplates3.id, templateId), eq(taskTemplates3.enabled, true))).run();
|
|
17080
|
+
}
|
|
17081
|
+
return null;
|
|
17082
|
+
}
|
|
17083
|
+
const isDelayed = tmpl.scheduleType === "delayed";
|
|
17084
|
+
const nextRunAt = isDelayed ? null : TaskTemplateService.calculateNextRunAt(
|
|
17085
|
+
tmpl.scheduleType,
|
|
17086
|
+
tmpl,
|
|
17087
|
+
nowMs
|
|
17088
|
+
);
|
|
17089
|
+
const task = tx.insert(schema_exports.tasks).values({
|
|
17090
|
+
name: `${options.namePrefix ?? ""}${tmpl.name}`,
|
|
17091
|
+
agent: tmpl.agent,
|
|
17092
|
+
model: tmpl.model ?? "default",
|
|
17093
|
+
prompt: tmpl.prompt,
|
|
17094
|
+
cwd: tmpl.cwd ?? null,
|
|
17095
|
+
category: tmpl.category ?? "general",
|
|
17096
|
+
importance: tmpl.importance ?? 3,
|
|
17097
|
+
urgency: tmpl.urgency ?? 3,
|
|
17098
|
+
batchId: tmpl.batchId,
|
|
17099
|
+
maxRetries: tmpl.maxRetries ?? 3,
|
|
17100
|
+
retryBackoffMs: tmpl.retryBackoffMs ?? 3e4,
|
|
17101
|
+
timeoutMs: tmpl.timeoutMs,
|
|
17102
|
+
templateId: tmpl.id,
|
|
17103
|
+
scheduledAt: options.advanceSchedule ? tmpl.nextRunAt ?? nowMs : nowMs
|
|
17104
|
+
}).returning().get();
|
|
17105
|
+
if (options.advanceSchedule) {
|
|
17106
|
+
tx.update(taskTemplates3).set({
|
|
17107
|
+
lastRunAt: nowMs,
|
|
17108
|
+
nextRunAt,
|
|
17109
|
+
enabled: isDelayed ? false : tmpl.enabled,
|
|
17110
|
+
updatedAt: nowMs
|
|
17111
|
+
}).where(and(eq(taskTemplates3.id, templateId), eq(taskTemplates3.enabled, true))).run();
|
|
17112
|
+
}
|
|
17113
|
+
return task;
|
|
17114
|
+
}, { behavior: "immediate" });
|
|
17115
|
+
}
|
|
17116
|
+
async function getDueTemplates(cursor = null, cutoffNow = Date.now()) {
|
|
17117
|
+
return await db.select().from(taskTemplates3).where(
|
|
17118
|
+
and(
|
|
17119
|
+
eq(taskTemplates3.enabled, true),
|
|
17120
|
+
sql`${taskTemplates3.nextRunAt} IS NOT NULL`,
|
|
17121
|
+
sql`${taskTemplates3.nextRunAt} <= ${cutoffNow}`,
|
|
17122
|
+
cursor ? sql`(
|
|
17123
|
+
${taskTemplates3.nextRunAt} > ${cursor.nextRunAt}
|
|
17124
|
+
OR (
|
|
17125
|
+
${taskTemplates3.nextRunAt} = ${cursor.nextRunAt}
|
|
17126
|
+
AND ${taskTemplates3.id} > ${cursor.id}
|
|
17127
|
+
)
|
|
17128
|
+
)` : void 0
|
|
17129
|
+
)
|
|
17130
|
+
).orderBy(asc(taskTemplates3.nextRunAt), asc(taskTemplates3.id)).limit(DUE_TEMPLATE_BATCH_SIZE);
|
|
17131
|
+
}
|
|
17132
|
+
async function initializeNextRunAt(templateId) {
|
|
17133
|
+
return db.transaction((tx) => {
|
|
17134
|
+
const tmpl = tx.select().from(taskTemplates3).where(and(
|
|
17135
|
+
eq(taskTemplates3.id, templateId),
|
|
17136
|
+
eq(taskTemplates3.enabled, true),
|
|
17137
|
+
isNull(taskTemplates3.nextRunAt)
|
|
17138
|
+
)).limit(1).get();
|
|
17139
|
+
if (!tmpl) return null;
|
|
17140
|
+
const nextRunAt = TaskTemplateService.calculateNextRunAt(
|
|
17141
|
+
tmpl.scheduleType,
|
|
17142
|
+
tmpl
|
|
17143
|
+
);
|
|
17144
|
+
if (nextRunAt == null) return null;
|
|
17145
|
+
return tx.update(taskTemplates3).set({ nextRunAt, updatedAt: Date.now() }).where(and(
|
|
17146
|
+
eq(taskTemplates3.id, templateId),
|
|
17147
|
+
eq(taskTemplates3.enabled, true),
|
|
17148
|
+
isNull(taskTemplates3.nextRunAt)
|
|
17149
|
+
)).returning().get() ?? null;
|
|
17150
|
+
}, { behavior: "immediate" });
|
|
17151
|
+
}
|
|
17152
|
+
var taskTemplates3, DUE_TEMPLATE_BATCH_SIZE;
|
|
17153
|
+
var init_job_templates = __esm({
|
|
17154
|
+
"src/gateway/scheduler/job-templates.ts"() {
|
|
17155
|
+
"use strict";
|
|
17156
|
+
init_db2();
|
|
17157
|
+
init_drizzle_orm();
|
|
17158
|
+
init_task_template_service();
|
|
17159
|
+
({ taskTemplates: taskTemplates3 } = schema_exports);
|
|
17160
|
+
DUE_TEMPLATE_BATCH_SIZE = 100;
|
|
17161
|
+
}
|
|
17162
|
+
});
|
|
17163
|
+
|
|
16262
17164
|
// node_modules/hono/dist/compose.js
|
|
16263
17165
|
var compose;
|
|
16264
17166
|
var init_compose = __esm({
|
|
@@ -18595,12 +19497,10 @@ var init_html2 = __esm({
|
|
|
18595
19497
|
|
|
18596
19498
|
// src/core/services/database-maintenance.service.ts
|
|
18597
19499
|
import { Database as Database3 } from "bun:sqlite";
|
|
18598
|
-
import { randomUUID } from "crypto";
|
|
19500
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
18599
19501
|
import {
|
|
18600
19502
|
chmodSync,
|
|
18601
|
-
|
|
18602
|
-
copyFileSync,
|
|
18603
|
-
existsSync as existsSync3,
|
|
19503
|
+
existsSync as existsSync5,
|
|
18604
19504
|
mkdirSync as mkdirSync2,
|
|
18605
19505
|
renameSync,
|
|
18606
19506
|
statSync,
|
|
@@ -18608,12 +19508,12 @@ import {
|
|
|
18608
19508
|
writeFileSync
|
|
18609
19509
|
} from "fs";
|
|
18610
19510
|
import { tmpdir } from "os";
|
|
18611
|
-
import { basename as basename2, dirname as
|
|
19511
|
+
import { basename as basename2, dirname as dirname5, resolve as resolve2 } from "path";
|
|
18612
19512
|
function timestamp() {
|
|
18613
19513
|
return (/* @__PURE__ */ new Date()).toISOString().replace(/[-:]/g, "").replace(/\.\d{3}Z$/, "Z");
|
|
18614
19514
|
}
|
|
18615
19515
|
function normalizedPath(path) {
|
|
18616
|
-
return path === ":memory:" ? path :
|
|
19516
|
+
return path === ":memory:" ? path : resolve2(path);
|
|
18617
19517
|
}
|
|
18618
19518
|
function safeUnlink(path) {
|
|
18619
19519
|
try {
|
|
@@ -18626,6 +19526,12 @@ function safeUnlink(path) {
|
|
|
18626
19526
|
function errorMessage(error) {
|
|
18627
19527
|
return error instanceof Error ? error.message : String(error);
|
|
18628
19528
|
}
|
|
19529
|
+
function quoteIdentifier(value) {
|
|
19530
|
+
return `"${value.replaceAll('"', '""')}"`;
|
|
19531
|
+
}
|
|
19532
|
+
function columnDefinitionsMatch(left, right) {
|
|
19533
|
+
return left.type.trim().toUpperCase() === right.type.trim().toUpperCase() && left.notNull === right.notNull && left.defaultValue === right.defaultValue && left.primaryKeyOrder === right.primaryKeyOrder;
|
|
19534
|
+
}
|
|
18629
19535
|
var REQUIRED_TABLES, DatabaseMaintenanceConflictError, DatabaseMaintenanceService;
|
|
18630
19536
|
var init_database_maintenance_service = __esm({
|
|
18631
19537
|
"src/core/services/database-maintenance.service.ts"() {
|
|
@@ -18662,9 +19568,11 @@ var init_database_maintenance_service = __esm({
|
|
|
18662
19568
|
this.assertNoRunningWork(sqlite2);
|
|
18663
19569
|
const before = this.readCounts(sqlite2);
|
|
18664
19570
|
backup = this.writeSnapshot(sqlite2, this.createBackupPath("pre-clear"));
|
|
18665
|
-
|
|
18666
|
-
sqlite2.exec("
|
|
18667
|
-
|
|
19571
|
+
const businessTables = this.readRestoreTables(sqlite2, "main");
|
|
19572
|
+
sqlite2.exec("PRAGMA defer_foreign_keys = ON");
|
|
19573
|
+
for (const table of businessTables.values()) {
|
|
19574
|
+
sqlite2.exec(`DELETE FROM main.${quoteIdentifier(table.name)}`);
|
|
19575
|
+
}
|
|
18668
19576
|
sqlite2.exec("COMMIT");
|
|
18669
19577
|
return {
|
|
18670
19578
|
backupPath: backup.path,
|
|
@@ -18686,78 +19594,102 @@ var init_database_maintenance_service = __esm({
|
|
|
18686
19594
|
const livePath = normalizedPath(DB_FILE_PATH);
|
|
18687
19595
|
if (source === livePath) throw new Error("\u6062\u590D\u6765\u6E90\u4E0D\u80FD\u662F\u5F53\u524D\u6570\u636E\u5E93\u6587\u4EF6");
|
|
18688
19596
|
const current = getSqlite();
|
|
18689
|
-
|
|
18690
|
-
this.assertNoRunningWork(current);
|
|
18691
|
-
if (!existsSync3(source) || !statSync(source).isFile()) {
|
|
18692
|
-
throw new Error(`\u5907\u4EFD\u6587\u4EF6\u4E0D\u5B58\u5728\uFF1A${source}`);
|
|
18693
|
-
}
|
|
18694
|
-
let sourceCheck;
|
|
18695
|
-
try {
|
|
18696
|
-
sourceCheck = this.inspectFile(source);
|
|
18697
|
-
} catch (error) {
|
|
18698
|
-
throw new Error(`\u5907\u4EFD\u6587\u4EF6\u65E0\u6548\uFF1A${errorMessage(error)}`);
|
|
18699
|
-
}
|
|
18700
|
-
if (!sourceCheck.ok) {
|
|
18701
|
-
throw new Error(`\u5907\u4EFD\u6587\u4EF6\u6821\u9A8C\u5931\u8D25\uFF1A${sourceCheck.integrityMessages.join("; ") || "schema/foreign key error"}`);
|
|
18702
|
-
}
|
|
18703
|
-
const stagePath = `${livePath}.restore-${process.pid}-${randomUUID()}.tmp`;
|
|
18704
|
-
const rollbackPath = `${livePath}.rollback-${process.pid}-${randomUUID()}.tmp`;
|
|
19597
|
+
const stagePath = `${livePath}.restore-${process.pid}-${randomUUID2()}.tmp`;
|
|
18705
19598
|
let safetyBackup = null;
|
|
18706
|
-
let
|
|
19599
|
+
let attached = false;
|
|
19600
|
+
let committed = false;
|
|
18707
19601
|
try {
|
|
18708
|
-
|
|
19602
|
+
current.exec("BEGIN EXCLUSIVE");
|
|
19603
|
+
this.assertGatewaySafe(current, false);
|
|
19604
|
+
this.assertNoRunningWork(current);
|
|
19605
|
+
if (!existsSync5(source)) {
|
|
19606
|
+
throw new Error(`\u5907\u4EFD\u6587\u4EF6\u4E0D\u5B58\u5728\uFF1A${source}`);
|
|
19607
|
+
}
|
|
19608
|
+
const sourceStat = statSync(source);
|
|
19609
|
+
if (!sourceStat.isFile()) throw new Error(`\u5907\u4EFD\u8DEF\u5F84\u4E0D\u662F\u6587\u4EF6\uFF1A${source}`);
|
|
19610
|
+
const liveStat = statSync(livePath);
|
|
19611
|
+
if (sourceStat.dev === liveStat.dev && sourceStat.ino === liveStat.ino) {
|
|
19612
|
+
throw new Error("\u6062\u590D\u6765\u6E90\u4E0D\u80FD\u662F\u5F53\u524D\u6570\u636E\u5E93\u7684\u7B26\u53F7\u94FE\u63A5\u6216\u786C\u94FE\u63A5\u522B\u540D");
|
|
19613
|
+
}
|
|
19614
|
+
let sourceCheck;
|
|
19615
|
+
try {
|
|
19616
|
+
const sourceDatabase = new Database3(source, { readonly: true, strict: true });
|
|
19617
|
+
try {
|
|
19618
|
+
sourceDatabase.exec("BEGIN");
|
|
19619
|
+
sourceCheck = this.inspect(sourceDatabase, source);
|
|
19620
|
+
if (!sourceCheck.ok) {
|
|
19621
|
+
throw new Error(
|
|
19622
|
+
sourceCheck.integrityMessages.join("; ") || "schema/foreign key error"
|
|
19623
|
+
);
|
|
19624
|
+
}
|
|
19625
|
+
writeFileSync(stagePath, sourceDatabase.serialize(), {
|
|
19626
|
+
flag: "wx",
|
|
19627
|
+
mode: 384
|
|
19628
|
+
});
|
|
19629
|
+
} finally {
|
|
19630
|
+
if (sourceDatabase.inTransaction) sourceDatabase.exec("ROLLBACK");
|
|
19631
|
+
sourceDatabase.close();
|
|
19632
|
+
}
|
|
19633
|
+
} catch (error) {
|
|
19634
|
+
throw new Error(`\u5907\u4EFD\u6587\u4EF6\u65E0\u6548\uFF1A${errorMessage(error)}`);
|
|
19635
|
+
}
|
|
18709
19636
|
chmodSync(stagePath, 384);
|
|
18710
19637
|
const staged = new Database3(stagePath);
|
|
18711
19638
|
try {
|
|
19639
|
+
staged.exec("PRAGMA journal_mode = DELETE;");
|
|
19640
|
+
staged.exec("PRAGMA busy_timeout = 5000;");
|
|
19641
|
+
migrateSqliteDatabase(staged);
|
|
18712
19642
|
const stagedCheck = this.inspect(staged, stagePath);
|
|
18713
19643
|
if (!stagedCheck.ok) throw new Error("\u6682\u5B58\u6062\u590D\u6587\u4EF6\u6821\u9A8C\u5931\u8D25");
|
|
18714
|
-
const now = Date.now();
|
|
18715
|
-
staged.query(
|
|
18716
|
-
"INSERT OR REPLACE INTO gateway_lock (id, pid, acquired_at, heartbeat_at, ready_at) VALUES (1, ?, ?, ?, NULL)"
|
|
18717
|
-
).run(process.pid, now, now);
|
|
18718
19644
|
} finally {
|
|
18719
19645
|
staged.close();
|
|
18720
19646
|
}
|
|
18721
|
-
const sqlite2 =
|
|
18722
|
-
|
|
19647
|
+
const sqlite2 = current;
|
|
19648
|
+
let recoveredRunningTasks = 0;
|
|
19649
|
+
let closedRunningRuns = 0;
|
|
18723
19650
|
try {
|
|
18724
19651
|
this.assertGatewaySafe(sqlite2, false);
|
|
18725
19652
|
this.assertNoRunningWork(sqlite2);
|
|
18726
19653
|
safetyBackup = this.writeSnapshot(sqlite2, this.createBackupPath("pre-restore"));
|
|
18727
|
-
|
|
18728
|
-
|
|
18729
|
-
|
|
18730
|
-
).run(process.pid, now, now);
|
|
18731
|
-
sqlite2.exec("COMMIT");
|
|
18732
|
-
} catch (error) {
|
|
18733
|
-
if (sqlite2.inTransaction) sqlite2.exec("ROLLBACK");
|
|
18734
|
-
throw error;
|
|
18735
|
-
}
|
|
18736
|
-
const checkpoint = sqlite2.query("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
18737
|
-
if (checkpoint && checkpoint.busy !== 0) {
|
|
18738
|
-
throw new DatabaseMaintenanceConflictError("\u6570\u636E\u5E93\u4ECD\u88AB\u5176\u4ED6\u8FDE\u63A5\u5360\u7528\uFF0C\u65E0\u6CD5\u5B89\u5168\u6062\u590D");
|
|
18739
|
-
}
|
|
18740
|
-
closeDb();
|
|
18741
|
-
safeUnlink(`${livePath}-wal`);
|
|
18742
|
-
safeUnlink(`${livePath}-shm`);
|
|
18743
|
-
renameSync(livePath, rollbackPath);
|
|
18744
|
-
liveMoved = true;
|
|
18745
|
-
renameSync(stagePath, livePath);
|
|
18746
|
-
const restored = getSqlite();
|
|
18747
|
-
restored.exec("BEGIN IMMEDIATE");
|
|
18748
|
-
let recoveredRunningTasks = 0;
|
|
18749
|
-
let closedRunningRuns = 0;
|
|
18750
|
-
try {
|
|
19654
|
+
sqlite2.query("ATTACH DATABASE ? AS restore_source").run(stagePath);
|
|
19655
|
+
attached = true;
|
|
19656
|
+
const restorePlan = this.buildRestorePlan(sqlite2);
|
|
18751
19657
|
recoveredRunningTasks = this.scalar(
|
|
18752
|
-
|
|
18753
|
-
"SELECT COUNT(*) AS count FROM tasks WHERE status = 'running'"
|
|
19658
|
+
sqlite2,
|
|
19659
|
+
"SELECT COUNT(*) AS count FROM restore_source.tasks WHERE status = 'running'"
|
|
18754
19660
|
);
|
|
18755
19661
|
closedRunningRuns = this.scalar(
|
|
18756
|
-
|
|
18757
|
-
"SELECT COUNT(*) AS count FROM task_runs WHERE status = 'running'"
|
|
19662
|
+
sqlite2,
|
|
19663
|
+
"SELECT COUNT(*) AS count FROM restore_source.task_runs WHERE status = 'running'"
|
|
18758
19664
|
);
|
|
19665
|
+
sqlite2.exec("PRAGMA defer_foreign_keys = ON");
|
|
19666
|
+
for (const table of restorePlan) {
|
|
19667
|
+
sqlite2.exec(`DELETE FROM main.${quoteIdentifier(table.name)}`);
|
|
19668
|
+
}
|
|
19669
|
+
for (const table of restorePlan) {
|
|
19670
|
+
if (!table.sourceExists) continue;
|
|
19671
|
+
const columns = table.columns.map((column) => quoteIdentifier(column.name)).join(", ");
|
|
19672
|
+
sqlite2.exec(`
|
|
19673
|
+
INSERT INTO main.${quoteIdentifier(table.name)} (${columns})
|
|
19674
|
+
SELECT ${columns} FROM restore_source.${quoteIdentifier(table.name)}
|
|
19675
|
+
`);
|
|
19676
|
+
}
|
|
19677
|
+
const liveTableNames = restorePlan.map((table) => table.name);
|
|
19678
|
+
const sourceTableNames = restorePlan.filter((table) => table.sourceExists).map((table) => table.name);
|
|
19679
|
+
const livePlaceholders = liveTableNames.map(() => "?").join(", ");
|
|
19680
|
+
sqlite2.query(
|
|
19681
|
+
`DELETE FROM main.sqlite_sequence WHERE name IN (${livePlaceholders})`
|
|
19682
|
+
).run(...liveTableNames);
|
|
19683
|
+
if (sourceTableNames.length > 0) {
|
|
19684
|
+
const sourcePlaceholders = sourceTableNames.map(() => "?").join(", ");
|
|
19685
|
+
sqlite2.query(`
|
|
19686
|
+
INSERT INTO main.sqlite_sequence(name, seq)
|
|
19687
|
+
SELECT name, seq FROM restore_source.sqlite_sequence
|
|
19688
|
+
WHERE name IN (${sourcePlaceholders})
|
|
19689
|
+
`).run(...sourceTableNames);
|
|
19690
|
+
}
|
|
18759
19691
|
const finishedAt = Math.floor(Date.now() / 1e3);
|
|
18760
|
-
|
|
19692
|
+
sqlite2.query(`
|
|
18761
19693
|
UPDATE task_runs
|
|
18762
19694
|
SET status = 'failed', finished_at = ?,
|
|
18763
19695
|
log = CASE
|
|
@@ -18767,20 +19699,34 @@ var init_database_maintenance_service = __esm({
|
|
|
18767
19699
|
END
|
|
18768
19700
|
WHERE status = 'running'
|
|
18769
19701
|
`).run(finishedAt);
|
|
18770
|
-
|
|
19702
|
+
sqlite2.exec(`
|
|
18771
19703
|
UPDATE tasks
|
|
18772
19704
|
SET status = 'pending', started_at = NULL, finished_at = NULL
|
|
18773
19705
|
WHERE status = 'running'
|
|
18774
19706
|
`);
|
|
18775
|
-
|
|
18776
|
-
|
|
19707
|
+
sqlite2.exec("DELETE FROM gateway_lock");
|
|
19708
|
+
const violations = sqlite2.query("PRAGMA foreign_key_check").all();
|
|
19709
|
+
if (violations.length > 0) {
|
|
19710
|
+
throw new Error(`\u6062\u590D\u6570\u636E\u5305\u542B ${violations.length} \u6761\u5916\u952E\u8FDD\u89C4`);
|
|
19711
|
+
}
|
|
19712
|
+
const transactionCheck = this.inspect(sqlite2, livePath);
|
|
19713
|
+
if (!transactionCheck.ok) throw new Error("\u6062\u590D\u540E\u7684\u6570\u636E\u5E93\u672A\u901A\u8FC7\u5B8C\u6574\u6027\u6821\u9A8C");
|
|
19714
|
+
sqlite2.exec("COMMIT");
|
|
19715
|
+
committed = true;
|
|
18777
19716
|
} catch (error) {
|
|
18778
|
-
if (
|
|
19717
|
+
if (sqlite2.inTransaction) sqlite2.exec("ROLLBACK");
|
|
18779
19718
|
throw error;
|
|
19719
|
+
} finally {
|
|
19720
|
+
if (attached && !sqlite2.inTransaction) {
|
|
19721
|
+
try {
|
|
19722
|
+
sqlite2.exec("DETACH DATABASE restore_source");
|
|
19723
|
+
} catch {
|
|
19724
|
+
}
|
|
19725
|
+
attached = false;
|
|
19726
|
+
}
|
|
18780
19727
|
}
|
|
18781
|
-
const check = this.inspect(
|
|
19728
|
+
const check = this.inspect(sqlite2, livePath);
|
|
18782
19729
|
if (!check.ok) throw new Error("\u6062\u590D\u540E\u7684\u6570\u636E\u5E93\u672A\u901A\u8FC7\u5B8C\u6574\u6027\u6821\u9A8C");
|
|
18783
|
-
safeUnlink(rollbackPath);
|
|
18784
19730
|
return {
|
|
18785
19731
|
sourcePath: source,
|
|
18786
19732
|
safetyBackupPath: safetyBackup.path,
|
|
@@ -18789,31 +19735,19 @@ var init_database_maintenance_service = __esm({
|
|
|
18789
19735
|
check
|
|
18790
19736
|
};
|
|
18791
19737
|
} catch (error) {
|
|
18792
|
-
|
|
18793
|
-
safeUnlink(`${livePath}-wal`);
|
|
18794
|
-
safeUnlink(`${livePath}-shm`);
|
|
19738
|
+
if (current.inTransaction) current.exec("ROLLBACK");
|
|
18795
19739
|
safeUnlink(stagePath);
|
|
18796
19740
|
safeUnlink(`${stagePath}-journal`);
|
|
18797
19741
|
safeUnlink(`${stagePath}-wal`);
|
|
18798
19742
|
safeUnlink(`${stagePath}-shm`);
|
|
18799
|
-
if (liveMoved && existsSync3(rollbackPath)) {
|
|
18800
|
-
safeUnlink(livePath);
|
|
18801
|
-
renameSync(rollbackPath, livePath);
|
|
18802
|
-
try {
|
|
18803
|
-
const original = getSqlite();
|
|
18804
|
-
original.query("DELETE FROM gateway_lock WHERE pid = ?").run(process.pid);
|
|
18805
|
-
} catch {
|
|
18806
|
-
}
|
|
18807
|
-
} else if (existsSync3(rollbackPath)) {
|
|
18808
|
-
safeUnlink(rollbackPath);
|
|
18809
|
-
} else {
|
|
18810
|
-
try {
|
|
18811
|
-
getSqlite().query("DELETE FROM gateway_lock WHERE pid = ?").run(process.pid);
|
|
18812
|
-
} catch {
|
|
18813
|
-
}
|
|
18814
|
-
}
|
|
18815
19743
|
const backupHint = safetyBackup ? `\uFF1B\u5F53\u524D\u5E93\u5B89\u5168\u5907\u4EFD\uFF1A${safetyBackup.path}` : "";
|
|
18816
|
-
|
|
19744
|
+
const rollbackHint = committed ? "\uFF1B\u4E8B\u52A1\u5DF2\u63D0\u4EA4\uFF0C\u672A\u81EA\u52A8\u8986\u76D6\u540E\u7EED\u5199\u5165" : "\uFF1B\u4E8B\u52A1\u5DF2\u56DE\u6EDA";
|
|
19745
|
+
throw new Error(`\u6062\u590D\u6570\u636E\u5E93\u5931\u8D25${rollbackHint}${backupHint}\uFF1A${errorMessage(error)}`);
|
|
19746
|
+
} finally {
|
|
19747
|
+
safeUnlink(stagePath);
|
|
19748
|
+
safeUnlink(`${stagePath}-journal`);
|
|
19749
|
+
safeUnlink(`${stagePath}-wal`);
|
|
19750
|
+
safeUnlink(`${stagePath}-shm`);
|
|
18817
19751
|
}
|
|
18818
19752
|
}
|
|
18819
19753
|
static inspectFile(path) {
|
|
@@ -18829,6 +19763,78 @@ var init_database_maintenance_service = __esm({
|
|
|
18829
19763
|
sqlite2.close();
|
|
18830
19764
|
}
|
|
18831
19765
|
}
|
|
19766
|
+
static buildRestorePlan(sqlite2) {
|
|
19767
|
+
const liveTables = this.readRestoreTables(sqlite2, "main");
|
|
19768
|
+
const sourceTables = this.readRestoreTables(sqlite2, "restore_source");
|
|
19769
|
+
for (const [tableName, sourceTable] of sourceTables) {
|
|
19770
|
+
const liveTable = liveTables.get(tableName);
|
|
19771
|
+
if (!liveTable) {
|
|
19772
|
+
throw new Error(`\u6062\u590D\u6765\u6E90\u5305\u542B\u5F53\u524D\u6570\u636E\u5E93\u4E0D\u8BA4\u8BC6\u7684\u4E1A\u52A1\u8868\uFF1A${tableName}`);
|
|
19773
|
+
}
|
|
19774
|
+
const liveColumns = new Map(liveTable.columns.map((column) => [column.name, column]));
|
|
19775
|
+
for (const sourceColumn of sourceTable.columns) {
|
|
19776
|
+
const liveColumn = liveColumns.get(sourceColumn.name);
|
|
19777
|
+
if (!liveColumn) {
|
|
19778
|
+
throw new Error(
|
|
19779
|
+
`\u6062\u590D\u6765\u6E90\u7684\u8868 ${tableName} \u5305\u542B\u5F53\u524D\u6570\u636E\u5E93\u4E0D\u8BA4\u8BC6\u7684\u53EF\u5199\u5217\uFF1A${sourceColumn.name}`
|
|
19780
|
+
);
|
|
19781
|
+
}
|
|
19782
|
+
if (!columnDefinitionsMatch(liveColumn, sourceColumn)) {
|
|
19783
|
+
throw new Error(`\u6062\u590D\u6765\u6E90\u7684\u8868 ${tableName}.${sourceColumn.name} \u4E0E\u5F53\u524D\u6570\u636E\u5E93\u5B9A\u4E49\u4E0D\u517C\u5BB9`);
|
|
19784
|
+
}
|
|
19785
|
+
}
|
|
19786
|
+
}
|
|
19787
|
+
const plan = [];
|
|
19788
|
+
for (const [tableName, liveTable] of liveTables) {
|
|
19789
|
+
const sourceTable = sourceTables.get(tableName);
|
|
19790
|
+
if (!sourceTable) {
|
|
19791
|
+
plan.push({ ...liveTable, columns: [], sourceExists: false });
|
|
19792
|
+
continue;
|
|
19793
|
+
}
|
|
19794
|
+
const sourceColumns = new Map(sourceTable.columns.map((column) => [column.name, column]));
|
|
19795
|
+
for (const liveColumn of liveTable.columns) {
|
|
19796
|
+
if (sourceColumns.has(liveColumn.name)) continue;
|
|
19797
|
+
if (liveColumn.primaryKeyOrder > 0 || liveColumn.notNull && liveColumn.defaultValue == null) {
|
|
19798
|
+
throw new Error(
|
|
19799
|
+
`\u5F53\u524D\u6570\u636E\u5E93\u7684\u8868 ${tableName}.${liveColumn.name} \u65E0\u6CD5\u4ECE\u65E7\u5907\u4EFD\u5B89\u5168\u8865\u9ED8\u8BA4\u503C`
|
|
19800
|
+
);
|
|
19801
|
+
}
|
|
19802
|
+
}
|
|
19803
|
+
plan.push({
|
|
19804
|
+
name: tableName,
|
|
19805
|
+
columns: sourceTable.columns,
|
|
19806
|
+
sourceExists: true
|
|
19807
|
+
});
|
|
19808
|
+
}
|
|
19809
|
+
return plan;
|
|
19810
|
+
}
|
|
19811
|
+
static readRestoreTables(sqlite2, databaseName) {
|
|
19812
|
+
const tableRows = sqlite2.query(`
|
|
19813
|
+
SELECT name
|
|
19814
|
+
FROM ${databaseName}.sqlite_schema
|
|
19815
|
+
WHERE type = 'table'
|
|
19816
|
+
AND name NOT LIKE 'sqlite_%'
|
|
19817
|
+
AND name NOT IN ('gateway_lock', '__drizzle_migrations')
|
|
19818
|
+
ORDER BY name
|
|
19819
|
+
`).all();
|
|
19820
|
+
const result = /* @__PURE__ */ new Map();
|
|
19821
|
+
for (const row of tableRows) {
|
|
19822
|
+
const columns = sqlite2.query(
|
|
19823
|
+
`PRAGMA ${databaseName}.table_xinfo(${quoteIdentifier(row.name)})`
|
|
19824
|
+
).all();
|
|
19825
|
+
result.set(row.name, {
|
|
19826
|
+
name: row.name,
|
|
19827
|
+
columns: columns.filter((column) => column.hidden === 0).map((column) => ({
|
|
19828
|
+
name: column.name,
|
|
19829
|
+
type: column.type,
|
|
19830
|
+
notNull: column.notnull !== 0,
|
|
19831
|
+
defaultValue: column.dflt_value,
|
|
19832
|
+
primaryKeyOrder: column.pk
|
|
19833
|
+
}))
|
|
19834
|
+
});
|
|
19835
|
+
}
|
|
19836
|
+
return result;
|
|
19837
|
+
}
|
|
18832
19838
|
static inspect(sqlite2, path) {
|
|
18833
19839
|
const tableRows = sqlite2.query(
|
|
18834
19840
|
"SELECT name FROM sqlite_master WHERE type = 'table'"
|
|
@@ -18842,7 +19848,7 @@ var init_database_maintenance_service = __esm({
|
|
|
18842
19848
|
const counts = missingTables.length === 0 ? this.readCounts(sqlite2) : { tasks: 0, taskRuns: 0, taskTemplates: 0 };
|
|
18843
19849
|
const runningTasks = missingTables.includes("tasks") ? 0 : this.scalar(sqlite2, "SELECT COUNT(*) AS count FROM tasks WHERE status = 'running'");
|
|
18844
19850
|
const runningRuns = missingTables.includes("task_runs") ? 0 : this.scalar(sqlite2, "SELECT COUNT(*) AS count FROM task_runs WHERE status = 'running'");
|
|
18845
|
-
const sizeBytes = path === ":memory:" || !
|
|
19851
|
+
const sizeBytes = path === ":memory:" || !existsSync5(path) ? sqlite2.serialize().byteLength : statSync(path).size;
|
|
18846
19852
|
return {
|
|
18847
19853
|
ok: integrityMessages.length === 1 && integrityMessages[0] === "ok" && foreignKeyViolations === 0 && missingTables.length === 0,
|
|
18848
19854
|
path: normalizedPath(path),
|
|
@@ -18863,9 +19869,9 @@ var init_database_maintenance_service = __esm({
|
|
|
18863
19869
|
if (DB_FILE_PATH !== ":memory:" && output === normalizedPath(DB_FILE_PATH)) {
|
|
18864
19870
|
throw new Error("\u5907\u4EFD\u8DEF\u5F84\u4E0D\u80FD\u8986\u76D6\u5F53\u524D\u6570\u636E\u5E93");
|
|
18865
19871
|
}
|
|
18866
|
-
if (
|
|
18867
|
-
mkdirSync2(
|
|
18868
|
-
const temporary = `${output}.tmp-${process.pid}-${
|
|
19872
|
+
if (existsSync5(output)) throw new Error(`\u5907\u4EFD\u6587\u4EF6\u5DF2\u5B58\u5728\uFF1A${output}`);
|
|
19873
|
+
mkdirSync2(dirname5(output), { recursive: true });
|
|
19874
|
+
const temporary = `${output}.tmp-${process.pid}-${randomUUID2()}`;
|
|
18869
19875
|
try {
|
|
18870
19876
|
writeFileSync(temporary, sqlite2.serialize(), { flag: "wx", mode: 384 });
|
|
18871
19877
|
const standalone = new Database3(temporary, { readwrite: true, create: false });
|
|
@@ -18889,9 +19895,9 @@ var init_database_maintenance_service = __esm({
|
|
|
18889
19895
|
}
|
|
18890
19896
|
}
|
|
18891
19897
|
static createBackupPath(kind) {
|
|
18892
|
-
const directory = DB_FILE_PATH === ":memory:" ? tmpdir() :
|
|
19898
|
+
const directory = DB_FILE_PATH === ":memory:" ? tmpdir() : dirname5(normalizedPath(DB_FILE_PATH));
|
|
18893
19899
|
const base = DB_FILE_PATH === ":memory:" ? "supertask-memory" : basename2(DB_FILE_PATH, ".db");
|
|
18894
|
-
return
|
|
19900
|
+
return resolve2(directory, `${base}.${kind}-${timestamp()}-${randomUUID2().slice(0, 8)}.db`);
|
|
18895
19901
|
}
|
|
18896
19902
|
static readCounts(sqlite2) {
|
|
18897
19903
|
return {
|
|
@@ -18931,8 +19937,8 @@ __export(web_exports, {
|
|
|
18931
19937
|
dashboardApp: () => dashboardApp,
|
|
18932
19938
|
default: () => web_default
|
|
18933
19939
|
});
|
|
18934
|
-
import { existsSync as
|
|
18935
|
-
import { dirname as
|
|
19940
|
+
import { existsSync as existsSync6, readFileSync as readFileSync3, writeFileSync as writeFileSync2, mkdirSync as mkdirSync3, renameSync as renameSync2 } from "fs";
|
|
19941
|
+
import { dirname as dirname6 } from "path";
|
|
18936
19942
|
function parsePositiveInteger(value) {
|
|
18937
19943
|
if (!/^\d+$/.test(value)) return null;
|
|
18938
19944
|
const parsed = Number(value);
|
|
@@ -18982,17 +19988,17 @@ function esc(s) {
|
|
|
18982
19988
|
}
|
|
18983
19989
|
function readCurrentConfig() {
|
|
18984
19990
|
const configPath = getConfigPath();
|
|
18985
|
-
if (!
|
|
19991
|
+
if (!existsSync6(configPath)) return {};
|
|
18986
19992
|
try {
|
|
18987
|
-
return JSON.parse(
|
|
19993
|
+
return JSON.parse(readFileSync3(configPath, "utf-8"));
|
|
18988
19994
|
} catch {
|
|
18989
19995
|
return {};
|
|
18990
19996
|
}
|
|
18991
19997
|
}
|
|
18992
19998
|
function writeConfig(cfg) {
|
|
18993
19999
|
const configPath = getConfigPath();
|
|
18994
|
-
const dir =
|
|
18995
|
-
if (!
|
|
20000
|
+
const dir = dirname6(configPath);
|
|
20001
|
+
if (!existsSync6(dir)) mkdirSync3(dir, { recursive: true });
|
|
18996
20002
|
const tempPath = `${configPath}.${process.pid}.tmp`;
|
|
18997
20003
|
writeFileSync2(tempPath, JSON.stringify(cfg, null, 2) + "\n", { mode: 384 });
|
|
18998
20004
|
renameSync2(tempPath, configPath);
|
|
@@ -19000,8 +20006,9 @@ function writeConfig(cfg) {
|
|
|
19000
20006
|
function renderLayout(title, activeTab, body) {
|
|
19001
20007
|
return `<!DOCTYPE html><html><head><meta charset="UTF-8"><title>${title} - SuperTask</title>${SHARED_STYLES}
|
|
19002
20008
|
<script>
|
|
19003
|
-
async function retryTask(id){if(!confirm('\u786E\u5B9A\u91CD\u8BD5\u4EFB\u52A1 #'+id+'?'))return;await fetch('/api/tasks/'+id+'/retry',{method:'POST'});location.reload();}
|
|
19004
|
-
async function
|
|
20009
|
+
async function retryTask(id){if(!confirm('\u786E\u5B9A\u91CD\u8BD5\u4EFB\u52A1 #'+id+'?'))return;const r=await fetch('/api/tasks/'+id+'/retry',{method:'POST'});const d=await r.json();if(!r.ok){alert('\u91CD\u8BD5\u5931\u8D25: '+d.error);return;}location.reload();}
|
|
20010
|
+
async function cancelTask(id){if(!confirm('\u786E\u5B9A\u53D6\u6D88\u4EFB\u52A1 #'+id+'?'))return;const r=await fetch('/api/tasks/'+id+'/cancel',{method:'POST'});const d=await r.json();if(!r.ok){alert('\u53D6\u6D88\u5931\u8D25: '+d.error);return;}location.reload();}
|
|
20011
|
+
async function deleteTask(id){if(!confirm('\u786E\u5B9A\u5220\u9664\u4EFB\u52A1 #'+id+'?'))return;const r=await fetch('/api/tasks/'+id,{method:'DELETE'});const d=await r.json();if(!r.ok){alert('\u5220\u9664\u5931\u8D25: '+d.error);return;}location.reload();}
|
|
19005
20012
|
async function showDetail(id){try{const r=await fetch('/api/tasks/'+id);const t=await r.json();document.getElementById('dc').textContent=JSON.stringify(t,null,2);document.getElementById('dd').showModal();}catch(e){alert('\u83B7\u53D6\u8BE6\u60C5\u5931\u8D25');}}
|
|
19006
20013
|
async function showRunDetail(id){try{const r=await fetch('/api/runs/'+id);const t=await r.json();document.getElementById('dc').textContent=JSON.stringify(t,null,2);document.getElementById('dd').showModal();}catch(e){alert('\u83B7\u53D6\u8BE6\u60C5\u5931\u8D25');}}
|
|
19007
20014
|
async function showTemplateDetail(id){try{const r=await fetch('/api/templates/'+id);const t=await r.json();document.getElementById('dc').textContent=JSON.stringify(t,null,2);document.getElementById('dd').showModal();}catch(e){alert('\u83B7\u53D6\u8BE6\u60C5\u5931\u8D25');}}
|
|
@@ -19083,6 +20090,7 @@ var init_web = __esm({
|
|
|
19083
20090
|
init_db2();
|
|
19084
20091
|
init_config();
|
|
19085
20092
|
init_health();
|
|
20093
|
+
init_job_templates();
|
|
19086
20094
|
app = new Hono2();
|
|
19087
20095
|
TASK_STATUSES = /* @__PURE__ */ new Set([
|
|
19088
20096
|
"pending",
|
|
@@ -19207,11 +20215,11 @@ var init_web = __esm({
|
|
|
19207
20215
|
if (statusFilter && !parsedStatus) return c.text("invalid status", 400);
|
|
19208
20216
|
const limit = 50;
|
|
19209
20217
|
const offset = (page - 1) * limit;
|
|
19210
|
-
const [
|
|
20218
|
+
const [tasks4, statsData] = await Promise.all([
|
|
19211
20219
|
TaskService.list({ limit, offset, ...parsedStatus ? { status: parsedStatus } : {} }),
|
|
19212
20220
|
TaskService.stats({})
|
|
19213
20221
|
]);
|
|
19214
|
-
const taskIds =
|
|
20222
|
+
const taskIds = tasks4.map((t) => t.id);
|
|
19215
20223
|
const latestRuns = await TaskRunService.getLatestByTaskIds(taskIds);
|
|
19216
20224
|
const counts = {
|
|
19217
20225
|
pending: statsData.pending || 0,
|
|
@@ -19230,9 +20238,10 @@ var init_web = __esm({
|
|
|
19230
20238
|
<a href="/?status=dead_letter" class="btn ${statusFilter === "dead_letter" ? "btn-primary" : ""}">Dead Letter</a>
|
|
19231
20239
|
</div>`;
|
|
19232
20240
|
let rows = "";
|
|
19233
|
-
for (const task of
|
|
20241
|
+
for (const task of tasks4) {
|
|
19234
20242
|
const status = safeStatus(task.status);
|
|
19235
20243
|
const st = status.toUpperCase();
|
|
20244
|
+
const executionActive = latestRuns.get(task.id)?.status === "running";
|
|
19236
20245
|
rows += `<tr>
|
|
19237
20246
|
<td class="mu">#${task.id}</td>
|
|
19238
20247
|
<td><div style="font-weight:500">${esc(task.name)}</div><div class="mu sm el">${esc(task.prompt.substring(0, 120))}</div></td>
|
|
@@ -19243,7 +20252,8 @@ var init_web = __esm({
|
|
|
19243
20252
|
<td>
|
|
19244
20253
|
<button class="btn btn-sm" onclick="showDetail(${task.id})">\u8BE6\u60C5</button>
|
|
19245
20254
|
${task.status === "failed" || task.status === "dead_letter" ? `<button class="btn btn-sm btn-warn" onclick="retryTask(${task.id})">\u91CD\u8BD5</button>` : ""}
|
|
19246
|
-
|
|
20255
|
+
${["pending", "running", "failed"].includes(task.status ?? "") ? `<button class="btn btn-sm btn-warn" onclick="cancelTask(${task.id})">\u53D6\u6D88</button>` : ""}
|
|
20256
|
+
${task.status === "running" || executionActive ? "" : `<button class="btn btn-sm btn-danger" onclick="deleteTask(${task.id})">\u5220\u9664</button>`}
|
|
19247
20257
|
</td></tr>`;
|
|
19248
20258
|
}
|
|
19249
20259
|
const qp = statusFilter ? `&status=${statusFilter}` : "";
|
|
@@ -19387,7 +20397,7 @@ var init_web = __esm({
|
|
|
19387
20397
|
const stats = await TaskService.stats({});
|
|
19388
20398
|
const runningRuns = await TaskRunService.getAllRunningRuns();
|
|
19389
20399
|
const templates = await TaskTemplateService.list(100);
|
|
19390
|
-
const configExists =
|
|
20400
|
+
const configExists = existsSync6(configPath);
|
|
19391
20401
|
let runRows = "";
|
|
19392
20402
|
if (runningRuns.length > 0) {
|
|
19393
20403
|
for (const run of runningRuns) {
|
|
@@ -19495,11 +20505,25 @@ var init_web = __esm({
|
|
|
19495
20505
|
if (task) return c.json({ success: true });
|
|
19496
20506
|
return await TaskService.getById(id) ? c.json({ error: "task status does not allow retry" }, 409) : c.json({ error: "not found" }, 404);
|
|
19497
20507
|
});
|
|
20508
|
+
app.post("/api/tasks/:id/cancel", async (c) => {
|
|
20509
|
+
const id = parsePositiveInteger(c.req.param("id"));
|
|
20510
|
+
if (id === null) return c.json({ error: "invalid id" }, 400);
|
|
20511
|
+
const task = await TaskService.cancel(id);
|
|
20512
|
+
if (task) return c.json({ success: true });
|
|
20513
|
+
return await TaskService.getById(id) ? c.json({ error: "task status does not allow cancellation" }, 409) : c.json({ error: "not found" }, 404);
|
|
20514
|
+
});
|
|
19498
20515
|
app.delete("/api/tasks/:id", async (c) => {
|
|
19499
20516
|
const id = parsePositiveInteger(c.req.param("id"));
|
|
19500
20517
|
if (id === null) return c.json({ error: "invalid id" }, 400);
|
|
19501
|
-
|
|
19502
|
-
|
|
20518
|
+
try {
|
|
20519
|
+
const deleted = await TaskService.delete(id);
|
|
20520
|
+
return deleted ? c.json({ success: true }) : c.json({ error: "not found" }, 404);
|
|
20521
|
+
} catch (error) {
|
|
20522
|
+
if (error instanceof TaskDeletionConflictError) {
|
|
20523
|
+
return c.json({ error: error.message }, 409);
|
|
20524
|
+
}
|
|
20525
|
+
throw error;
|
|
20526
|
+
}
|
|
19503
20527
|
});
|
|
19504
20528
|
app.post("/api/templates/:id/enable", async (c) => {
|
|
19505
20529
|
const id = parsePositiveInteger(c.req.param("id"));
|
|
@@ -19524,21 +20548,8 @@ var init_web = __esm({
|
|
|
19524
20548
|
if (id === null) return c.json({ error: "invalid id" }, 400);
|
|
19525
20549
|
const tmpl = await TaskTemplateService.getById(id);
|
|
19526
20550
|
if (!tmpl) return c.json({ error: "not found" }, 404);
|
|
19527
|
-
const task = await
|
|
19528
|
-
|
|
19529
|
-
agent: tmpl.agent,
|
|
19530
|
-
model: tmpl.model,
|
|
19531
|
-
prompt: tmpl.prompt,
|
|
19532
|
-
cwd: tmpl.cwd,
|
|
19533
|
-
category: tmpl.category,
|
|
19534
|
-
importance: tmpl.importance,
|
|
19535
|
-
urgency: tmpl.urgency,
|
|
19536
|
-
batchId: tmpl.batchId,
|
|
19537
|
-
maxRetries: tmpl.maxRetries,
|
|
19538
|
-
retryBackoffMs: tmpl.retryBackoffMs,
|
|
19539
|
-
timeoutMs: tmpl.timeoutMs,
|
|
19540
|
-
templateId: tmpl.id
|
|
19541
|
-
});
|
|
20551
|
+
const task = await triggerTaskFromTemplate(id);
|
|
20552
|
+
if (!task) return c.json({ error: "maxInstances reached" }, 409);
|
|
19542
20553
|
return c.json({ success: true, taskId: task.id });
|
|
19543
20554
|
});
|
|
19544
20555
|
app.put("/api/config", async (c) => {
|
|
@@ -19580,6 +20591,7 @@ var init_web = __esm({
|
|
|
19580
20591
|
});
|
|
19581
20592
|
dashboardApp = app;
|
|
19582
20593
|
web_default = {
|
|
20594
|
+
hostname: "127.0.0.1",
|
|
19583
20595
|
port: 4680,
|
|
19584
20596
|
fetch: app.fetch
|
|
19585
20597
|
};
|
|
@@ -19595,9 +20607,21 @@ init_task_service();
|
|
|
19595
20607
|
init_task_run_service();
|
|
19596
20608
|
init_health();
|
|
19597
20609
|
init_process_control();
|
|
20610
|
+
init_launch_protocol();
|
|
19598
20611
|
import { spawn } from "child_process";
|
|
20612
|
+
import { fileURLToPath as fileURLToPath3 } from "url";
|
|
20613
|
+
import { existsSync as existsSync3 } from "fs";
|
|
20614
|
+
import { dirname as dirname3, join as join3 } from "path";
|
|
20615
|
+
import { randomUUID } from "crypto";
|
|
19599
20616
|
var DEFAULT_MAX_OUTPUT_CHARS = 64 * 1024;
|
|
19600
20617
|
var FORBIDDEN_AGENT = "supertask-runner";
|
|
20618
|
+
function assertWorkerProcessIsolationSupported(platform = process.platform) {
|
|
20619
|
+
if (platform === "win32") {
|
|
20620
|
+
throw new Error(
|
|
20621
|
+
"Windows Worker \u5DF2\u5B89\u5168\u7981\u7528\uFF1A\u5F53\u524D\u8FD0\u884C\u65F6\u65E0\u6CD5\u7528 Job Object \u8BC1\u660E\u6574\u4E2A OpenCode \u8FDB\u7A0B\u6811\u5DF2\u9000\u51FA"
|
|
20622
|
+
);
|
|
20623
|
+
}
|
|
20624
|
+
}
|
|
19601
20625
|
var WorkerEngine = class {
|
|
19602
20626
|
activeBatchIds = /* @__PURE__ */ new Set();
|
|
19603
20627
|
runningTasks = /* @__PURE__ */ new Map();
|
|
@@ -19614,10 +20638,13 @@ var WorkerEngine = class {
|
|
|
19614
20638
|
this.maxOutputChars = options.maxOutputChars ?? DEFAULT_MAX_OUTPUT_CHARS;
|
|
19615
20639
|
}
|
|
19616
20640
|
start() {
|
|
20641
|
+
assertWorkerProcessIsolationSupported();
|
|
19617
20642
|
this.stopped = false;
|
|
19618
20643
|
markGatewayActivity("worker");
|
|
19619
20644
|
this.poll();
|
|
19620
|
-
this.heartbeatTimer = setInterval(() =>
|
|
20645
|
+
this.heartbeatTimer = setInterval(() => {
|
|
20646
|
+
this.runDetached(this.updateHeartbeats(), "worker heartbeat cycle failed");
|
|
20647
|
+
}, this.cfg.heartbeatIntervalMs);
|
|
19621
20648
|
}
|
|
19622
20649
|
async stop(gracePeriodMs = 0) {
|
|
19623
20650
|
this.stopped = true;
|
|
@@ -19636,14 +20663,23 @@ var WorkerEngine = class {
|
|
|
19636
20663
|
clearInterval(this.heartbeatTimer);
|
|
19637
20664
|
this.heartbeatTimer = null;
|
|
19638
20665
|
}
|
|
19639
|
-
const
|
|
19640
|
-
const
|
|
19641
|
-
|
|
19642
|
-
entry.
|
|
19643
|
-
|
|
19644
|
-
|
|
19645
|
-
|
|
19646
|
-
|
|
20666
|
+
const interrupted = [];
|
|
20667
|
+
const entries = [...this.runningTasks.values()];
|
|
20668
|
+
await Promise.all(entries.map(async (entry) => {
|
|
20669
|
+
if (entry.settled) {
|
|
20670
|
+
if (entry.settlementPromise) await entry.settlementPromise;
|
|
20671
|
+
return;
|
|
20672
|
+
}
|
|
20673
|
+
const termination = entry.termination ?? {
|
|
20674
|
+
kind: "shutdown",
|
|
20675
|
+
message: "Gateway shutdown"
|
|
20676
|
+
};
|
|
20677
|
+
const terminated = await this.terminateEntry(entry, termination);
|
|
20678
|
+
if (terminated && termination.kind === "shutdown") {
|
|
20679
|
+
interrupted.push({ taskId: entry.task.id, runId: entry.runId });
|
|
20680
|
+
}
|
|
20681
|
+
}));
|
|
20682
|
+
return interrupted;
|
|
19647
20683
|
}
|
|
19648
20684
|
getRunningTaskIds() {
|
|
19649
20685
|
return [...this.runningTasks.keys()];
|
|
@@ -19651,24 +20687,40 @@ var WorkerEngine = class {
|
|
|
19651
20687
|
getRunningCount() {
|
|
19652
20688
|
return this.runningTasks.size;
|
|
19653
20689
|
}
|
|
20690
|
+
ownsRun(taskId, runId) {
|
|
20691
|
+
return this.runningTasks.get(taskId)?.runId === runId;
|
|
20692
|
+
}
|
|
19654
20693
|
poll() {
|
|
19655
20694
|
if (this.stopped) return;
|
|
19656
20695
|
markGatewayActivity("worker");
|
|
19657
|
-
this.pollCyclePromise = this.tryDispatch().
|
|
20696
|
+
this.pollCyclePromise = this.tryDispatch().then((healthy) => {
|
|
20697
|
+
if (healthy) markGatewaySuccess("worker");
|
|
20698
|
+
}).catch((err) => {
|
|
20699
|
+
markGatewayFailure("worker", err);
|
|
20700
|
+
this.logError("worker poll failed", err);
|
|
20701
|
+
}).finally(() => {
|
|
19658
20702
|
this.pollCyclePromise = null;
|
|
19659
20703
|
if (this.stopped) return;
|
|
19660
20704
|
this.pollTimer = setTimeout(() => this.poll(), this.cfg.pollIntervalMs);
|
|
19661
20705
|
});
|
|
19662
20706
|
}
|
|
19663
20707
|
async tryDispatch() {
|
|
20708
|
+
await TaskService.resetOrphanRunningToPending();
|
|
19664
20709
|
await this.reconcileCancelledTasks();
|
|
20710
|
+
await this.reconcileQuarantinedTasks();
|
|
20711
|
+
const quarantined = [...this.runningTasks.values()].filter((entry) => entry.quarantined);
|
|
20712
|
+
if (quarantined.length > 0) {
|
|
20713
|
+
return false;
|
|
20714
|
+
}
|
|
19665
20715
|
while (!this.stopped && this.runningTasks.size < this.cfg.maxConcurrency) {
|
|
20716
|
+
const databaseRunningCount = await TaskService.countRunning();
|
|
20717
|
+
if (databaseRunningCount >= this.cfg.maxConcurrency) break;
|
|
19666
20718
|
let task;
|
|
19667
20719
|
try {
|
|
19668
20720
|
task = await TaskService.next({ excludedBatchIds: [...this.activeBatchIds] });
|
|
19669
20721
|
} catch (err) {
|
|
19670
20722
|
this.logError("task claim failed", err);
|
|
19671
|
-
|
|
20723
|
+
throw err;
|
|
19672
20724
|
}
|
|
19673
20725
|
if (!task) break;
|
|
19674
20726
|
if (this.stopped) break;
|
|
@@ -19679,12 +20731,19 @@ var WorkerEngine = class {
|
|
|
19679
20731
|
this.releaseBatch(task);
|
|
19680
20732
|
break;
|
|
19681
20733
|
}
|
|
20734
|
+
let runId = null;
|
|
19682
20735
|
try {
|
|
20736
|
+
const launchIdentity = `gateway-${process.pid}:launch:${randomUUID()}`;
|
|
19683
20737
|
const run = await TaskRunService.create({
|
|
19684
20738
|
taskId: task.id,
|
|
19685
20739
|
model: this.resolveModel(task.model),
|
|
19686
|
-
status: "running"
|
|
20740
|
+
status: "running",
|
|
20741
|
+
workerPid: process.pid,
|
|
20742
|
+
lockedAt: Date.now(),
|
|
20743
|
+
lockedBy: launchIdentity,
|
|
20744
|
+
launchProtocol: TOKEN_GUARDIAN_LAUNCH_PROTOCOL
|
|
19687
20745
|
});
|
|
20746
|
+
runId = run.id;
|
|
19688
20747
|
if (this.stopped) {
|
|
19689
20748
|
await TaskRunService.fail(run.id, "Gateway shutdown before spawn");
|
|
19690
20749
|
await TaskService.resetRunningToPending([task.id]);
|
|
@@ -19693,49 +20752,82 @@ var WorkerEngine = class {
|
|
|
19693
20752
|
}
|
|
19694
20753
|
if (task.agent === FORBIDDEN_AGENT) {
|
|
19695
20754
|
const message = `\u7981\u6B62\u6267\u884C\u9012\u5F52 Agent: ${FORBIDDEN_AGENT}`;
|
|
19696
|
-
await
|
|
19697
|
-
await TaskService.fail(task.id, message, {}, { setDeadLetter: true });
|
|
20755
|
+
await TaskService.failRun(task.id, run.id, message, { setDeadLetter: true });
|
|
19698
20756
|
this.releaseBatch(task);
|
|
19699
20757
|
continue;
|
|
19700
20758
|
}
|
|
19701
|
-
this.spawnTask(task, run.id);
|
|
20759
|
+
await this.spawnTask(task, run.id, launchIdentity);
|
|
19702
20760
|
} catch (err) {
|
|
19703
20761
|
this.releaseBatch(task);
|
|
19704
20762
|
const message = `Worker \u542F\u52A8\u4EFB\u52A1\u5931\u8D25\uFF1A${err instanceof Error ? err.message : String(err)}`;
|
|
19705
20763
|
try {
|
|
19706
|
-
|
|
20764
|
+
if (runId == null) {
|
|
20765
|
+
await TaskService.fail(task.id, message);
|
|
20766
|
+
} else {
|
|
20767
|
+
const failed = await TaskService.failRun(task.id, runId, message);
|
|
20768
|
+
if (!failed) {
|
|
20769
|
+
await TaskRunService.fail(runId, `${message}
|
|
20770
|
+
\u4EFB\u52A1\u72B6\u6001\u5DF2\u88AB\u5176\u4ED6\u64CD\u4F5C\u6539\u53D8`);
|
|
20771
|
+
}
|
|
20772
|
+
}
|
|
19707
20773
|
} catch (failErr) {
|
|
19708
20774
|
this.logError("failed to compensate task startup", failErr, task.id);
|
|
19709
20775
|
}
|
|
19710
20776
|
this.logError("task dispatch failed", err, task.id);
|
|
19711
20777
|
}
|
|
19712
20778
|
}
|
|
20779
|
+
return ![...this.runningTasks.values()].some((entry) => entry.quarantined);
|
|
20780
|
+
}
|
|
20781
|
+
launcherEntry() {
|
|
20782
|
+
const extension = import.meta.url.endsWith(".ts") ? "ts" : "js";
|
|
20783
|
+
const moduleDir = dirname3(fileURLToPath3(import.meta.url));
|
|
20784
|
+
const candidates = [
|
|
20785
|
+
join3(moduleDir, `launcher.${extension}`),
|
|
20786
|
+
join3(moduleDir, `../worker/launcher.${extension}`)
|
|
20787
|
+
];
|
|
20788
|
+
const entry = candidates.find((candidate) => existsSync3(candidate));
|
|
20789
|
+
if (!entry) throw new Error(`Worker launcher \u4E0D\u5B58\u5728\uFF1A${candidates.join(", ")}`);
|
|
20790
|
+
return entry;
|
|
19713
20791
|
}
|
|
19714
|
-
spawnTask(task, runId) {
|
|
20792
|
+
async spawnTask(task, runId, launchIdentity) {
|
|
19715
20793
|
const model = this.resolveModel(task.model);
|
|
19716
20794
|
const args = ["run", "--agent", task.agent, "--format", "json"];
|
|
19717
20795
|
if (model) args.push("-m", model);
|
|
19718
20796
|
args.push(task.prompt);
|
|
19719
|
-
const child = spawn(
|
|
20797
|
+
const child = spawn(process.execPath, [
|
|
20798
|
+
this.launcherEntry(),
|
|
20799
|
+
LAUNCH_IDENTITY_ARGUMENT,
|
|
20800
|
+
launchIdentity,
|
|
20801
|
+
this.opencodeBin,
|
|
20802
|
+
...args
|
|
20803
|
+
], {
|
|
19720
20804
|
cwd: task.cwd || process.cwd(),
|
|
19721
|
-
|
|
20805
|
+
env: {
|
|
20806
|
+
...process.env,
|
|
20807
|
+
[MANAGED_RUN_ENV]: MANAGED_RUN_ENV_VALUE
|
|
20808
|
+
},
|
|
20809
|
+
stdio: ["pipe", "pipe", "pipe", "ipc"],
|
|
19722
20810
|
detached: process.platform !== "win32"
|
|
19723
20811
|
});
|
|
19724
20812
|
const entry = {
|
|
19725
20813
|
task,
|
|
19726
20814
|
runId,
|
|
20815
|
+
launchIdentity,
|
|
19727
20816
|
child,
|
|
19728
20817
|
output: "",
|
|
19729
20818
|
sessionId: null,
|
|
19730
20819
|
timeoutTimer: null,
|
|
19731
|
-
|
|
20820
|
+
termination: null,
|
|
20821
|
+
terminationPromise: null,
|
|
20822
|
+
settlementPromise: null,
|
|
20823
|
+
guardianDrained: false,
|
|
20824
|
+
quarantined: false,
|
|
19732
20825
|
settled: false
|
|
19733
20826
|
};
|
|
19734
20827
|
this.runningTasks.set(task.id, entry);
|
|
19735
20828
|
const handleData = (data) => {
|
|
19736
20829
|
const text2 = data.toString();
|
|
19737
20830
|
entry.output = (entry.output + text2).slice(-this.maxOutputChars);
|
|
19738
|
-
process.stdout.write(text2);
|
|
19739
20831
|
const match2 = entry.output.match(/"sessionID"\s*:\s*"(ses_[^"]+)"/);
|
|
19740
20832
|
if (match2?.[1] && match2[1] !== entry.sessionId) {
|
|
19741
20833
|
entry.sessionId = match2[1];
|
|
@@ -19746,107 +20838,210 @@ var WorkerEngine = class {
|
|
|
19746
20838
|
};
|
|
19747
20839
|
child.stdout?.on("data", handleData);
|
|
19748
20840
|
child.stderr?.on("data", handleData);
|
|
19749
|
-
child.
|
|
19750
|
-
|
|
20841
|
+
child.on("message", (message) => {
|
|
20842
|
+
if (isMatchingDrainProof(message, entry.launchIdentity)) {
|
|
20843
|
+
entry.guardianDrained = true;
|
|
20844
|
+
}
|
|
20845
|
+
});
|
|
20846
|
+
let spawnError = null;
|
|
20847
|
+
const spawned = new Promise((resolve3, reject) => {
|
|
20848
|
+
child.once("spawn", resolve3);
|
|
20849
|
+
child.once("error", (error) => {
|
|
20850
|
+
spawnError = error;
|
|
20851
|
+
reject(error);
|
|
20852
|
+
});
|
|
19751
20853
|
});
|
|
19752
20854
|
child.once("close", (code, signal) => {
|
|
19753
|
-
|
|
19754
|
-
void this.finishEntry(entry, code, failure);
|
|
20855
|
+
this.handleChildClose(entry, code, signal, spawnError);
|
|
19755
20856
|
});
|
|
20857
|
+
try {
|
|
20858
|
+
await spawned;
|
|
20859
|
+
} catch (error) {
|
|
20860
|
+
entry.settled = true;
|
|
20861
|
+
this.runningTasks.delete(task.id);
|
|
20862
|
+
this.releaseBatch(task);
|
|
20863
|
+
throw error;
|
|
20864
|
+
}
|
|
20865
|
+
const childPid = child.pid;
|
|
20866
|
+
if (!childPid) {
|
|
20867
|
+
entry.settled = true;
|
|
20868
|
+
this.runningTasks.delete(task.id);
|
|
20869
|
+
this.releaseBatch(task);
|
|
20870
|
+
throw new Error("launcher \u672A\u8FD4\u56DE PID");
|
|
20871
|
+
}
|
|
20872
|
+
let pidRecorded = false;
|
|
20873
|
+
try {
|
|
20874
|
+
pidRecorded = await TaskRunService.updatePid(
|
|
20875
|
+
runId,
|
|
20876
|
+
process.pid,
|
|
20877
|
+
childPid,
|
|
20878
|
+
launchIdentity
|
|
20879
|
+
) !== null;
|
|
20880
|
+
} catch (error) {
|
|
20881
|
+
await this.terminateForFailure(
|
|
20882
|
+
entry,
|
|
20883
|
+
`\u8BB0\u5F55 Worker PID \u5931\u8D25\uFF1A${error instanceof Error ? error.message : String(error)}`
|
|
20884
|
+
);
|
|
20885
|
+
return;
|
|
20886
|
+
}
|
|
20887
|
+
if (!pidRecorded) {
|
|
20888
|
+
await this.terminateForFailure(entry, "\u8BB0\u5F55 Worker PID \u5931\u8D25\uFF1Arun \u5DF2\u4E0D\u518D\u5904\u4E8E running \u72B6\u6001");
|
|
20889
|
+
return;
|
|
20890
|
+
}
|
|
20891
|
+
if (!child.stdin) {
|
|
20892
|
+
await this.terminateForFailure(entry, "\u653E\u884C OpenCode \u5931\u8D25\uFF1Alauncher stdin \u4E0D\u53EF\u7528");
|
|
20893
|
+
return;
|
|
20894
|
+
}
|
|
20895
|
+
try {
|
|
20896
|
+
await new Promise((resolve3, reject) => {
|
|
20897
|
+
child.stdin.end("START\n", (error) => {
|
|
20898
|
+
if (error) reject(error);
|
|
20899
|
+
else resolve3();
|
|
20900
|
+
});
|
|
20901
|
+
});
|
|
20902
|
+
} catch (error) {
|
|
20903
|
+
await this.terminateForFailure(
|
|
20904
|
+
entry,
|
|
20905
|
+
`\u653E\u884C OpenCode \u5931\u8D25\uFF1A${error instanceof Error ? error.message : String(error)}`
|
|
20906
|
+
);
|
|
20907
|
+
return;
|
|
20908
|
+
}
|
|
19756
20909
|
const timeoutMs = task.timeoutMs ?? this.cfg.taskTimeoutMs;
|
|
19757
20910
|
if (timeoutMs > 0) {
|
|
19758
20911
|
entry.timeoutTimer = setTimeout(() => {
|
|
19759
|
-
this.
|
|
19760
|
-
|
|
20912
|
+
this.runDetached(
|
|
20913
|
+
this.terminateForFailure(entry, `\u4EFB\u52A1\u8D85\u65F6\uFF08${timeoutMs}ms\uFF09`, "SIGKILL"),
|
|
20914
|
+
"timeout termination failed",
|
|
20915
|
+
entry.task.id
|
|
20916
|
+
);
|
|
19761
20917
|
}, timeoutMs);
|
|
19762
20918
|
}
|
|
19763
|
-
TaskRunService.updatePid(runId, process.pid, child.pid ?? 0).catch((err) => {
|
|
19764
|
-
this.signalEntry(entry, "SIGKILL");
|
|
19765
|
-
void this.finishEntry(entry, null, `\u8BB0\u5F55 Worker PID \u5931\u8D25\uFF1A${err instanceof Error ? err.message : String(err)}`);
|
|
19766
|
-
});
|
|
19767
20919
|
}
|
|
19768
|
-
|
|
19769
|
-
if (entry.settled) return;
|
|
19770
|
-
entry.
|
|
19771
|
-
|
|
19772
|
-
|
|
19773
|
-
|
|
19774
|
-
|
|
19775
|
-
|
|
19776
|
-
|
|
19777
|
-
|
|
19778
|
-
|
|
19779
|
-
|
|
19780
|
-
|
|
19781
|
-
|
|
19782
|
-
|
|
19783
|
-
|
|
19784
|
-
|
|
19785
|
-
await TaskRunService.done(entry.runId, log);
|
|
19786
|
-
console.log(JSON.stringify({
|
|
19787
|
-
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
19788
|
-
level: "info",
|
|
19789
|
-
msg: "task done",
|
|
19790
|
-
taskId: entry.task.id
|
|
19791
|
-
}));
|
|
19792
|
-
return;
|
|
19793
|
-
}
|
|
19794
|
-
await TaskRunService.fail(entry.runId, "\u4EFB\u52A1\u72B6\u6001\u5DF2\u88AB\u5176\u4ED6\u64CD\u4F5C\u6539\u53D8");
|
|
20920
|
+
handleChildClose(entry, code, signal, spawnError) {
|
|
20921
|
+
if (entry.settled || entry.termination) return;
|
|
20922
|
+
if (!entry.guardianDrained) {
|
|
20923
|
+
const pid = entry.child.pid;
|
|
20924
|
+
const presence = pid == null ? spawnError == null ? "unknown" : "not-running" : inspectSpawnedProcessTreePresence(pid);
|
|
20925
|
+
const message = "guardian \u672A\u63D0\u4F9B\u8FDB\u7A0B\u6811\u6392\u7A7A\u8BC1\u660E";
|
|
20926
|
+
if (presence !== "not-running") {
|
|
20927
|
+
if (entry.timeoutTimer) {
|
|
20928
|
+
clearTimeout(entry.timeoutTimer);
|
|
20929
|
+
entry.timeoutTimer = null;
|
|
20930
|
+
}
|
|
20931
|
+
entry.termination = { kind: "failure", message };
|
|
20932
|
+
entry.quarantined = true;
|
|
20933
|
+
markGatewayFailure(
|
|
20934
|
+
"worker",
|
|
20935
|
+
new Error(`${message}\uFF1B\u4EFB\u52A1 #${entry.task.id} \u4FDD\u6301\u9694\u79BB`)
|
|
20936
|
+
);
|
|
19795
20937
|
return;
|
|
19796
20938
|
}
|
|
19797
|
-
|
|
19798
|
-
|
|
19799
|
-
|
|
19800
|
-
|
|
19801
|
-
|
|
19802
|
-
|
|
19803
|
-
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
19804
|
-
level: "error",
|
|
19805
|
-
msg: "task failed",
|
|
19806
|
-
taskId: entry.task.id,
|
|
19807
|
-
error: failure
|
|
19808
|
-
}));
|
|
19809
|
-
} finally {
|
|
19810
|
-
this.runningTasks.delete(entry.task.id);
|
|
19811
|
-
this.releaseBatch(entry.task);
|
|
19812
|
-
}
|
|
19813
|
-
}
|
|
19814
|
-
async reconcileCancelledTasks() {
|
|
19815
|
-
for (const entry of [...this.runningTasks.values()]) {
|
|
19816
|
-
try {
|
|
19817
|
-
const task = await TaskService.getById(entry.task.id);
|
|
19818
|
-
if (task?.status === "cancelled") await this.cancelEntry(entry);
|
|
19819
|
-
} catch (err) {
|
|
19820
|
-
this.logError("cancel reconciliation failed", err, entry.task.id);
|
|
19821
|
-
}
|
|
20939
|
+
this.runDetached(
|
|
20940
|
+
this.settleEntry(entry, null, message),
|
|
20941
|
+
"task settlement failed",
|
|
20942
|
+
entry.task.id
|
|
20943
|
+
);
|
|
20944
|
+
return;
|
|
19822
20945
|
}
|
|
20946
|
+
const failure = code === 0 ? void 0 : `${spawnError ? "\u65E0\u6CD5\u542F\u52A8 opencode" : "opencode \u9000\u51FA\u7801"} ${spawnError?.message ?? code ?? "null"}${signal ? `\uFF0C\u4FE1\u53F7 ${signal}` : ""}`;
|
|
20947
|
+
this.runDetached(
|
|
20948
|
+
this.settleEntry(entry, code, failure),
|
|
20949
|
+
"task settlement failed",
|
|
20950
|
+
entry.task.id
|
|
20951
|
+
);
|
|
19823
20952
|
}
|
|
19824
|
-
|
|
19825
|
-
if (entry.
|
|
20953
|
+
settleEntry(entry, code, failure) {
|
|
20954
|
+
if (entry.settlementPromise) return entry.settlementPromise;
|
|
20955
|
+
if (entry.settled) return Promise.resolve(false);
|
|
19826
20956
|
entry.settled = true;
|
|
19827
20957
|
if (entry.timeoutTimer) {
|
|
19828
20958
|
clearTimeout(entry.timeoutTimer);
|
|
19829
20959
|
entry.timeoutTimer = null;
|
|
19830
20960
|
}
|
|
19831
|
-
|
|
19832
|
-
|
|
19833
|
-
|
|
19834
|
-
|
|
19835
|
-
|
|
19836
|
-
|
|
20961
|
+
const settlement = this.commitEntry(entry, code, failure).then(() => true).catch((error) => {
|
|
20962
|
+
markGatewayFailure("worker", error);
|
|
20963
|
+
this.logError("task settlement failed", error, entry.task.id);
|
|
20964
|
+
return false;
|
|
20965
|
+
}).finally(() => {
|
|
20966
|
+
this.runningTasks.delete(entry.task.id);
|
|
20967
|
+
this.releaseBatch(entry.task);
|
|
20968
|
+
});
|
|
20969
|
+
entry.settlementPromise = settlement;
|
|
20970
|
+
return settlement;
|
|
20971
|
+
}
|
|
20972
|
+
async commitEntry(entry, code, failure) {
|
|
20973
|
+
const termination = entry.termination;
|
|
20974
|
+
if (termination?.kind === "shutdown") return;
|
|
20975
|
+
if (termination?.kind === "cancel") {
|
|
20976
|
+
const output2 = entry.output.trim();
|
|
20977
|
+
const log2 = `${termination.message}${output2 ? `
|
|
20978
|
+
${output2}` : ""}`;
|
|
20979
|
+
await TaskRunService.fail(entry.runId, log2);
|
|
19837
20980
|
console.log(JSON.stringify({
|
|
19838
20981
|
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
19839
20982
|
level: "info",
|
|
19840
20983
|
msg: "running task cancelled",
|
|
19841
20984
|
taskId: entry.task.id
|
|
19842
20985
|
}));
|
|
19843
|
-
|
|
19844
|
-
|
|
19845
|
-
|
|
20986
|
+
return;
|
|
20987
|
+
}
|
|
20988
|
+
const currentRun = await TaskRunService.getById(entry.runId);
|
|
20989
|
+
if (!currentRun || currentRun.status !== "running") return;
|
|
20990
|
+
const output = entry.output.trim();
|
|
20991
|
+
const log = failure ? `${failure}${output ? `
|
|
20992
|
+
${output}` : ""}` : output;
|
|
20993
|
+
if (code === 0 && !failure) {
|
|
20994
|
+
const completed = await TaskService.completeRun(entry.task.id, entry.runId, log);
|
|
20995
|
+
if (completed) {
|
|
20996
|
+
console.log(JSON.stringify({
|
|
20997
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
20998
|
+
level: "info",
|
|
20999
|
+
msg: "task done",
|
|
21000
|
+
taskId: entry.task.id
|
|
21001
|
+
}));
|
|
21002
|
+
return;
|
|
21003
|
+
}
|
|
21004
|
+
await TaskRunService.fail(entry.runId, "\u4EFB\u52A1\u6216\u6267\u884C\u8BB0\u5F55\u72B6\u6001\u5DF2\u88AB\u5176\u4ED6\u64CD\u4F5C\u6539\u53D8");
|
|
21005
|
+
return;
|
|
21006
|
+
}
|
|
21007
|
+
const failed = await TaskService.failRun(entry.task.id, entry.runId, log);
|
|
21008
|
+
if (!failed) {
|
|
21009
|
+
await TaskRunService.fail(
|
|
21010
|
+
entry.runId,
|
|
21011
|
+
`${log}${log ? "\n" : ""}\u4EFB\u52A1\u72B6\u6001\u5DF2\u88AB\u5176\u4ED6\u64CD\u4F5C\u6539\u53D8`
|
|
21012
|
+
);
|
|
21013
|
+
this.logError("task failure state transition rejected", failure ?? "unknown failure", entry.task.id);
|
|
21014
|
+
}
|
|
21015
|
+
console.error(JSON.stringify({
|
|
21016
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
21017
|
+
level: "error",
|
|
21018
|
+
msg: "task failed",
|
|
21019
|
+
taskId: entry.task.id,
|
|
21020
|
+
error: failure
|
|
21021
|
+
}));
|
|
21022
|
+
}
|
|
21023
|
+
async reconcileCancelledTasks() {
|
|
21024
|
+
for (const entry of [...this.runningTasks.values()]) {
|
|
21025
|
+
try {
|
|
21026
|
+
const task = await TaskService.getById(entry.task.id);
|
|
21027
|
+
if (task?.status === "cancelled") await this.cancelEntry(entry);
|
|
21028
|
+
} catch (err) {
|
|
21029
|
+
this.logError("cancel reconciliation failed", err, entry.task.id);
|
|
21030
|
+
}
|
|
21031
|
+
}
|
|
21032
|
+
}
|
|
21033
|
+
async reconcileQuarantinedTasks() {
|
|
21034
|
+
for (const entry of [...this.runningTasks.values()]) {
|
|
21035
|
+
if (!entry.quarantined || !entry.termination) continue;
|
|
21036
|
+
await this.terminateEntry(entry, entry.termination);
|
|
19846
21037
|
}
|
|
19847
21038
|
}
|
|
21039
|
+
async cancelEntry(entry) {
|
|
21040
|
+
await this.terminateEntry(entry, { kind: "cancel", message: "\u4EFB\u52A1\u5DF2\u53D6\u6D88" });
|
|
21041
|
+
}
|
|
19848
21042
|
async updateHeartbeats() {
|
|
19849
21043
|
for (const entry of this.runningTasks.values()) {
|
|
21044
|
+
if (entry.quarantined) continue;
|
|
19850
21045
|
try {
|
|
19851
21046
|
await TaskRunService.heartbeat(entry.runId);
|
|
19852
21047
|
} catch (err) {
|
|
@@ -19854,26 +21049,75 @@ ${output}` : ""}`;
|
|
|
19854
21049
|
}
|
|
19855
21050
|
}
|
|
19856
21051
|
}
|
|
19857
|
-
|
|
19858
|
-
|
|
19859
|
-
|
|
21052
|
+
async terminateForFailure(entry, message, initialSignal = "SIGTERM") {
|
|
21053
|
+
return this.terminateEntry(entry, { kind: "failure", message }, initialSignal);
|
|
21054
|
+
}
|
|
21055
|
+
terminateEntry(entry, termination, initialSignal = "SIGTERM") {
|
|
21056
|
+
if (entry.terminationPromise) return entry.terminationPromise;
|
|
21057
|
+
if (entry.settled) return entry.settlementPromise ?? Promise.resolve(false);
|
|
21058
|
+
entry.termination ??= termination;
|
|
21059
|
+
if (entry.timeoutTimer) {
|
|
21060
|
+
clearTimeout(entry.timeoutTimer);
|
|
21061
|
+
entry.timeoutTimer = null;
|
|
19860
21062
|
}
|
|
19861
|
-
|
|
19862
|
-
|
|
19863
|
-
|
|
19864
|
-
|
|
19865
|
-
}
|
|
19866
|
-
entry.child.once("close", () => {
|
|
19867
|
-
clearTimeout(timeout);
|
|
19868
|
-
resolve2();
|
|
19869
|
-
});
|
|
19870
|
-
this.signalEntry(entry, "SIGTERM");
|
|
21063
|
+
entry.quarantined = true;
|
|
21064
|
+
const terminationPromise = this.completeTermination(entry, initialSignal).finally(() => {
|
|
21065
|
+
if (!entry.settled && entry.terminationPromise === terminationPromise) {
|
|
21066
|
+
entry.terminationPromise = null;
|
|
21067
|
+
}
|
|
19871
21068
|
});
|
|
21069
|
+
entry.terminationPromise = terminationPromise;
|
|
21070
|
+
return terminationPromise;
|
|
21071
|
+
}
|
|
21072
|
+
async completeTermination(entry, initialSignal) {
|
|
21073
|
+
const termination = entry.termination;
|
|
21074
|
+
if (!termination) return false;
|
|
21075
|
+
try {
|
|
21076
|
+
const exited = await this.killEntry(entry, initialSignal);
|
|
21077
|
+
if (!exited) {
|
|
21078
|
+
markGatewayFailure(
|
|
21079
|
+
"worker",
|
|
21080
|
+
new Error(`${termination.message}\uFF1B\u4EFB\u52A1 #${entry.task.id} \u7684\u8FDB\u7A0B\u65E0\u6CD5\u786E\u8BA4\u9000\u51FA`)
|
|
21081
|
+
);
|
|
21082
|
+
return false;
|
|
21083
|
+
}
|
|
21084
|
+
entry.quarantined = false;
|
|
21085
|
+
return await this.settleEntry(
|
|
21086
|
+
entry,
|
|
21087
|
+
null,
|
|
21088
|
+
termination.kind === "failure" ? termination.message : void 0
|
|
21089
|
+
);
|
|
21090
|
+
} catch (error) {
|
|
21091
|
+
markGatewayFailure("worker", error);
|
|
21092
|
+
this.logError("task termination failed", error, entry.task.id);
|
|
21093
|
+
return false;
|
|
21094
|
+
}
|
|
21095
|
+
}
|
|
21096
|
+
async killEntry(entry, initialSignal = "SIGTERM") {
|
|
21097
|
+
if (entry.guardianDrained && (entry.child.exitCode !== null || entry.child.signalCode !== null)) return true;
|
|
21098
|
+
const pid = entry.child.pid;
|
|
21099
|
+
if (!pid) return false;
|
|
21100
|
+
const initialResult = this.signalEntry(entry, initialSignal);
|
|
21101
|
+
if (initialResult === "not-running") return true;
|
|
21102
|
+
if (initialResult !== "signalled") return false;
|
|
21103
|
+
if (await waitForSpawnedProcessTreeExit(pid, 2500)) return true;
|
|
21104
|
+
if (initialSignal === "SIGKILL") return false;
|
|
21105
|
+
if (entry.guardianDrained && (entry.child.exitCode !== null || entry.child.signalCode !== null)) return true;
|
|
21106
|
+
const forcedResult = this.signalEntry(entry, "SIGKILL");
|
|
21107
|
+
if (forcedResult === "not-running") return true;
|
|
21108
|
+
if (forcedResult !== "signalled") return false;
|
|
21109
|
+
return waitForSpawnedProcessTreeExit(pid, 2500);
|
|
19872
21110
|
}
|
|
19873
21111
|
signalEntry(entry, signal) {
|
|
19874
21112
|
const pid = entry.child.pid;
|
|
19875
|
-
if (!pid) return;
|
|
19876
|
-
|
|
21113
|
+
if (!pid) return "not-running";
|
|
21114
|
+
return signalRecordedProcessTreeWithResult(
|
|
21115
|
+
pid,
|
|
21116
|
+
signal,
|
|
21117
|
+
this.opencodeBin,
|
|
21118
|
+
TOKEN_GUARDIAN_LAUNCH_PROTOCOL,
|
|
21119
|
+
entry.launchIdentity
|
|
21120
|
+
);
|
|
19877
21121
|
}
|
|
19878
21122
|
releaseBatch(task) {
|
|
19879
21123
|
if (task.batchId) this.activeBatchIds.delete(task.batchId);
|
|
@@ -19882,6 +21126,12 @@ ${output}` : ""}`;
|
|
|
19882
21126
|
if (!taskModel || taskModel === "default") return null;
|
|
19883
21127
|
return taskModel;
|
|
19884
21128
|
}
|
|
21129
|
+
runDetached(operation, message, taskId) {
|
|
21130
|
+
operation.catch((error) => {
|
|
21131
|
+
markGatewayFailure("worker", error);
|
|
21132
|
+
this.logError(message, error, taskId);
|
|
21133
|
+
});
|
|
21134
|
+
}
|
|
19885
21135
|
logError(message, error, taskId) {
|
|
19886
21136
|
console.error(JSON.stringify({
|
|
19887
21137
|
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
@@ -19896,56 +21146,136 @@ ${output}` : ""}`;
|
|
|
19896
21146
|
// src/gateway/watchdog/heartbeat.ts
|
|
19897
21147
|
init_task_run_service();
|
|
19898
21148
|
init_task_service();
|
|
19899
|
-
init_backoff();
|
|
19900
21149
|
init_process_control();
|
|
19901
|
-
|
|
21150
|
+
init_launch_protocol();
|
|
21151
|
+
async function checkHeartbeats(heartbeatTimeoutMs, isOwnedRun = () => false, shouldStop = () => false) {
|
|
19902
21152
|
const staleRuns = await TaskRunService.getStaleRuns(heartbeatTimeoutMs);
|
|
19903
|
-
|
|
21153
|
+
const result = {
|
|
21154
|
+
staleRuns: staleRuns.length,
|
|
21155
|
+
recoveredRuns: 0,
|
|
21156
|
+
quarantinedRuns: 0,
|
|
21157
|
+
failedRuns: 0
|
|
21158
|
+
};
|
|
21159
|
+
if (staleRuns.length === 0) return result;
|
|
19904
21160
|
for (const run of staleRuns) {
|
|
21161
|
+
await Bun.sleep(1);
|
|
21162
|
+
if (shouldStop()) break;
|
|
21163
|
+
if (isOwnedRun(run.taskId, run.runId)) continue;
|
|
19905
21164
|
try {
|
|
19906
|
-
if (run.
|
|
21165
|
+
if (run.launchProtocol != null && run.launchProtocol !== LEGACY_GUARDIAN_LAUNCH_PROTOCOL && run.launchProtocol !== TOKEN_GUARDIAN_LAUNCH_PROTOCOL) {
|
|
21166
|
+
console.warn(JSON.stringify({
|
|
21167
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
21168
|
+
level: "warn",
|
|
21169
|
+
msg: "unknown launch protocol remains quarantined",
|
|
21170
|
+
taskId: run.taskId,
|
|
21171
|
+
runId: run.runId,
|
|
21172
|
+
launchProtocol: run.launchProtocol,
|
|
21173
|
+
remediation: "\u5347\u7EA7\u5230\u652F\u6301\u8BE5\u534F\u8BAE\u7684 SuperTask \u7248\u672C\uFF1B\u7981\u6B62\u4F7F\u7528 run abandon \u7ED5\u8FC7\u672A\u77E5\u534F\u8BAE"
|
|
21174
|
+
}));
|
|
21175
|
+
result.quarantinedRuns += 1;
|
|
21176
|
+
continue;
|
|
21177
|
+
}
|
|
21178
|
+
if (run.childPid != null && run.launchProtocol !== TOKEN_GUARDIAN_LAUNCH_PROTOCOL) {
|
|
21179
|
+
if (!isProcessAlive(run.childPid) && !isSpawnedProcessTreeAlive(run.childPid)) {
|
|
21180
|
+
} else {
|
|
21181
|
+
console.warn(JSON.stringify({
|
|
21182
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
21183
|
+
level: "warn",
|
|
21184
|
+
msg: "stale run has no per-run process identity; run remains quarantined",
|
|
21185
|
+
taskId: run.taskId,
|
|
21186
|
+
runId: run.runId,
|
|
21187
|
+
childPid: run.childPid,
|
|
21188
|
+
launchProtocol: run.launchProtocol,
|
|
21189
|
+
remediation: "\u65E7\u542F\u52A8\u534F\u8BAE\u4E0D\u80FD\u6392\u9664 PID \u590D\u7528\uFF0C\u7981\u6B62\u81EA\u52A8\u7EC8\u6B62\uFF1B\u8BF7\u4EBA\u5DE5\u786E\u8BA4\u8FDB\u7A0B\u4E0E\u6570\u636E\u72B6\u6001"
|
|
21190
|
+
}));
|
|
21191
|
+
result.quarantinedRuns += 1;
|
|
21192
|
+
continue;
|
|
21193
|
+
}
|
|
21194
|
+
}
|
|
21195
|
+
if (run.childPid == null && run.launchProtocol == null) {
|
|
21196
|
+
console.warn(JSON.stringify({
|
|
21197
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
21198
|
+
level: "warn",
|
|
21199
|
+
msg: "legacy stale run has no recorded child pid; run remains quarantined",
|
|
21200
|
+
taskId: run.taskId,
|
|
21201
|
+
runId: run.runId,
|
|
21202
|
+
remediation: {
|
|
21203
|
+
taskCwd: run.taskCwd,
|
|
21204
|
+
owner: run.ownerAlive ? `owner PID ${run.workerPid} \u4ECD\u5B58\u6D3B\uFF0C\u5FC5\u987B\u5148\u786E\u8BA4\u5E76\u505C\u6B62\u5BF9\u5E94\u8FDB\u7A0B` : "owner PID \u5DF2\u9000\u51FA\u6216\u672A\u8BB0\u5F55",
|
|
21205
|
+
cancelCommand: run.taskStatus === "cancelled" ? null : run.taskCwd == null ? `\u4EFB\u52A1\u6CA1\u6709\u8BB0\u5F55 cwd\uFF0C\u8BF7\u5728 Dashboard \u53D6\u6D88\u4EFB\u52A1 #${run.taskId}` : `\u5728 ${run.taskCwd} \u6267\u884C: supertask cancel --id ${run.taskId}`,
|
|
21206
|
+
abandonCommand: `\u786E\u8BA4\u6CA1\u6709\u9057\u7559 OpenCode \u8FDB\u7A0B\u540E\u6267\u884C: supertask run abandon --id ${run.runId} --confirm ABANDON`
|
|
21207
|
+
}
|
|
21208
|
+
}));
|
|
21209
|
+
result.quarantinedRuns += 1;
|
|
21210
|
+
continue;
|
|
21211
|
+
}
|
|
21212
|
+
if (run.childPid != null && run.childPid > 0 && run.launchProtocol === TOKEN_GUARDIAN_LAUNCH_PROTOCOL) {
|
|
19907
21213
|
const expectedExecutable = process.env.SUPERTASK_OPENCODE_BIN ?? "opencode";
|
|
19908
|
-
const
|
|
19909
|
-
|
|
21214
|
+
const signalResult = signalRecordedProcessTreeWithResult(
|
|
21215
|
+
run.childPid,
|
|
21216
|
+
"SIGKILL",
|
|
21217
|
+
expectedExecutable,
|
|
21218
|
+
run.launchProtocol,
|
|
21219
|
+
run.lockedBy
|
|
21220
|
+
);
|
|
21221
|
+
if (signalResult === "signalled") {
|
|
21222
|
+
const exited = await waitForSpawnedProcessTreeExit(run.childPid);
|
|
21223
|
+
if (!exited) {
|
|
21224
|
+
console.warn(JSON.stringify({
|
|
21225
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
21226
|
+
level: "warn",
|
|
21227
|
+
msg: "stale child process did not exit; task remains quarantined",
|
|
21228
|
+
taskId: run.taskId,
|
|
21229
|
+
runId: run.runId,
|
|
21230
|
+
childPid: run.childPid
|
|
21231
|
+
}));
|
|
21232
|
+
result.quarantinedRuns += 1;
|
|
21233
|
+
continue;
|
|
21234
|
+
}
|
|
21235
|
+
} else if (signalResult !== "not-running") {
|
|
19910
21236
|
console.warn(JSON.stringify({
|
|
19911
21237
|
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
19912
21238
|
level: "warn",
|
|
19913
|
-
msg: "stale child process
|
|
21239
|
+
msg: "stale child process could not be safely terminated; task remains quarantined",
|
|
19914
21240
|
taskId: run.taskId,
|
|
19915
21241
|
runId: run.runId,
|
|
19916
|
-
childPid: run.childPid
|
|
21242
|
+
childPid: run.childPid,
|
|
21243
|
+
reason: signalResult
|
|
19917
21244
|
}));
|
|
21245
|
+
result.quarantinedRuns += 1;
|
|
21246
|
+
continue;
|
|
19918
21247
|
}
|
|
19919
21248
|
}
|
|
19920
|
-
await
|
|
19921
|
-
|
|
19922
|
-
|
|
19923
|
-
|
|
19924
|
-
|
|
21249
|
+
const recovery = await TaskService.recoverRun(
|
|
21250
|
+
run.taskId,
|
|
21251
|
+
run.runId,
|
|
21252
|
+
`\u6267\u884C\u6240\u6709\u8005\u9000\u51FA\u6216\u5FC3\u8DF3\u8D85\u65F6 (${heartbeatTimeoutMs / 1e3}s)\uFF0CWatchdog \u5DF2\u786E\u8BA4\u5B50\u8FDB\u7A0B\u7ED3\u675F`
|
|
21253
|
+
);
|
|
21254
|
+
if (!recovery) continue;
|
|
21255
|
+
result.recoveredRuns += 1;
|
|
21256
|
+
if (recovery.status === "dead_letter") {
|
|
19925
21257
|
console.log(JSON.stringify({
|
|
19926
21258
|
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
19927
21259
|
level: "warn",
|
|
19928
21260
|
msg: "task dead_letter",
|
|
19929
21261
|
taskId: run.taskId,
|
|
19930
21262
|
runId: run.runId,
|
|
19931
|
-
retryCount:
|
|
19932
|
-
maxRetries
|
|
21263
|
+
retryCount: recovery.retryCount,
|
|
21264
|
+
maxRetries: run.taskMaxRetries
|
|
19933
21265
|
}));
|
|
19934
21266
|
} else {
|
|
19935
|
-
const backoffMs = computeBackoff(newRetryCount, run.taskRetryBackoffMs);
|
|
19936
|
-
const retryAfter = Date.now() + backoffMs;
|
|
19937
|
-
await TaskService.markPendingForRetry(run.taskId, retryAfter, newRetryCount);
|
|
19938
21267
|
console.log(JSON.stringify({
|
|
19939
21268
|
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
19940
21269
|
level: "warn",
|
|
19941
21270
|
msg: "task retry scheduled",
|
|
19942
21271
|
taskId: run.taskId,
|
|
19943
21272
|
runId: run.runId,
|
|
19944
|
-
retryCount:
|
|
19945
|
-
retryAfterMs:
|
|
21273
|
+
retryCount: recovery.retryCount,
|
|
21274
|
+
retryAfterMs: recovery.retryAfterMs == null ? null : Math.max(0, recovery.retryAfterMs - Date.now())
|
|
19946
21275
|
}));
|
|
19947
21276
|
}
|
|
19948
21277
|
} catch (err) {
|
|
21278
|
+
result.failedRuns += 1;
|
|
19949
21279
|
console.error(JSON.stringify({
|
|
19950
21280
|
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
19951
21281
|
level: "error",
|
|
@@ -19956,47 +21286,46 @@ async function checkHeartbeats(heartbeatTimeoutMs) {
|
|
|
19956
21286
|
}));
|
|
19957
21287
|
}
|
|
19958
21288
|
}
|
|
21289
|
+
return result;
|
|
19959
21290
|
}
|
|
19960
21291
|
|
|
19961
21292
|
// src/gateway/watchdog/cleanup.ts
|
|
19962
21293
|
init_task_service();
|
|
19963
|
-
|
|
19964
|
-
|
|
19965
|
-
|
|
19966
|
-
|
|
19967
|
-
|
|
19968
|
-
|
|
19969
|
-
const oldTasks = await db.select({ id: tasksTable.id }).from(tasksTable).where(
|
|
19970
|
-
and(
|
|
19971
|
-
sql`${tasksTable.status} IN ('done', 'failed', 'dead_letter')`,
|
|
19972
|
-
sql`${tasksTable.finishedAt} IS NOT NULL`,
|
|
19973
|
-
sql`${tasksTable.finishedAt} < ${cutoffSec}`
|
|
19974
|
-
)
|
|
21294
|
+
init_task_template_service();
|
|
21295
|
+
async function cleanupOldRecords(retentionDays, shouldStop = () => false) {
|
|
21296
|
+
const deletedTasks = await TaskService.deleteOlderThan(retentionDays, shouldStop);
|
|
21297
|
+
const deletedDelayedTemplates = await TaskTemplateService.deleteExpiredDelayed(
|
|
21298
|
+
retentionDays,
|
|
21299
|
+
shouldStop
|
|
19975
21300
|
);
|
|
19976
|
-
if (
|
|
19977
|
-
|
|
19978
|
-
|
|
19979
|
-
|
|
19980
|
-
|
|
19981
|
-
|
|
19982
|
-
|
|
19983
|
-
|
|
19984
|
-
|
|
19985
|
-
deletedRuns
|
|
19986
|
-
}));
|
|
21301
|
+
if (deletedTasks > 0 || deletedDelayedTemplates > 0) {
|
|
21302
|
+
console.log(JSON.stringify({
|
|
21303
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
21304
|
+
level: "info",
|
|
21305
|
+
msg: "cleanup completed",
|
|
21306
|
+
deletedTasks,
|
|
21307
|
+
deletedDelayedTemplates
|
|
21308
|
+
}));
|
|
21309
|
+
}
|
|
19987
21310
|
return deletedTasks;
|
|
19988
21311
|
}
|
|
19989
21312
|
|
|
19990
21313
|
// src/gateway/watchdog/index.ts
|
|
19991
21314
|
init_health();
|
|
19992
21315
|
var Watchdog = class {
|
|
19993
|
-
constructor(cfg) {
|
|
21316
|
+
constructor(cfg, isOwnedRun = () => false) {
|
|
19994
21317
|
this.cfg = cfg;
|
|
21318
|
+
this.isOwnedRun = isOwnedRun;
|
|
19995
21319
|
}
|
|
19996
21320
|
cfg;
|
|
21321
|
+
isOwnedRun;
|
|
19997
21322
|
stopped = false;
|
|
19998
21323
|
heartbeatTimer = null;
|
|
19999
21324
|
cleanupTimer = null;
|
|
21325
|
+
checkingHeartbeats = false;
|
|
21326
|
+
cleaning = false;
|
|
21327
|
+
heartbeatCheckPromise = null;
|
|
21328
|
+
cleanupPromise = null;
|
|
20000
21329
|
start() {
|
|
20001
21330
|
this.stopped = false;
|
|
20002
21331
|
markGatewayActivity("watchdog");
|
|
@@ -20008,8 +21337,9 @@ var Watchdog = class {
|
|
|
20008
21337
|
() => this.runCleanup(),
|
|
20009
21338
|
this.cfg.watchdog.cleanupIntervalMs
|
|
20010
21339
|
);
|
|
21340
|
+
void this.runHeartbeatCheck();
|
|
20011
21341
|
}
|
|
20012
|
-
stop() {
|
|
21342
|
+
async stop() {
|
|
20013
21343
|
this.stopped = true;
|
|
20014
21344
|
if (this.heartbeatTimer) {
|
|
20015
21345
|
clearInterval(this.heartbeatTimer);
|
|
@@ -20019,108 +21349,79 @@ var Watchdog = class {
|
|
|
20019
21349
|
clearInterval(this.cleanupTimer);
|
|
20020
21350
|
this.cleanupTimer = null;
|
|
20021
21351
|
}
|
|
21352
|
+
await Promise.all([
|
|
21353
|
+
this.heartbeatCheckPromise,
|
|
21354
|
+
this.cleanupPromise
|
|
21355
|
+
]);
|
|
20022
21356
|
}
|
|
20023
|
-
|
|
20024
|
-
if (this.stopped) return;
|
|
21357
|
+
runHeartbeatCheck() {
|
|
21358
|
+
if (this.stopped) return Promise.resolve();
|
|
21359
|
+
if (this.heartbeatCheckPromise) return this.heartbeatCheckPromise;
|
|
21360
|
+
this.checkingHeartbeats = true;
|
|
20025
21361
|
markGatewayActivity("watchdog");
|
|
20026
|
-
|
|
20027
|
-
|
|
20028
|
-
|
|
20029
|
-
|
|
20030
|
-
|
|
20031
|
-
|
|
20032
|
-
|
|
20033
|
-
|
|
20034
|
-
|
|
20035
|
-
|
|
21362
|
+
const operation = (async () => {
|
|
21363
|
+
try {
|
|
21364
|
+
const result = await checkHeartbeats(
|
|
21365
|
+
this.cfg.watchdog.heartbeatTimeoutMs,
|
|
21366
|
+
this.isOwnedRun,
|
|
21367
|
+
() => this.stopped
|
|
21368
|
+
);
|
|
21369
|
+
if (result.quarantinedRuns > 0 || result.failedRuns > 0) {
|
|
21370
|
+
throw new Error(
|
|
21371
|
+
`Watchdog \u6709 ${result.quarantinedRuns} \u4E2A\u9694\u79BB run\u3001${result.failedRuns} \u4E2A\u6062\u590D\u5931\u8D25 run`
|
|
21372
|
+
);
|
|
21373
|
+
}
|
|
21374
|
+
markGatewaySuccess("watchdog");
|
|
21375
|
+
} catch (err) {
|
|
21376
|
+
markGatewayFailure("watchdog", err);
|
|
21377
|
+
console.error(JSON.stringify({
|
|
21378
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
21379
|
+
level: "error",
|
|
21380
|
+
msg: "watchdog heartbeat check failed",
|
|
21381
|
+
error: err instanceof Error ? err.message : String(err)
|
|
21382
|
+
}));
|
|
21383
|
+
} finally {
|
|
21384
|
+
this.checkingHeartbeats = false;
|
|
21385
|
+
}
|
|
21386
|
+
})();
|
|
21387
|
+
this.heartbeatCheckPromise = operation.finally(() => {
|
|
21388
|
+
this.heartbeatCheckPromise = null;
|
|
21389
|
+
});
|
|
21390
|
+
return this.heartbeatCheckPromise;
|
|
20036
21391
|
}
|
|
20037
|
-
|
|
20038
|
-
if (this.stopped) return;
|
|
20039
|
-
|
|
20040
|
-
|
|
20041
|
-
|
|
20042
|
-
|
|
20043
|
-
|
|
20044
|
-
|
|
20045
|
-
|
|
20046
|
-
|
|
20047
|
-
|
|
20048
|
-
|
|
21392
|
+
runCleanup() {
|
|
21393
|
+
if (this.stopped) return Promise.resolve();
|
|
21394
|
+
if (this.cleanupPromise) return this.cleanupPromise;
|
|
21395
|
+
this.cleaning = true;
|
|
21396
|
+
markGatewayActivity("watchdogCleanup");
|
|
21397
|
+
const operation = (async () => {
|
|
21398
|
+
try {
|
|
21399
|
+
await cleanupOldRecords(
|
|
21400
|
+
this.cfg.watchdog.retentionDays,
|
|
21401
|
+
() => this.stopped
|
|
21402
|
+
);
|
|
21403
|
+
markGatewaySuccess("watchdogCleanup");
|
|
21404
|
+
} catch (err) {
|
|
21405
|
+
markGatewayFailure("watchdogCleanup", err);
|
|
21406
|
+
console.error(JSON.stringify({
|
|
21407
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
21408
|
+
level: "error",
|
|
21409
|
+
msg: "watchdog cleanup failed",
|
|
21410
|
+
error: err instanceof Error ? err.message : String(err)
|
|
21411
|
+
}));
|
|
21412
|
+
} finally {
|
|
21413
|
+
this.cleaning = false;
|
|
21414
|
+
}
|
|
21415
|
+
})();
|
|
21416
|
+
this.cleanupPromise = operation.finally(() => {
|
|
21417
|
+
this.cleanupPromise = null;
|
|
21418
|
+
});
|
|
21419
|
+
return this.cleanupPromise;
|
|
20049
21420
|
}
|
|
20050
21421
|
};
|
|
20051
21422
|
|
|
20052
|
-
// src/gateway/scheduler/job-templates.ts
|
|
20053
|
-
init_db2();
|
|
20054
|
-
init_drizzle_orm();
|
|
20055
|
-
init_task_service();
|
|
20056
|
-
init_task_template_service();
|
|
20057
|
-
var { taskTemplates: taskTemplates3 } = schema_exports;
|
|
20058
|
-
async function cloneTaskFromTemplate(templateId) {
|
|
20059
|
-
const rows = await db.select().from(taskTemplates3).where(eq(taskTemplates3.id, templateId)).limit(1);
|
|
20060
|
-
const tmpl = rows[0];
|
|
20061
|
-
if (!tmpl || !tmpl.enabled) return null;
|
|
20062
|
-
const activeCount = await db.select({ count: sql`count(*)` }).from(schema_exports.tasks).where(
|
|
20063
|
-
and(
|
|
20064
|
-
eq(schema_exports.tasks.templateId, templateId),
|
|
20065
|
-
sql`${schema_exports.tasks.status} IN ('pending', 'running')`
|
|
20066
|
-
)
|
|
20067
|
-
).then((r) => r[0].count);
|
|
20068
|
-
if (activeCount >= (tmpl.maxInstances ?? 1)) return null;
|
|
20069
|
-
const nowMs = Date.now();
|
|
20070
|
-
const isDelayed = tmpl.scheduleType === "delayed";
|
|
20071
|
-
const nextRunAt = isDelayed ? null : TaskTemplateService.calculateNextRunAt(
|
|
20072
|
-
tmpl.scheduleType,
|
|
20073
|
-
tmpl,
|
|
20074
|
-
nowMs
|
|
20075
|
-
);
|
|
20076
|
-
const task = await TaskService.add({
|
|
20077
|
-
name: tmpl.name,
|
|
20078
|
-
agent: tmpl.agent,
|
|
20079
|
-
model: tmpl.model ?? "default",
|
|
20080
|
-
prompt: tmpl.prompt,
|
|
20081
|
-
cwd: tmpl.cwd ?? null,
|
|
20082
|
-
category: tmpl.category ?? "general",
|
|
20083
|
-
importance: tmpl.importance ?? 3,
|
|
20084
|
-
urgency: tmpl.urgency ?? 3,
|
|
20085
|
-
batchId: tmpl.batchId,
|
|
20086
|
-
maxRetries: tmpl.maxRetries ?? 3,
|
|
20087
|
-
retryBackoffMs: tmpl.retryBackoffMs ?? 3e4,
|
|
20088
|
-
timeoutMs: tmpl.timeoutMs,
|
|
20089
|
-
templateId: tmpl.id,
|
|
20090
|
-
scheduledAt: nowMs
|
|
20091
|
-
});
|
|
20092
|
-
await db.update(taskTemplates3).set({
|
|
20093
|
-
lastRunAt: nowMs,
|
|
20094
|
-
nextRunAt,
|
|
20095
|
-
enabled: isDelayed ? false : tmpl.enabled,
|
|
20096
|
-
updatedAt: nowMs
|
|
20097
|
-
}).where(eq(taskTemplates3.id, templateId));
|
|
20098
|
-
return task;
|
|
20099
|
-
}
|
|
20100
|
-
async function getDueTemplates() {
|
|
20101
|
-
const nowMs = Date.now();
|
|
20102
|
-
return await db.select().from(taskTemplates3).where(
|
|
20103
|
-
and(
|
|
20104
|
-
eq(taskTemplates3.enabled, true),
|
|
20105
|
-
sql`${taskTemplates3.nextRunAt} IS NOT NULL`,
|
|
20106
|
-
sql`${taskTemplates3.nextRunAt} <= ${nowMs}`
|
|
20107
|
-
)
|
|
20108
|
-
).orderBy(asc(taskTemplates3.nextRunAt), asc(taskTemplates3.id));
|
|
20109
|
-
}
|
|
20110
|
-
async function initializeNextRunAt(templateId) {
|
|
20111
|
-
const rows = await db.select().from(taskTemplates3).where(eq(taskTemplates3.id, templateId)).limit(1);
|
|
20112
|
-
const tmpl = rows[0];
|
|
20113
|
-
if (!tmpl || tmpl.nextRunAt != null) return;
|
|
20114
|
-
const nextRunAt = TaskTemplateService.calculateNextRunAt(
|
|
20115
|
-
tmpl.scheduleType,
|
|
20116
|
-
tmpl
|
|
20117
|
-
);
|
|
20118
|
-
if (nextRunAt != null) {
|
|
20119
|
-
await db.update(taskTemplates3).set({ nextRunAt }).where(eq(taskTemplates3.id, templateId));
|
|
20120
|
-
}
|
|
20121
|
-
}
|
|
20122
|
-
|
|
20123
21423
|
// src/gateway/scheduler/index.ts
|
|
21424
|
+
init_job_templates();
|
|
20124
21425
|
init_db2();
|
|
20125
21426
|
init_drizzle_orm();
|
|
20126
21427
|
init_health();
|
|
@@ -20132,11 +21433,13 @@ var Scheduler = class {
|
|
|
20132
21433
|
stopped = false;
|
|
20133
21434
|
ticking = false;
|
|
20134
21435
|
timer = null;
|
|
21436
|
+
dueSweep = null;
|
|
20135
21437
|
async start() {
|
|
20136
21438
|
if (!this.cfg.scheduler.enabled) return;
|
|
20137
21439
|
this.stopped = false;
|
|
20138
21440
|
markGatewayActivity("scheduler");
|
|
20139
21441
|
await this.initializeTemplates();
|
|
21442
|
+
markGatewaySuccess("scheduler");
|
|
20140
21443
|
this.timer = setInterval(() => this.tick(), this.cfg.scheduler.checkIntervalMs);
|
|
20141
21444
|
}
|
|
20142
21445
|
stop() {
|
|
@@ -20150,8 +21453,11 @@ var Scheduler = class {
|
|
|
20150
21453
|
if (this.stopped || this.ticking) return;
|
|
20151
21454
|
markGatewayActivity("scheduler");
|
|
20152
21455
|
this.ticking = true;
|
|
21456
|
+
let hadFailure = false;
|
|
20153
21457
|
try {
|
|
20154
|
-
const
|
|
21458
|
+
const sweep = this.dueSweep ?? { cutoffNow: Date.now(), cursor: null };
|
|
21459
|
+
const dueTemplates = await getDueTemplates(sweep.cursor, sweep.cutoffNow);
|
|
21460
|
+
const lastTemplate = dueTemplates.at(-1);
|
|
20155
21461
|
for (const tmpl of dueTemplates) {
|
|
20156
21462
|
try {
|
|
20157
21463
|
const task = await cloneTaskFromTemplate(tmpl.id);
|
|
@@ -20165,6 +21471,8 @@ var Scheduler = class {
|
|
|
20165
21471
|
}));
|
|
20166
21472
|
}
|
|
20167
21473
|
} catch (err) {
|
|
21474
|
+
hadFailure = true;
|
|
21475
|
+
markGatewayFailure("scheduler", err);
|
|
20168
21476
|
console.error(JSON.stringify({
|
|
20169
21477
|
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
20170
21478
|
level: "error",
|
|
@@ -20174,7 +21482,13 @@ var Scheduler = class {
|
|
|
20174
21482
|
}));
|
|
20175
21483
|
}
|
|
20176
21484
|
}
|
|
21485
|
+
this.dueSweep = dueTemplates.length < DUE_TEMPLATE_BATCH_SIZE || lastTemplate?.nextRunAt == null ? null : {
|
|
21486
|
+
cutoffNow: sweep.cutoffNow,
|
|
21487
|
+
cursor: { nextRunAt: lastTemplate.nextRunAt, id: lastTemplate.id }
|
|
21488
|
+
};
|
|
21489
|
+
if (!hadFailure) markGatewaySuccess("scheduler");
|
|
20177
21490
|
} catch (err) {
|
|
21491
|
+
markGatewayFailure("scheduler", err);
|
|
20178
21492
|
console.error(JSON.stringify({
|
|
20179
21493
|
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
20180
21494
|
level: "error",
|
|
@@ -20187,9 +21501,22 @@ var Scheduler = class {
|
|
|
20187
21501
|
}
|
|
20188
21502
|
async initializeTemplates() {
|
|
20189
21503
|
const { taskTemplates: taskTemplates4 } = schema_exports;
|
|
20190
|
-
const
|
|
20191
|
-
|
|
20192
|
-
|
|
21504
|
+
const batchSize = 100;
|
|
21505
|
+
let cursor = 0;
|
|
21506
|
+
while (!this.stopped) {
|
|
21507
|
+
const templates = await db.select({ id: taskTemplates4.id }).from(taskTemplates4).where(and(
|
|
21508
|
+
eq(taskTemplates4.enabled, true),
|
|
21509
|
+
isNull(taskTemplates4.nextRunAt),
|
|
21510
|
+
gt(taskTemplates4.id, cursor)
|
|
21511
|
+
)).orderBy(asc(taskTemplates4.id)).limit(batchSize);
|
|
21512
|
+
if (templates.length === 0) break;
|
|
21513
|
+
for (const tmpl of templates) {
|
|
21514
|
+
if (this.stopped) break;
|
|
21515
|
+
await initializeNextRunAt(tmpl.id);
|
|
21516
|
+
}
|
|
21517
|
+
cursor = templates.at(-1).id;
|
|
21518
|
+
if (templates.length < batchSize) break;
|
|
21519
|
+
await Bun.sleep(0);
|
|
20193
21520
|
}
|
|
20194
21521
|
}
|
|
20195
21522
|
};
|
|
@@ -20197,10 +21524,45 @@ var Scheduler = class {
|
|
|
20197
21524
|
// src/gateway/index.ts
|
|
20198
21525
|
init_db2();
|
|
20199
21526
|
init_task_service();
|
|
20200
|
-
init_task_run_service();
|
|
20201
21527
|
init_health();
|
|
20202
21528
|
init_process_control();
|
|
21529
|
+
|
|
21530
|
+
// src/core/package-version.ts
|
|
21531
|
+
import { existsSync as existsSync4, readFileSync as readFileSync2 } from "fs";
|
|
21532
|
+
import { dirname as dirname4, join as join4 } from "path";
|
|
21533
|
+
import { fileURLToPath as fileURLToPath4 } from "url";
|
|
21534
|
+
function getPackageVersion() {
|
|
21535
|
+
let directory = dirname4(fileURLToPath4(import.meta.url));
|
|
21536
|
+
for (let depth = 0; depth < 5; depth += 1) {
|
|
21537
|
+
const packagePath = join4(directory, "package.json");
|
|
21538
|
+
if (existsSync4(packagePath)) {
|
|
21539
|
+
try {
|
|
21540
|
+
const pkg = JSON.parse(readFileSync2(packagePath, "utf8"));
|
|
21541
|
+
if (typeof pkg.version === "string") return pkg.version;
|
|
21542
|
+
} catch {
|
|
21543
|
+
return "0.0.0";
|
|
21544
|
+
}
|
|
21545
|
+
}
|
|
21546
|
+
directory = dirname4(directory);
|
|
21547
|
+
}
|
|
21548
|
+
return "0.0.0";
|
|
21549
|
+
}
|
|
21550
|
+
|
|
21551
|
+
// src/gateway/index.ts
|
|
20203
21552
|
var STALE_THRESHOLD_MS = 3e4;
|
|
21553
|
+
async function runGatewayShutdownStep(failures, step, operation) {
|
|
21554
|
+
try {
|
|
21555
|
+
await operation();
|
|
21556
|
+
} catch (error) {
|
|
21557
|
+
failures.push({
|
|
21558
|
+
step,
|
|
21559
|
+
error: error instanceof Error ? error.message : String(error)
|
|
21560
|
+
});
|
|
21561
|
+
}
|
|
21562
|
+
}
|
|
21563
|
+
function resolveGatewayShutdownExitCode(requestedExitCode, failures) {
|
|
21564
|
+
return failures.length > 0 ? 1 : requestedExitCode;
|
|
21565
|
+
}
|
|
20204
21566
|
function acquireLock() {
|
|
20205
21567
|
const now = Date.now();
|
|
20206
21568
|
const pid = process.pid;
|
|
@@ -20209,15 +21571,26 @@ function acquireLock() {
|
|
|
20209
21571
|
const existing = sqlite.prepare("SELECT id, pid, heartbeat_at FROM gateway_lock WHERE id = 1").get();
|
|
20210
21572
|
if (existing) {
|
|
20211
21573
|
const lockHolderAlive = existing.pid !== pid && isProcessAlive(existing.pid);
|
|
20212
|
-
if (
|
|
20213
|
-
|
|
20214
|
-
|
|
21574
|
+
if (lockHolderAlive) {
|
|
21575
|
+
const identity = identifyGatewayProcess(existing.pid);
|
|
21576
|
+
const lockIsFresh = now - existing.heartbeat_at < STALE_THRESHOLD_MS;
|
|
21577
|
+
if (lockIsFresh || identity !== "mismatch") {
|
|
21578
|
+
sqlite.exec("ROLLBACK");
|
|
21579
|
+
console.error(JSON.stringify({
|
|
21580
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
21581
|
+
level: "fatal",
|
|
21582
|
+
msg: "another Gateway instance is already running",
|
|
21583
|
+
existingPid: existing.pid,
|
|
21584
|
+
identity
|
|
21585
|
+
}));
|
|
21586
|
+
return false;
|
|
21587
|
+
}
|
|
21588
|
+
console.warn(JSON.stringify({
|
|
20215
21589
|
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
20216
|
-
level: "
|
|
20217
|
-
msg: "
|
|
21590
|
+
level: "warn",
|
|
21591
|
+
msg: "taking over stale Gateway lock from a reused PID",
|
|
20218
21592
|
existingPid: existing.pid
|
|
20219
21593
|
}));
|
|
20220
|
-
return false;
|
|
20221
21594
|
}
|
|
20222
21595
|
sqlite.exec("DELETE FROM gateway_lock WHERE id = 1");
|
|
20223
21596
|
}
|
|
@@ -20249,18 +21622,22 @@ function releaseLock() {
|
|
|
20249
21622
|
}
|
|
20250
21623
|
function updateLockHeartbeat() {
|
|
20251
21624
|
try {
|
|
20252
|
-
sqlite.
|
|
20253
|
-
"UPDATE gateway_lock SET heartbeat_at = ? WHERE pid = ?"
|
|
20254
|
-
|
|
20255
|
-
|
|
21625
|
+
const result = sqlite.query(
|
|
21626
|
+
"UPDATE gateway_lock SET heartbeat_at = ? WHERE id = 1 AND pid = ?"
|
|
21627
|
+
).run(Date.now(), process.pid);
|
|
21628
|
+
return result.changes === 1;
|
|
20256
21629
|
} catch {
|
|
21630
|
+
return false;
|
|
20257
21631
|
}
|
|
20258
21632
|
}
|
|
20259
21633
|
function markGatewayReady() {
|
|
20260
|
-
|
|
20261
|
-
|
|
20262
|
-
|
|
20263
|
-
);
|
|
21634
|
+
const now = Date.now();
|
|
21635
|
+
const result = sqlite.query(
|
|
21636
|
+
"UPDATE gateway_lock SET heartbeat_at = ?, ready_at = ?, version = ? WHERE id = 1 AND pid = ?"
|
|
21637
|
+
).run(now, now, getPackageVersion(), process.pid);
|
|
21638
|
+
if (result.changes !== 1) {
|
|
21639
|
+
throw new Error("Gateway \u5728\u5C31\u7EEA\u524D\u5931\u53BB\u4E86\u6570\u636E\u5E93\u5355\u5B9E\u4F8B\u9501");
|
|
21640
|
+
}
|
|
20264
21641
|
}
|
|
20265
21642
|
function markGatewayNotReady() {
|
|
20266
21643
|
try {
|
|
@@ -20270,89 +21647,159 @@ function markGatewayNotReady() {
|
|
|
20270
21647
|
}
|
|
20271
21648
|
async function main() {
|
|
20272
21649
|
console.log(JSON.stringify({ ts: (/* @__PURE__ */ new Date()).toISOString(), level: "info", msg: "SuperTask Gateway starting", pid: process.pid }));
|
|
21650
|
+
assertWorkerProcessIsolationSupported();
|
|
20273
21651
|
if (!acquireLock()) {
|
|
20274
21652
|
process.exit(1);
|
|
20275
21653
|
}
|
|
20276
|
-
const heartbeatTimer = setInterval(updateLockHeartbeat, 1e4);
|
|
20277
|
-
heartbeatTimer.unref();
|
|
20278
21654
|
const cfg = loadConfig();
|
|
20279
21655
|
const worker = new WorkerEngine(cfg);
|
|
20280
|
-
const watchdog = new Watchdog(cfg);
|
|
21656
|
+
const watchdog = new Watchdog(cfg, (taskId, runId) => worker.ownsRun(taskId, runId));
|
|
20281
21657
|
const scheduler = new Scheduler(cfg);
|
|
20282
|
-
|
|
20283
|
-
|
|
20284
|
-
console.log(JSON.stringify({
|
|
20285
|
-
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
20286
|
-
level: "warn",
|
|
20287
|
-
msg: "reset orphan running tasks to pending",
|
|
20288
|
-
count: recoveredOrphans
|
|
20289
|
-
}));
|
|
20290
|
-
}
|
|
20291
|
-
initializeGatewayHealth({
|
|
20292
|
-
workerPollIntervalMs: cfg.worker.pollIntervalMs,
|
|
20293
|
-
schedulerEnabled: cfg.scheduler.enabled,
|
|
20294
|
-
schedulerCheckIntervalMs: cfg.scheduler.checkIntervalMs,
|
|
20295
|
-
watchdogCheckIntervalMs: cfg.watchdog.checkIntervalMs
|
|
20296
|
-
});
|
|
20297
|
-
worker.start();
|
|
20298
|
-
watchdog.start();
|
|
20299
|
-
await scheduler.start();
|
|
20300
|
-
if (cfg.dashboard.enabled) {
|
|
20301
|
-
const { dashboardApp: dashboardApp2 } = await Promise.resolve().then(() => (init_web(), web_exports));
|
|
20302
|
-
Bun.serve({
|
|
20303
|
-
hostname: "127.0.0.1",
|
|
20304
|
-
port: cfg.dashboard.port,
|
|
20305
|
-
fetch: dashboardApp2.fetch
|
|
20306
|
-
});
|
|
20307
|
-
console.log(JSON.stringify({
|
|
20308
|
-
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
20309
|
-
level: "info",
|
|
20310
|
-
msg: "Dashboard started",
|
|
20311
|
-
url: `http://localhost:${cfg.dashboard.port}`
|
|
20312
|
-
}));
|
|
20313
|
-
}
|
|
20314
|
-
markGatewayReady();
|
|
20315
|
-
console.log(JSON.stringify({
|
|
20316
|
-
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
20317
|
-
level: "info",
|
|
20318
|
-
msg: "Gateway started",
|
|
20319
|
-
maxConcurrency: cfg.worker.maxConcurrency,
|
|
20320
|
-
schedulerEnabled: cfg.scheduler.enabled
|
|
20321
|
-
}));
|
|
21658
|
+
let heartbeatTimer = null;
|
|
21659
|
+
let dashboardServer = null;
|
|
20322
21660
|
let shuttingDown = false;
|
|
20323
|
-
const shutdown = async (signal) => {
|
|
21661
|
+
const shutdown = async (signal, exitCode = 0) => {
|
|
20324
21662
|
if (shuttingDown) return;
|
|
20325
21663
|
shuttingDown = true;
|
|
20326
|
-
|
|
20327
|
-
|
|
20328
|
-
|
|
20329
|
-
|
|
20330
|
-
|
|
20331
|
-
|
|
20332
|
-
|
|
20333
|
-
|
|
20334
|
-
|
|
20335
|
-
|
|
20336
|
-
|
|
20337
|
-
|
|
20338
|
-
|
|
21664
|
+
const failures = [];
|
|
21665
|
+
try {
|
|
21666
|
+
console.log(JSON.stringify({ ts: (/* @__PURE__ */ new Date()).toISOString(), level: "info", msg: `received ${signal}, shutting down...` }));
|
|
21667
|
+
if (heartbeatTimer) {
|
|
21668
|
+
clearInterval(heartbeatTimer);
|
|
21669
|
+
heartbeatTimer = null;
|
|
21670
|
+
}
|
|
21671
|
+
markGatewayNotReady();
|
|
21672
|
+
await runGatewayShutdownStep(failures, "dashboard.stop", () => {
|
|
21673
|
+
const server = dashboardServer;
|
|
21674
|
+
dashboardServer = null;
|
|
21675
|
+
server?.stop(true);
|
|
21676
|
+
});
|
|
21677
|
+
await runGatewayShutdownStep(failures, "scheduler.stop", () => scheduler.stop());
|
|
21678
|
+
await runGatewayShutdownStep(failures, "watchdog.stop", () => watchdog.stop());
|
|
21679
|
+
let interruptedRuns = [];
|
|
21680
|
+
await runGatewayShutdownStep(failures, "worker.stop", async () => {
|
|
21681
|
+
interruptedRuns = await worker.stop(cfg.worker.shutdownGracePeriodMs);
|
|
21682
|
+
});
|
|
21683
|
+
let resetCount = 0;
|
|
21684
|
+
for (const run of interruptedRuns) {
|
|
21685
|
+
await runGatewayShutdownStep(
|
|
21686
|
+
failures,
|
|
21687
|
+
`task.interrupt:${run.taskId}:${run.runId}`,
|
|
21688
|
+
async () => {
|
|
21689
|
+
if (await TaskService.interruptRun(
|
|
21690
|
+
run.taskId,
|
|
21691
|
+
run.runId,
|
|
21692
|
+
"Gateway shutdown after child exit"
|
|
21693
|
+
)) {
|
|
21694
|
+
resetCount += 1;
|
|
21695
|
+
}
|
|
21696
|
+
}
|
|
21697
|
+
);
|
|
21698
|
+
}
|
|
21699
|
+
if (resetCount > 0) {
|
|
21700
|
+
console.log(JSON.stringify({
|
|
21701
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
21702
|
+
level: "info",
|
|
21703
|
+
msg: "reset confirmed interrupted tasks to pending",
|
|
21704
|
+
count: resetCount
|
|
21705
|
+
}));
|
|
21706
|
+
}
|
|
21707
|
+
await runGatewayShutdownStep(failures, "lock.release", () => releaseLock());
|
|
21708
|
+
await runGatewayShutdownStep(failures, "health.reset", () => resetGatewayHealth());
|
|
21709
|
+
await runGatewayShutdownStep(failures, "database.close", () => closeDb());
|
|
21710
|
+
if (failures.length > 0) {
|
|
21711
|
+
console.error(JSON.stringify({
|
|
21712
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
21713
|
+
level: "error",
|
|
21714
|
+
msg: "Gateway stopped with shutdown failures",
|
|
21715
|
+
failures
|
|
21716
|
+
}));
|
|
21717
|
+
} else {
|
|
21718
|
+
console.log(JSON.stringify({ ts: (/* @__PURE__ */ new Date()).toISOString(), level: "info", msg: "Gateway stopped" }));
|
|
21719
|
+
}
|
|
21720
|
+
} finally {
|
|
21721
|
+
process.exit(resolveGatewayShutdownExitCode(exitCode, failures));
|
|
20339
21722
|
}
|
|
20340
|
-
releaseLock();
|
|
20341
|
-
resetGatewayHealth();
|
|
20342
|
-
closeDb();
|
|
20343
|
-
console.log(JSON.stringify({ ts: (/* @__PURE__ */ new Date()).toISOString(), level: "info", msg: "Gateway stopped" }));
|
|
20344
|
-
process.exit(0);
|
|
20345
21723
|
};
|
|
20346
|
-
process.on("SIGTERM", () =>
|
|
20347
|
-
|
|
21724
|
+
process.on("SIGTERM", () => {
|
|
21725
|
+
void shutdown("SIGTERM");
|
|
21726
|
+
});
|
|
21727
|
+
process.on("SIGINT", () => {
|
|
21728
|
+
void shutdown("SIGINT");
|
|
21729
|
+
});
|
|
20348
21730
|
process.on("uncaughtException", (err) => {
|
|
20349
21731
|
console.error(JSON.stringify({ ts: (/* @__PURE__ */ new Date()).toISOString(), level: "fatal", msg: "uncaughtException", error: err.message, stack: err.stack }));
|
|
20350
|
-
|
|
21732
|
+
void shutdown("uncaughtException", 1);
|
|
20351
21733
|
});
|
|
20352
21734
|
process.on("unhandledRejection", (reason) => {
|
|
20353
21735
|
console.error(JSON.stringify({ ts: (/* @__PURE__ */ new Date()).toISOString(), level: "fatal", msg: "unhandledRejection", reason: String(reason) }));
|
|
20354
|
-
|
|
21736
|
+
void shutdown("unhandledRejection", 1);
|
|
20355
21737
|
});
|
|
21738
|
+
heartbeatTimer = setInterval(() => {
|
|
21739
|
+
if (updateLockHeartbeat()) return;
|
|
21740
|
+
console.error(JSON.stringify({
|
|
21741
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
21742
|
+
level: "fatal",
|
|
21743
|
+
msg: "Gateway lost its database lock and will stop",
|
|
21744
|
+
pid: process.pid
|
|
21745
|
+
}));
|
|
21746
|
+
void shutdown("database lock lost", 1);
|
|
21747
|
+
}, 1e4);
|
|
21748
|
+
heartbeatTimer.unref();
|
|
21749
|
+
try {
|
|
21750
|
+
const recoveredOrphans = await TaskService.resetOrphanRunningToPending();
|
|
21751
|
+
if (recoveredOrphans > 0) {
|
|
21752
|
+
console.log(JSON.stringify({
|
|
21753
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
21754
|
+
level: "warn",
|
|
21755
|
+
msg: "reset orphan running tasks to pending",
|
|
21756
|
+
count: recoveredOrphans
|
|
21757
|
+
}));
|
|
21758
|
+
}
|
|
21759
|
+
await TaskService.resolveBlockedDependencies();
|
|
21760
|
+
initializeGatewayHealth({
|
|
21761
|
+
workerPollIntervalMs: cfg.worker.pollIntervalMs,
|
|
21762
|
+
schedulerEnabled: cfg.scheduler.enabled,
|
|
21763
|
+
schedulerCheckIntervalMs: cfg.scheduler.checkIntervalMs,
|
|
21764
|
+
watchdogCheckIntervalMs: cfg.watchdog.checkIntervalMs,
|
|
21765
|
+
watchdogCleanupIntervalMs: cfg.watchdog.cleanupIntervalMs
|
|
21766
|
+
});
|
|
21767
|
+
await scheduler.start();
|
|
21768
|
+
if (shuttingDown) return;
|
|
21769
|
+
if (cfg.dashboard.enabled) {
|
|
21770
|
+
const { dashboardApp: dashboardApp2 } = await Promise.resolve().then(() => (init_web(), web_exports));
|
|
21771
|
+
if (shuttingDown) return;
|
|
21772
|
+
dashboardServer = Bun.serve({
|
|
21773
|
+
hostname: "127.0.0.1",
|
|
21774
|
+
port: cfg.dashboard.port,
|
|
21775
|
+
fetch: dashboardApp2.fetch
|
|
21776
|
+
});
|
|
21777
|
+
console.log(JSON.stringify({
|
|
21778
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
21779
|
+
level: "info",
|
|
21780
|
+
msg: "Dashboard started",
|
|
21781
|
+
url: `http://localhost:${cfg.dashboard.port}`
|
|
21782
|
+
}));
|
|
21783
|
+
}
|
|
21784
|
+
watchdog.start();
|
|
21785
|
+
worker.start();
|
|
21786
|
+
markGatewayReady();
|
|
21787
|
+
console.log(JSON.stringify({
|
|
21788
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
21789
|
+
level: "info",
|
|
21790
|
+
msg: "Gateway started",
|
|
21791
|
+
maxConcurrency: cfg.worker.maxConcurrency,
|
|
21792
|
+
schedulerEnabled: cfg.scheduler.enabled
|
|
21793
|
+
}));
|
|
21794
|
+
} catch (error) {
|
|
21795
|
+
console.error(JSON.stringify({
|
|
21796
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
21797
|
+
level: "fatal",
|
|
21798
|
+
msg: "Gateway startup failed",
|
|
21799
|
+
error: error instanceof Error ? error.message : String(error)
|
|
21800
|
+
}));
|
|
21801
|
+
await shutdown("startup failure", 1);
|
|
21802
|
+
}
|
|
20356
21803
|
}
|
|
20357
21804
|
if (import.meta.main) {
|
|
20358
21805
|
main();
|
|
@@ -20360,6 +21807,9 @@ if (import.meta.main) {
|
|
|
20360
21807
|
export {
|
|
20361
21808
|
acquireLock,
|
|
20362
21809
|
main,
|
|
20363
|
-
releaseLock
|
|
21810
|
+
releaseLock,
|
|
21811
|
+
resolveGatewayShutdownExitCode,
|
|
21812
|
+
runGatewayShutdownStep,
|
|
21813
|
+
updateLockHeartbeat
|
|
20364
21814
|
};
|
|
20365
21815
|
//# sourceMappingURL=index.js.map
|