opencode-supertask 0.1.25 → 0.1.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +55 -17
- package/dist/cli/index.js +3564 -1090
- package/dist/cli/index.js.map +1 -1
- package/dist/daemon/pm2-supervisor.d.ts +3 -0
- package/dist/daemon/pm2-supervisor.js +145 -0
- package/dist/daemon/pm2-supervisor.js.map +1 -0
- package/dist/gateway/index.d.ts +8 -1
- package/dist/gateway/index.js +2131 -681
- package/dist/gateway/index.js.map +1 -1
- package/dist/plugin/supertask.d.ts +2 -1
- package/dist/plugin/supertask.js +1352 -345
- package/dist/plugin/supertask.js.map +1 -1
- package/dist/web/index.d.ts +1 -0
- package/dist/web/index.js +1042 -264
- package/dist/web/index.js.map +1 -1
- package/dist/worker/index.d.ts +17 -3
- package/dist/worker/index.js +1204 -247
- package/dist/worker/index.js.map +1 -1
- package/dist/worker/launcher.d.ts +9 -0
- package/dist/worker/launcher.js +164 -0
- package/dist/worker/launcher.js.map +1 -0
- package/drizzle/0005_natural_ma_gnuci.sql +1 -0
- package/drizzle/0006_worried_agent_zero.sql +2 -0
- package/drizzle/0007_lean_titanium_man.sql +1 -0
- package/drizzle/meta/0005_snapshot.json +559 -0
- package/drizzle/meta/0006_snapshot.json +575 -0
- package/drizzle/meta/0007_snapshot.json +585 -0
- package/drizzle/meta/_journal.json +21 -0
- package/package.json +1 -1
package/dist/worker/index.js
CHANGED
|
@@ -1036,7 +1036,7 @@ function sql(strings, ...params) {
|
|
|
1036
1036
|
return new SQL([new StringChunk(str)]);
|
|
1037
1037
|
}
|
|
1038
1038
|
sql2.raw = raw;
|
|
1039
|
-
function
|
|
1039
|
+
function join3(chunks, separator) {
|
|
1040
1040
|
const result = [];
|
|
1041
1041
|
for (const [i, chunk] of chunks.entries()) {
|
|
1042
1042
|
if (i > 0 && separator !== void 0) {
|
|
@@ -1046,7 +1046,7 @@ function sql(strings, ...params) {
|
|
|
1046
1046
|
}
|
|
1047
1047
|
return new SQL(result);
|
|
1048
1048
|
}
|
|
1049
|
-
sql2.join =
|
|
1049
|
+
sql2.join = join3;
|
|
1050
1050
|
function identifier(value) {
|
|
1051
1051
|
return new Name(value);
|
|
1052
1052
|
}
|
|
@@ -3342,7 +3342,7 @@ var SQLiteSelectQueryBuilderBase = class extends TypedQueryBuilder {
|
|
|
3342
3342
|
return (table, on) => {
|
|
3343
3343
|
const baseTableName = this.tableName;
|
|
3344
3344
|
const tableName = getTableLikeName(table);
|
|
3345
|
-
if (typeof tableName === "string" && this.config.joins?.some((
|
|
3345
|
+
if (typeof tableName === "string" && this.config.joins?.some((join3) => join3.alias === tableName)) {
|
|
3346
3346
|
throw new Error(`Alias "${tableName}" is already used in this query`);
|
|
3347
3347
|
}
|
|
3348
3348
|
if (!this.isPartialSelect) {
|
|
@@ -4155,7 +4155,7 @@ var SQLiteUpdateBase = class extends QueryPromise {
|
|
|
4155
4155
|
createJoin(joinType) {
|
|
4156
4156
|
return (table, on) => {
|
|
4157
4157
|
const tableName = getTableLikeName(table);
|
|
4158
|
-
if (typeof tableName === "string" && this.config.joins.some((
|
|
4158
|
+
if (typeof tableName === "string" && this.config.joins.some((join3) => join3.alias === tableName)) {
|
|
4159
4159
|
throw new Error(`Alias "${tableName}" is already used in this query`);
|
|
4160
4160
|
}
|
|
4161
4161
|
if (typeof on === "function") {
|
|
@@ -5185,7 +5185,9 @@ var tasks = sqliteTable("tasks", {
|
|
|
5185
5185
|
}, (table) => [
|
|
5186
5186
|
index("tasks_queue_idx").on(table.status, table.retryAfter, table.urgency, table.importance, table.createdAt, table.id),
|
|
5187
5187
|
index("tasks_batch_status_idx").on(table.batchId, table.status),
|
|
5188
|
-
index("tasks_template_status_idx").on(table.templateId, table.status)
|
|
5188
|
+
index("tasks_template_status_idx").on(table.templateId, table.status),
|
|
5189
|
+
index("tasks_depends_on_status_idx").on(table.dependsOn, table.status),
|
|
5190
|
+
index("tasks_cleanup_idx").on(table.finishedAt, table.id, table.status)
|
|
5189
5191
|
]);
|
|
5190
5192
|
var TASK_CATEGORIES = [
|
|
5191
5193
|
"translate",
|
|
@@ -5208,7 +5210,8 @@ var taskRuns = sqliteTable("task_runs", {
|
|
|
5208
5210
|
lockedBy: text("locked_by"),
|
|
5209
5211
|
heartbeatAt: integer("heartbeat_at"),
|
|
5210
5212
|
workerPid: integer("worker_pid"),
|
|
5211
|
-
childPid: integer("child_pid")
|
|
5213
|
+
childPid: integer("child_pid"),
|
|
5214
|
+
launchProtocol: text("launch_protocol")
|
|
5212
5215
|
}, (table) => [
|
|
5213
5216
|
index("task_runs_task_started_idx").on(table.taskId, table.startedAt, table.id),
|
|
5214
5217
|
index("task_runs_status_heartbeat_idx").on(table.status, table.heartbeatAt)
|
|
@@ -5238,7 +5241,13 @@ var taskTemplates = sqliteTable("task_templates", {
|
|
|
5238
5241
|
createdAt: integer("created_at").default(0),
|
|
5239
5242
|
updatedAt: integer("updated_at").default(0)
|
|
5240
5243
|
}, (table) => [
|
|
5241
|
-
index("task_templates_due_idx").on(table.enabled, table.nextRunAt, table.id)
|
|
5244
|
+
index("task_templates_due_idx").on(table.enabled, table.nextRunAt, table.id),
|
|
5245
|
+
index("task_templates_retention_idx").on(
|
|
5246
|
+
table.scheduleType,
|
|
5247
|
+
table.enabled,
|
|
5248
|
+
table.lastRunAt,
|
|
5249
|
+
table.id
|
|
5250
|
+
)
|
|
5242
5251
|
]);
|
|
5243
5252
|
|
|
5244
5253
|
// src/core/db/index.ts
|
|
@@ -5263,36 +5272,47 @@ function getMigrationsFolder() {
|
|
|
5263
5272
|
}
|
|
5264
5273
|
return join(__dirname, "../../drizzle");
|
|
5265
5274
|
}
|
|
5266
|
-
function
|
|
5267
|
-
|
|
5268
|
-
if (!existsSync(dataDir)) {
|
|
5269
|
-
mkdirSync(dataDir, { recursive: true });
|
|
5270
|
-
}
|
|
5271
|
-
_sqlite = new Database2(DB_FILE_PATH);
|
|
5272
|
-
_sqlite.exec("PRAGMA journal_mode = WAL;");
|
|
5273
|
-
_sqlite.exec("PRAGMA busy_timeout = 5000;");
|
|
5274
|
-
_sqlite.exec(`
|
|
5275
|
+
function ensureGatewayLock(sqliteDb) {
|
|
5276
|
+
sqliteDb.exec(`
|
|
5275
5277
|
CREATE TABLE IF NOT EXISTS gateway_lock (
|
|
5276
5278
|
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
5277
5279
|
pid INTEGER NOT NULL,
|
|
5278
5280
|
acquired_at INTEGER NOT NULL,
|
|
5279
5281
|
heartbeat_at INTEGER NOT NULL,
|
|
5280
|
-
ready_at INTEGER
|
|
5282
|
+
ready_at INTEGER,
|
|
5283
|
+
version TEXT
|
|
5281
5284
|
);
|
|
5282
5285
|
`);
|
|
5283
|
-
const
|
|
5284
|
-
if (!
|
|
5285
|
-
|
|
5286
|
+
const columns = sqliteDb.query("PRAGMA table_info(gateway_lock)").all();
|
|
5287
|
+
if (!columns.some((column) => column.name === "ready_at")) {
|
|
5288
|
+
sqliteDb.exec("ALTER TABLE gateway_lock ADD COLUMN ready_at INTEGER;");
|
|
5289
|
+
}
|
|
5290
|
+
if (!columns.some((column) => column.name === "version")) {
|
|
5291
|
+
sqliteDb.exec("ALTER TABLE gateway_lock ADD COLUMN version TEXT;");
|
|
5292
|
+
}
|
|
5293
|
+
}
|
|
5294
|
+
function migrateSqliteDatabase(sqliteDb) {
|
|
5295
|
+
ensureGatewayLock(sqliteDb);
|
|
5296
|
+
const drizzleDb = drizzle(sqliteDb, { schema: schema_exports });
|
|
5297
|
+
migrate(drizzleDb, { migrationsFolder: getMigrationsFolder() });
|
|
5298
|
+
sqliteDb.exec("PRAGMA foreign_keys = ON;");
|
|
5299
|
+
const violations = sqliteDb.query("PRAGMA foreign_key_check;").all();
|
|
5300
|
+
if (violations.length > 0) {
|
|
5301
|
+
throw new Error(`\u68C0\u6D4B\u5230 ${violations.length} \u6761\u5B64\u7ACB\u5173\u8054\u8BB0\u5F55\uFF0C\u8BF7\u5148\u4FEE\u590D\u6570\u636E\u518D\u542F\u52A8`);
|
|
5302
|
+
}
|
|
5303
|
+
return drizzleDb;
|
|
5304
|
+
}
|
|
5305
|
+
function initDb() {
|
|
5306
|
+
const dataDir = dirname(DB_FILE_PATH);
|
|
5307
|
+
if (!existsSync(dataDir)) {
|
|
5308
|
+
mkdirSync(dataDir, { recursive: true });
|
|
5286
5309
|
}
|
|
5287
|
-
|
|
5310
|
+
_sqlite = new Database2(DB_FILE_PATH);
|
|
5311
|
+
_sqlite.exec("PRAGMA journal_mode = WAL;");
|
|
5312
|
+
_sqlite.exec("PRAGMA busy_timeout = 5000;");
|
|
5288
5313
|
if (!_migrationRan) {
|
|
5289
5314
|
try {
|
|
5290
|
-
|
|
5291
|
-
_sqlite.exec("PRAGMA foreign_keys = ON;");
|
|
5292
|
-
const violations = _sqlite.query("PRAGMA foreign_key_check;").all();
|
|
5293
|
-
if (violations.length > 0) {
|
|
5294
|
-
throw new Error(`\u68C0\u6D4B\u5230 ${violations.length} \u6761\u5B64\u7ACB\u5173\u8054\u8BB0\u5F55\uFF0C\u8BF7\u5148\u4FEE\u590D\u6570\u636E\u518D\u542F\u52A8`);
|
|
5295
|
-
}
|
|
5315
|
+
_db = migrateSqliteDatabase(_sqlite);
|
|
5296
5316
|
_migrationRan = true;
|
|
5297
5317
|
} catch (err) {
|
|
5298
5318
|
const msg = err instanceof Error ? err.message : String(err);
|
|
@@ -5303,6 +5323,8 @@ function initDb() {
|
|
|
5303
5323
|
console.error(`[supertask] migration failed: ${msg}`);
|
|
5304
5324
|
throw new Error(`[supertask] DB migration failed: ${msg}`);
|
|
5305
5325
|
}
|
|
5326
|
+
} else {
|
|
5327
|
+
_db = drizzle(_sqlite, { schema: schema_exports });
|
|
5306
5328
|
}
|
|
5307
5329
|
return _db;
|
|
5308
5330
|
}
|
|
@@ -5339,6 +5361,58 @@ function computeBackoff(retryCount, baseMs = 3e4, maxMs = MAX_BACKOFF_MS) {
|
|
|
5339
5361
|
|
|
5340
5362
|
// src/core/services/task.service.ts
|
|
5341
5363
|
var { tasks: tasks2, taskRuns: taskRuns2 } = schema_exports;
|
|
5364
|
+
var cleanupInvocation = 0;
|
|
5365
|
+
var TaskDeletionConflictError = class extends Error {
|
|
5366
|
+
constructor(message) {
|
|
5367
|
+
super(message);
|
|
5368
|
+
this.name = "TaskDeletionConflictError";
|
|
5369
|
+
}
|
|
5370
|
+
};
|
|
5371
|
+
function hasNoExecutableDependents() {
|
|
5372
|
+
return sql`NOT EXISTS (
|
|
5373
|
+
SELECT 1 FROM tasks AS dependent_task
|
|
5374
|
+
WHERE dependent_task.depends_on = ${tasks2.id}
|
|
5375
|
+
AND dependent_task.status IN ('pending', 'running', 'failed', 'dead_letter')
|
|
5376
|
+
)`;
|
|
5377
|
+
}
|
|
5378
|
+
function hasViableDependency() {
|
|
5379
|
+
return or(
|
|
5380
|
+
isNull(tasks2.dependsOn),
|
|
5381
|
+
sql`EXISTS (
|
|
5382
|
+
SELECT 1 FROM tasks AS dependency_task
|
|
5383
|
+
WHERE dependency_task.id = ${tasks2.dependsOn}
|
|
5384
|
+
AND dependency_task.cwd IS ${tasks2.cwd}
|
|
5385
|
+
AND (
|
|
5386
|
+
dependency_task.status IN ('pending', 'running', 'done')
|
|
5387
|
+
OR (
|
|
5388
|
+
dependency_task.status = 'failed'
|
|
5389
|
+
AND dependency_task.retry_count <= dependency_task.max_retries
|
|
5390
|
+
)
|
|
5391
|
+
)
|
|
5392
|
+
)`
|
|
5393
|
+
);
|
|
5394
|
+
}
|
|
5395
|
+
function blockedDependentsOf(prerequisiteId) {
|
|
5396
|
+
return sql`${tasks2.id} IN (
|
|
5397
|
+
WITH RECURSIVE blocked_task(id) AS (
|
|
5398
|
+
SELECT direct_dependent.id
|
|
5399
|
+
FROM tasks AS direct_dependent
|
|
5400
|
+
WHERE direct_dependent.depends_on = ${prerequisiteId}
|
|
5401
|
+
AND direct_dependent.status IN ('pending', 'failed')
|
|
5402
|
+
AND EXISTS (
|
|
5403
|
+
SELECT 1 FROM tasks AS prerequisite
|
|
5404
|
+
WHERE prerequisite.id = ${prerequisiteId}
|
|
5405
|
+
AND prerequisite.status IN ('dead_letter', 'cancelled')
|
|
5406
|
+
)
|
|
5407
|
+
UNION
|
|
5408
|
+
SELECT descendant.id
|
|
5409
|
+
FROM tasks AS descendant
|
|
5410
|
+
INNER JOIN blocked_task ON descendant.depends_on = blocked_task.id
|
|
5411
|
+
WHERE descendant.status IN ('pending', 'failed')
|
|
5412
|
+
)
|
|
5413
|
+
SELECT id FROM blocked_task
|
|
5414
|
+
)`;
|
|
5415
|
+
}
|
|
5342
5416
|
var TaskService = class {
|
|
5343
5417
|
static buildScopeWhere(scope) {
|
|
5344
5418
|
const conditions = [];
|
|
@@ -5349,8 +5423,28 @@ var TaskService = class {
|
|
|
5349
5423
|
}
|
|
5350
5424
|
static async add(data) {
|
|
5351
5425
|
this.validateNewTask(data);
|
|
5352
|
-
|
|
5353
|
-
|
|
5426
|
+
return db.transaction((tx) => {
|
|
5427
|
+
if (data.dependsOn != null) {
|
|
5428
|
+
const dependency = tx.select({
|
|
5429
|
+
id: tasks2.id,
|
|
5430
|
+
cwd: tasks2.cwd,
|
|
5431
|
+
status: tasks2.status,
|
|
5432
|
+
retryCount: tasks2.retryCount,
|
|
5433
|
+
maxRetries: tasks2.maxRetries
|
|
5434
|
+
}).from(tasks2).where(eq(tasks2.id, data.dependsOn)).get();
|
|
5435
|
+
if (!dependency) {
|
|
5436
|
+
throw new Error(`dependsOn \u6307\u5411\u7684\u4EFB\u52A1 #${data.dependsOn} \u4E0D\u5B58\u5728`);
|
|
5437
|
+
}
|
|
5438
|
+
if ((dependency.cwd ?? null) !== (data.cwd ?? null)) {
|
|
5439
|
+
throw new Error("dependsOn \u5FC5\u987B\u6307\u5411\u540C\u4E00 cwd \u7684\u4EFB\u52A1");
|
|
5440
|
+
}
|
|
5441
|
+
const dependencyIsRecoverable = dependency.status === "pending" || dependency.status === "running" || dependency.status === "done" || dependency.status === "failed" && (dependency.retryCount ?? 0) <= (dependency.maxRetries ?? 3);
|
|
5442
|
+
if (!dependencyIsRecoverable) {
|
|
5443
|
+
throw new Error(`dependsOn \u6307\u5411\u7684\u4EFB\u52A1 #${data.dependsOn} \u5DF2\u8FDB\u5165\u4E0D\u53EF\u6062\u590D\u7EC8\u6001`);
|
|
5444
|
+
}
|
|
5445
|
+
}
|
|
5446
|
+
return tx.insert(tasks2).values(data).returning().get();
|
|
5447
|
+
}, { behavior: "immediate" });
|
|
5354
5448
|
}
|
|
5355
5449
|
static validateNewTask(data) {
|
|
5356
5450
|
if (!data.name.trim()) throw new Error("name \u4E0D\u80FD\u4E3A\u7A7A");
|
|
@@ -5402,23 +5496,54 @@ var TaskService = class {
|
|
|
5402
5496
|
if (batchFilter) {
|
|
5403
5497
|
conditions.push(batchFilter);
|
|
5404
5498
|
}
|
|
5405
|
-
const
|
|
5499
|
+
const result = await db.select().from(tasks2).where(and(
|
|
5500
|
+
...conditions,
|
|
5501
|
+
or(
|
|
5502
|
+
isNull(tasks2.dependsOn),
|
|
5503
|
+
sql`EXISTS (
|
|
5504
|
+
SELECT 1 FROM tasks AS dependency_task
|
|
5505
|
+
WHERE dependency_task.id = ${tasks2.dependsOn}
|
|
5506
|
+
AND dependency_task.status = 'done'
|
|
5507
|
+
AND dependency_task.cwd IS ${tasks2.cwd}
|
|
5508
|
+
)`
|
|
5509
|
+
),
|
|
5510
|
+
or(
|
|
5511
|
+
isNull(tasks2.batchId),
|
|
5512
|
+
sql`NOT EXISTS (
|
|
5513
|
+
SELECT 1 FROM tasks AS running_batch_task
|
|
5514
|
+
WHERE running_batch_task.batch_id = ${tasks2.batchId}
|
|
5515
|
+
AND (
|
|
5516
|
+
running_batch_task.status = 'running'
|
|
5517
|
+
OR EXISTS (
|
|
5518
|
+
SELECT 1 FROM task_runs AS running_batch_run
|
|
5519
|
+
WHERE running_batch_run.task_id = running_batch_task.id
|
|
5520
|
+
AND running_batch_run.status = 'running'
|
|
5521
|
+
)
|
|
5522
|
+
)
|
|
5523
|
+
)`
|
|
5524
|
+
),
|
|
5525
|
+
sql`NOT EXISTS (
|
|
5526
|
+
SELECT 1 FROM task_runs AS candidate_active_run
|
|
5527
|
+
WHERE candidate_active_run.task_id = ${tasks2.id}
|
|
5528
|
+
AND candidate_active_run.status = 'running'
|
|
5529
|
+
)`
|
|
5530
|
+
)).orderBy(
|
|
5406
5531
|
desc(tasks2.urgency),
|
|
5407
5532
|
desc(tasks2.importance),
|
|
5408
5533
|
asc(tasks2.createdAt),
|
|
5409
5534
|
asc(tasks2.id)
|
|
5410
|
-
);
|
|
5411
|
-
|
|
5412
|
-
|
|
5413
|
-
|
|
5414
|
-
|
|
5415
|
-
|
|
5416
|
-
|
|
5417
|
-
|
|
5418
|
-
|
|
5419
|
-
|
|
5420
|
-
|
|
5421
|
-
|
|
5535
|
+
).limit(1);
|
|
5536
|
+
return result[0] ?? null;
|
|
5537
|
+
}
|
|
5538
|
+
static async countRunning() {
|
|
5539
|
+
return db.transaction((tx) => {
|
|
5540
|
+
const runningTasks = tx.select({ count: sql`count(*)` }).from(tasks2).where(eq(tasks2.status, "running")).get();
|
|
5541
|
+
const runsWithoutRunningTask = tx.select({ count: sql`count(DISTINCT ${taskRuns2.taskId})` }).from(taskRuns2).innerJoin(tasks2, eq(tasks2.id, taskRuns2.taskId)).where(and(
|
|
5542
|
+
eq(taskRuns2.status, "running"),
|
|
5543
|
+
sql`${tasks2.status} <> 'running'`
|
|
5544
|
+
)).get();
|
|
5545
|
+
return Number(runningTasks?.count ?? 0) + Number(runsWithoutRunningTask?.count ?? 0);
|
|
5546
|
+
}, { behavior: "deferred" });
|
|
5422
5547
|
}
|
|
5423
5548
|
static async start(id, scope = {}) {
|
|
5424
5549
|
const conditions = [
|
|
@@ -5454,27 +5579,202 @@ var TaskService = class {
|
|
|
5454
5579
|
return result[0] || null;
|
|
5455
5580
|
}
|
|
5456
5581
|
static async fail(id, log, scope = {}, options) {
|
|
5457
|
-
|
|
5458
|
-
|
|
5459
|
-
|
|
5460
|
-
|
|
5461
|
-
|
|
5462
|
-
|
|
5463
|
-
|
|
5464
|
-
|
|
5465
|
-
|
|
5466
|
-
|
|
5582
|
+
return db.transaction((tx) => {
|
|
5583
|
+
const current = tx.select().from(tasks2).where(and(
|
|
5584
|
+
eq(tasks2.id, id),
|
|
5585
|
+
eq(tasks2.status, "running"),
|
|
5586
|
+
...this.buildScopeWhere(scope)
|
|
5587
|
+
)).get();
|
|
5588
|
+
if (!current) return null;
|
|
5589
|
+
const newRetryCount = (current.retryCount ?? 0) + 1;
|
|
5590
|
+
const maxRetries = current.maxRetries ?? 3;
|
|
5591
|
+
const isDeadLetter = options?.setDeadLetter ?? newRetryCount > maxRetries;
|
|
5592
|
+
const failed = tx.update(tasks2).set({
|
|
5593
|
+
status: isDeadLetter ? "dead_letter" : "failed",
|
|
5594
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
5595
|
+
resultLog: log,
|
|
5596
|
+
retryCount: newRetryCount,
|
|
5597
|
+
retryAfter: isDeadLetter ? null : options?.retryAfterMs ?? Date.now() + computeBackoff(
|
|
5598
|
+
newRetryCount,
|
|
5599
|
+
current.retryBackoffMs ?? 3e4
|
|
5600
|
+
)
|
|
5601
|
+
}).where(and(eq(tasks2.id, id), eq(tasks2.status, "running"))).returning().get();
|
|
5602
|
+
if (failed?.status === "dead_letter") {
|
|
5603
|
+
tx.update(tasks2).set({
|
|
5604
|
+
status: "dead_letter",
|
|
5605
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
5606
|
+
retryAfter: null,
|
|
5607
|
+
resultLog: `\u4F9D\u8D56\u4EFB\u52A1 #${id} \u5DF2\u8FDB\u5165\u4E0D\u53EF\u6062\u590D\u7EC8\u6001`
|
|
5608
|
+
}).where(blockedDependentsOf(id)).run();
|
|
5609
|
+
}
|
|
5610
|
+
return failed ?? null;
|
|
5611
|
+
}, { behavior: "immediate" });
|
|
5612
|
+
}
|
|
5613
|
+
static async completeRun(taskId, runId, log) {
|
|
5614
|
+
return db.transaction((tx) => {
|
|
5615
|
+
const currentTask = tx.select().from(tasks2).where(and(eq(tasks2.id, taskId), eq(tasks2.status, "running"))).get();
|
|
5616
|
+
const currentRun = tx.select({ id: taskRuns2.id }).from(taskRuns2).where(and(
|
|
5617
|
+
eq(taskRuns2.id, runId),
|
|
5618
|
+
eq(taskRuns2.taskId, taskId),
|
|
5619
|
+
eq(taskRuns2.status, "running")
|
|
5620
|
+
)).get();
|
|
5621
|
+
if (!currentTask || !currentRun) return null;
|
|
5622
|
+
const finishedAt = /* @__PURE__ */ new Date();
|
|
5623
|
+
const completed = tx.update(tasks2).set({
|
|
5624
|
+
status: "done",
|
|
5625
|
+
finishedAt,
|
|
5626
|
+
resultLog: log,
|
|
5627
|
+
retryAfter: null
|
|
5628
|
+
}).where(and(eq(tasks2.id, taskId), eq(tasks2.status, "running"))).returning().get();
|
|
5629
|
+
if (!completed) return null;
|
|
5630
|
+
tx.update(taskRuns2).set({ status: "done", finishedAt, log }).where(and(eq(taskRuns2.id, runId), eq(taskRuns2.status, "running"))).run();
|
|
5631
|
+
return completed;
|
|
5632
|
+
}, { behavior: "immediate" });
|
|
5633
|
+
}
|
|
5634
|
+
static async failRun(taskId, runId, log, options) {
|
|
5635
|
+
const failed = db.transaction((tx) => {
|
|
5636
|
+
const currentTask = tx.select().from(tasks2).where(and(eq(tasks2.id, taskId), eq(tasks2.status, "running"))).get();
|
|
5637
|
+
const currentRun = tx.select({ id: taskRuns2.id }).from(taskRuns2).where(and(
|
|
5638
|
+
eq(taskRuns2.id, runId),
|
|
5639
|
+
eq(taskRuns2.taskId, taskId),
|
|
5640
|
+
eq(taskRuns2.status, "running")
|
|
5641
|
+
)).get();
|
|
5642
|
+
if (!currentTask || !currentRun) return null;
|
|
5643
|
+
const newRetryCount = (currentTask.retryCount ?? 0) + 1;
|
|
5644
|
+
const maxRetries = currentTask.maxRetries ?? 3;
|
|
5645
|
+
const isDeadLetter = options?.setDeadLetter ?? newRetryCount > maxRetries;
|
|
5646
|
+
const finishedAt = /* @__PURE__ */ new Date();
|
|
5647
|
+
const retryAfter = isDeadLetter ? null : options?.retryAfterMs ?? Date.now() + computeBackoff(
|
|
5648
|
+
newRetryCount,
|
|
5649
|
+
currentTask.retryBackoffMs ?? 3e4
|
|
5650
|
+
);
|
|
5651
|
+
const failed2 = tx.update(tasks2).set({
|
|
5652
|
+
status: isDeadLetter ? "dead_letter" : "failed",
|
|
5653
|
+
finishedAt,
|
|
5654
|
+
resultLog: log,
|
|
5655
|
+
retryCount: newRetryCount,
|
|
5656
|
+
retryAfter
|
|
5657
|
+
}).where(and(eq(tasks2.id, taskId), eq(tasks2.status, "running"))).returning().get();
|
|
5658
|
+
if (!failed2) return null;
|
|
5659
|
+
tx.update(taskRuns2).set({ status: "failed", finishedAt, log }).where(and(eq(taskRuns2.id, runId), eq(taskRuns2.status, "running"))).run();
|
|
5660
|
+
if (failed2.status === "dead_letter") {
|
|
5661
|
+
tx.update(tasks2).set({
|
|
5662
|
+
status: "dead_letter",
|
|
5663
|
+
finishedAt,
|
|
5664
|
+
retryAfter: null,
|
|
5665
|
+
resultLog: `\u4F9D\u8D56\u4EFB\u52A1 #${taskId} \u5DF2\u8FDB\u5165\u4E0D\u53EF\u6062\u590D\u7EC8\u6001`
|
|
5666
|
+
}).where(blockedDependentsOf(taskId)).run();
|
|
5667
|
+
}
|
|
5668
|
+
return failed2;
|
|
5669
|
+
}, { behavior: "immediate" });
|
|
5670
|
+
return failed;
|
|
5671
|
+
}
|
|
5672
|
+
static async recoverRun(taskId, runId, log) {
|
|
5673
|
+
const recovery = db.transaction((tx) => {
|
|
5674
|
+
const currentTask = tx.select().from(tasks2).where(and(eq(tasks2.id, taskId), eq(tasks2.status, "running"))).get();
|
|
5675
|
+
const currentRun = tx.select({ id: taskRuns2.id }).from(taskRuns2).where(and(
|
|
5676
|
+
eq(taskRuns2.id, runId),
|
|
5677
|
+
eq(taskRuns2.taskId, taskId),
|
|
5678
|
+
eq(taskRuns2.status, "running")
|
|
5679
|
+
)).get();
|
|
5680
|
+
if (!currentRun) return null;
|
|
5681
|
+
const finishedAt = /* @__PURE__ */ new Date();
|
|
5682
|
+
tx.update(taskRuns2).set({ status: "failed", finishedAt, log }).where(and(eq(taskRuns2.id, runId), eq(taskRuns2.status, "running"))).run();
|
|
5683
|
+
if (!currentTask) return null;
|
|
5684
|
+
const retryCount = (currentTask.retryCount ?? 0) + 1;
|
|
5685
|
+
const maxRetries = currentTask.maxRetries ?? 3;
|
|
5686
|
+
const isDeadLetter = retryCount > maxRetries;
|
|
5687
|
+
const recoveryStatus = isDeadLetter ? "dead_letter" : "pending";
|
|
5688
|
+
const retryAfterMs = isDeadLetter ? null : Date.now() + computeBackoff(
|
|
5689
|
+
retryCount,
|
|
5690
|
+
currentTask.retryBackoffMs ?? 3e4
|
|
5691
|
+
);
|
|
5692
|
+
tx.update(tasks2).set({
|
|
5693
|
+
status: recoveryStatus,
|
|
5694
|
+
startedAt: null,
|
|
5695
|
+
finishedAt: isDeadLetter ? finishedAt : null,
|
|
5696
|
+
retryCount,
|
|
5697
|
+
retryAfter: retryAfterMs,
|
|
5698
|
+
resultLog: log
|
|
5699
|
+
}).where(and(eq(tasks2.id, taskId), eq(tasks2.status, "running"))).run();
|
|
5700
|
+
if (recoveryStatus === "dead_letter") {
|
|
5701
|
+
tx.update(tasks2).set({
|
|
5702
|
+
status: "dead_letter",
|
|
5703
|
+
finishedAt,
|
|
5704
|
+
retryAfter: null,
|
|
5705
|
+
resultLog: `\u4F9D\u8D56\u4EFB\u52A1 #${taskId} \u5DF2\u8FDB\u5165\u4E0D\u53EF\u6062\u590D\u7EC8\u6001`
|
|
5706
|
+
}).where(blockedDependentsOf(taskId)).run();
|
|
5707
|
+
}
|
|
5708
|
+
return {
|
|
5709
|
+
status: recoveryStatus,
|
|
5710
|
+
retryCount,
|
|
5711
|
+
retryAfterMs
|
|
5712
|
+
};
|
|
5713
|
+
}, { behavior: "immediate" });
|
|
5714
|
+
return recovery;
|
|
5715
|
+
}
|
|
5716
|
+
static async interruptRun(taskId, runId, log) {
|
|
5717
|
+
return db.transaction((tx) => {
|
|
5718
|
+
const currentRun = tx.select({ id: taskRuns2.id }).from(taskRuns2).where(and(
|
|
5719
|
+
eq(taskRuns2.id, runId),
|
|
5720
|
+
eq(taskRuns2.taskId, taskId),
|
|
5721
|
+
eq(taskRuns2.status, "running")
|
|
5722
|
+
)).get();
|
|
5723
|
+
if (!currentRun) return false;
|
|
5724
|
+
const updatedRun = tx.update(taskRuns2).set({ status: "failed", finishedAt: /* @__PURE__ */ new Date(), log }).where(and(eq(taskRuns2.id, runId), eq(taskRuns2.status, "running"))).returning({ id: taskRuns2.id }).get();
|
|
5725
|
+
tx.update(tasks2).set({
|
|
5726
|
+
status: "pending",
|
|
5727
|
+
startedAt: null,
|
|
5728
|
+
finishedAt: null,
|
|
5729
|
+
retryAfter: null,
|
|
5730
|
+
resultLog: log
|
|
5731
|
+
}).where(and(eq(tasks2.id, taskId), eq(tasks2.status, "running"))).run();
|
|
5732
|
+
return updatedRun != null;
|
|
5733
|
+
}, { behavior: "immediate" });
|
|
5734
|
+
}
|
|
5735
|
+
static async resolveBlockedDependencies() {
|
|
5736
|
+
const finishedAt = /* @__PURE__ */ new Date();
|
|
5737
|
+
const result = await db.update(tasks2).set({
|
|
5738
|
+
status: "dead_letter",
|
|
5739
|
+
finishedAt,
|
|
5740
|
+
retryAfter: null,
|
|
5741
|
+
resultLog: "\u4F9D\u8D56\u4EFB\u52A1\u4E0D\u5B58\u5728\u3001\u8DE8\u9879\u76EE\u6216\u5DF2\u8FDB\u5165\u4E0D\u53EF\u6062\u590D\u7EC8\u6001"
|
|
5742
|
+
}).where(sql`${tasks2.id} IN (
|
|
5743
|
+
WITH RECURSIVE blocked_task(id) AS (
|
|
5744
|
+
SELECT candidate.id
|
|
5745
|
+
FROM tasks AS candidate
|
|
5746
|
+
WHERE candidate.status IN ('pending', 'failed')
|
|
5747
|
+
AND candidate.depends_on IS NOT NULL
|
|
5748
|
+
AND NOT EXISTS (
|
|
5749
|
+
SELECT 1 FROM tasks AS viable_dependency
|
|
5750
|
+
WHERE viable_dependency.id = candidate.depends_on
|
|
5751
|
+
AND viable_dependency.cwd IS candidate.cwd
|
|
5752
|
+
AND (
|
|
5753
|
+
viable_dependency.status IN ('pending', 'running', 'done')
|
|
5754
|
+
OR (
|
|
5755
|
+
viable_dependency.status = 'failed'
|
|
5756
|
+
AND viable_dependency.retry_count <= viable_dependency.max_retries
|
|
5757
|
+
)
|
|
5758
|
+
)
|
|
5759
|
+
)
|
|
5760
|
+
UNION
|
|
5761
|
+
SELECT descendant.id
|
|
5762
|
+
FROM tasks AS descendant
|
|
5763
|
+
INNER JOIN blocked_task ON descendant.depends_on = blocked_task.id
|
|
5764
|
+
WHERE descendant.status IN ('pending', 'failed')
|
|
5765
|
+
)
|
|
5766
|
+
SELECT id FROM blocked_task
|
|
5767
|
+
)`).returning();
|
|
5768
|
+
return result.length;
|
|
5769
|
+
}
|
|
5770
|
+
static async rejectBlockedDependents(prerequisiteId) {
|
|
5467
5771
|
const result = await db.update(tasks2).set({
|
|
5468
|
-
status:
|
|
5772
|
+
status: "dead_letter",
|
|
5469
5773
|
finishedAt: /* @__PURE__ */ new Date(),
|
|
5470
|
-
|
|
5471
|
-
|
|
5472
|
-
|
|
5473
|
-
|
|
5474
|
-
current.retryBackoffMs ?? 3e4
|
|
5475
|
-
)
|
|
5476
|
-
}).where(and(...conditions)).returning();
|
|
5477
|
-
return result[0] || null;
|
|
5774
|
+
retryAfter: null,
|
|
5775
|
+
resultLog: `\u4F9D\u8D56\u4EFB\u52A1 #${prerequisiteId} \u5DF2\u8FDB\u5165\u4E0D\u53EF\u6062\u590D\u7EC8\u6001`
|
|
5776
|
+
}).where(blockedDependentsOf(prerequisiteId)).returning({ id: tasks2.id });
|
|
5777
|
+
return result.length;
|
|
5478
5778
|
}
|
|
5479
5779
|
static async markPendingForRetry(id, retryAfterMs, retryCount) {
|
|
5480
5780
|
const result = await db.update(tasks2).set({
|
|
@@ -5487,8 +5787,18 @@ var TaskService = class {
|
|
|
5487
5787
|
return result[0] || null;
|
|
5488
5788
|
}
|
|
5489
5789
|
static async markDeadLetter(id, retryCount) {
|
|
5490
|
-
|
|
5491
|
-
|
|
5790
|
+
return db.transaction((tx) => {
|
|
5791
|
+
const finishedAt = /* @__PURE__ */ new Date();
|
|
5792
|
+
const task = tx.update(tasks2).set({ status: "dead_letter", finishedAt, retryCount }).where(eq(tasks2.id, id)).returning().get();
|
|
5793
|
+
if (!task) return null;
|
|
5794
|
+
tx.update(tasks2).set({
|
|
5795
|
+
status: "dead_letter",
|
|
5796
|
+
finishedAt,
|
|
5797
|
+
retryAfter: null,
|
|
5798
|
+
resultLog: `\u4F9D\u8D56\u4EFB\u52A1 #${id} \u5DF2\u8FDB\u5165\u4E0D\u53EF\u6062\u590D\u7EC8\u6001`
|
|
5799
|
+
}).where(blockedDependentsOf(id)).run();
|
|
5800
|
+
return task;
|
|
5801
|
+
}, { behavior: "immediate" });
|
|
5492
5802
|
}
|
|
5493
5803
|
static async resetRunningToPending(ids) {
|
|
5494
5804
|
if (ids.length === 0) return 0;
|
|
@@ -5531,41 +5841,91 @@ var TaskService = class {
|
|
|
5531
5841
|
),
|
|
5532
5842
|
...this.buildScopeWhere(scope)
|
|
5533
5843
|
];
|
|
5534
|
-
|
|
5535
|
-
|
|
5536
|
-
|
|
5537
|
-
|
|
5538
|
-
|
|
5539
|
-
|
|
5844
|
+
return db.transaction((tx) => {
|
|
5845
|
+
const finishedAt = /* @__PURE__ */ new Date();
|
|
5846
|
+
const task = tx.update(tasks2).set({
|
|
5847
|
+
status: "cancelled",
|
|
5848
|
+
finishedAt,
|
|
5849
|
+
retryAfter: null
|
|
5850
|
+
}).where(and(...conditions)).returning().get();
|
|
5851
|
+
if (!task) return null;
|
|
5852
|
+
tx.update(tasks2).set({
|
|
5853
|
+
status: "dead_letter",
|
|
5854
|
+
finishedAt,
|
|
5855
|
+
retryAfter: null,
|
|
5856
|
+
resultLog: `\u4F9D\u8D56\u4EFB\u52A1 #${id} \u5DF2\u8FDB\u5165\u4E0D\u53EF\u6062\u590D\u7EC8\u6001`
|
|
5857
|
+
}).where(blockedDependentsOf(id)).run();
|
|
5858
|
+
return task;
|
|
5859
|
+
}, { behavior: "immediate" });
|
|
5540
5860
|
}
|
|
5541
5861
|
static async retry(id, scope = {}) {
|
|
5542
|
-
const current = await this.getById(id, scope);
|
|
5543
|
-
if (!current) return null;
|
|
5544
|
-
if (current.status !== "failed" && current.status !== "dead_letter") return null;
|
|
5545
|
-
const conditions = [eq(tasks2.id, id), ...this.buildScopeWhere(scope)];
|
|
5546
|
-
const result = await db.update(tasks2).set({
|
|
5547
|
-
status: "pending",
|
|
5548
|
-
startedAt: null,
|
|
5549
|
-
finishedAt: null,
|
|
5550
|
-
retryAfter: null,
|
|
5551
|
-
retryCount: 0
|
|
5552
|
-
}).where(and(...conditions)).returning();
|
|
5553
|
-
return result[0] || null;
|
|
5554
|
-
}
|
|
5555
|
-
static async retryBatch(batchId, scope = {}) {
|
|
5556
5862
|
const conditions = [
|
|
5557
|
-
eq(tasks2.
|
|
5863
|
+
eq(tasks2.id, id),
|
|
5558
5864
|
or(eq(tasks2.status, "failed"), eq(tasks2.status, "dead_letter")),
|
|
5559
5865
|
...this.buildScopeWhere(scope)
|
|
5560
5866
|
];
|
|
5561
|
-
|
|
5867
|
+
return db.transaction((tx) => tx.update(tasks2).set({
|
|
5562
5868
|
status: "pending",
|
|
5563
5869
|
startedAt: null,
|
|
5564
5870
|
finishedAt: null,
|
|
5565
5871
|
retryAfter: null,
|
|
5566
5872
|
retryCount: 0
|
|
5567
|
-
}).where(and(...conditions)).returning();
|
|
5568
|
-
|
|
5873
|
+
}).where(and(...conditions, hasViableDependency())).returning().get() ?? null, { behavior: "immediate" });
|
|
5874
|
+
}
|
|
5875
|
+
static async retryBatch(batchId, scope = {}) {
|
|
5876
|
+
const sqlite2 = getSqlite();
|
|
5877
|
+
const scopeFilter = scope.cwd === void 0 ? "" : "AND candidate.cwd = ?";
|
|
5878
|
+
const parameters = scope.cwd === void 0 ? [batchId] : [batchId, scope.cwd];
|
|
5879
|
+
return db.transaction(() => sqlite2.query(`
|
|
5880
|
+
WITH RECURSIVE
|
|
5881
|
+
candidate(id, cwd, depends_on) AS MATERIALIZED (
|
|
5882
|
+
SELECT candidate.id, candidate.cwd, candidate.depends_on
|
|
5883
|
+
FROM tasks AS candidate
|
|
5884
|
+
WHERE candidate.batch_id = ?
|
|
5885
|
+
AND candidate.status IN ('failed', 'dead_letter')
|
|
5886
|
+
${scopeFilter}
|
|
5887
|
+
),
|
|
5888
|
+
retryable(id, cwd, depends_on) AS (
|
|
5889
|
+
SELECT candidate.id,
|
|
5890
|
+
candidate.cwd,
|
|
5891
|
+
candidate.depends_on
|
|
5892
|
+
FROM candidate
|
|
5893
|
+
WHERE candidate.depends_on IS NULL
|
|
5894
|
+
OR (
|
|
5895
|
+
NOT EXISTS (
|
|
5896
|
+
SELECT 1 FROM candidate AS internal_parent
|
|
5897
|
+
WHERE internal_parent.id = candidate.depends_on
|
|
5898
|
+
)
|
|
5899
|
+
AND EXISTS (
|
|
5900
|
+
SELECT 1 FROM tasks AS external_parent
|
|
5901
|
+
WHERE external_parent.id = candidate.depends_on
|
|
5902
|
+
AND external_parent.cwd IS candidate.cwd
|
|
5903
|
+
AND (
|
|
5904
|
+
external_parent.status IN ('pending', 'running', 'done')
|
|
5905
|
+
OR (
|
|
5906
|
+
external_parent.status = 'failed'
|
|
5907
|
+
AND external_parent.retry_count <= external_parent.max_retries
|
|
5908
|
+
)
|
|
5909
|
+
)
|
|
5910
|
+
)
|
|
5911
|
+
)
|
|
5912
|
+
UNION
|
|
5913
|
+
SELECT child.id,
|
|
5914
|
+
child.cwd,
|
|
5915
|
+
child.depends_on
|
|
5916
|
+
FROM candidate AS child
|
|
5917
|
+
INNER JOIN retryable AS parent
|
|
5918
|
+
ON child.depends_on = parent.id
|
|
5919
|
+
AND child.cwd IS parent.cwd
|
|
5920
|
+
)
|
|
5921
|
+
UPDATE tasks
|
|
5922
|
+
SET status = 'pending',
|
|
5923
|
+
started_at = NULL,
|
|
5924
|
+
finished_at = NULL,
|
|
5925
|
+
retry_after = NULL,
|
|
5926
|
+
retry_count = 0
|
|
5927
|
+
WHERE id IN (SELECT id FROM retryable)
|
|
5928
|
+
`).run(...parameters).changes, { behavior: "immediate" });
|
|
5569
5929
|
}
|
|
5570
5930
|
static async getById(id, scope = {}) {
|
|
5571
5931
|
const conditions = [eq(tasks2.id, id), ...this.buildScopeWhere(scope)];
|
|
@@ -5630,28 +5990,322 @@ var TaskService = class {
|
|
|
5630
5990
|
return stats;
|
|
5631
5991
|
}
|
|
5632
5992
|
static async delete(id, scope = {}) {
|
|
5633
|
-
const conditions = [
|
|
5634
|
-
|
|
5635
|
-
|
|
5636
|
-
|
|
5993
|
+
const conditions = [
|
|
5994
|
+
eq(tasks2.id, id),
|
|
5995
|
+
sql`${tasks2.status} <> 'running'`,
|
|
5996
|
+
sql`NOT EXISTS (
|
|
5997
|
+
SELECT 1 FROM ${taskRuns2}
|
|
5998
|
+
WHERE ${taskRuns2.taskId} = ${tasks2.id}
|
|
5999
|
+
AND ${taskRuns2.status} = 'running'
|
|
6000
|
+
)`,
|
|
6001
|
+
hasNoExecutableDependents(),
|
|
6002
|
+
...this.buildScopeWhere(scope)
|
|
6003
|
+
];
|
|
5637
6004
|
const result = await db.delete(tasks2).where(and(...conditions)).returning();
|
|
5638
|
-
|
|
6005
|
+
if (result.length > 0) {
|
|
6006
|
+
await db.delete(taskRuns2).where(eq(taskRuns2.taskId, id));
|
|
6007
|
+
return true;
|
|
6008
|
+
}
|
|
6009
|
+
if (!await this.getById(id, scope)) return false;
|
|
6010
|
+
const dependent = await db.select({ id: tasks2.id }).from(tasks2).where(and(
|
|
6011
|
+
eq(tasks2.dependsOn, id),
|
|
6012
|
+
sql`${tasks2.status} IN ('pending', 'running', 'failed', 'dead_letter')`
|
|
6013
|
+
)).orderBy(asc(tasks2.id)).limit(1);
|
|
6014
|
+
if (dependent[0]) {
|
|
6015
|
+
throw new TaskDeletionConflictError(
|
|
6016
|
+
`\u4EFB\u52A1 #${id} \u4ECD\u88AB\u53EF\u6267\u884C\u4EFB\u52A1 #${dependent[0].id} \u4F9D\u8D56\uFF0C\u8BF7\u5148\u5904\u7406\u4F9D\u8D56\u4EFB\u52A1`
|
|
6017
|
+
);
|
|
6018
|
+
}
|
|
6019
|
+
throw new TaskDeletionConflictError(
|
|
6020
|
+
`\u4EFB\u52A1 #${id} \u6B63\u5728\u8FD0\u884C\uFF0C\u8BF7\u5148\u53D6\u6D88\u4EFB\u52A1\u5E76\u7B49\u5F85\u6267\u884C\u8FDB\u7A0B\u9000\u51FA`
|
|
6021
|
+
);
|
|
5639
6022
|
}
|
|
5640
|
-
static async deleteOlderThan(retentionDays) {
|
|
6023
|
+
static async deleteOlderThan(retentionDays, shouldStop = () => false) {
|
|
5641
6024
|
const cutoffSec = Math.floor(Date.now() / 1e3) - retentionDays * 86400;
|
|
5642
|
-
const
|
|
5643
|
-
|
|
5644
|
-
|
|
5645
|
-
|
|
5646
|
-
|
|
5647
|
-
|
|
5648
|
-
|
|
5649
|
-
|
|
6025
|
+
const batchSize = 500;
|
|
6026
|
+
const sqlite2 = getSqlite();
|
|
6027
|
+
cleanupInvocation += 1;
|
|
6028
|
+
const candidateTable = `cleanup_candidates_${process.pid}_${cleanupInvocation}`;
|
|
6029
|
+
let deletedTotal = 0;
|
|
6030
|
+
sqlite2.exec(`
|
|
6031
|
+
CREATE TEMP TABLE ${candidateTable} (
|
|
6032
|
+
id INTEGER NOT NULL PRIMARY KEY
|
|
6033
|
+
) WITHOUT ROWID;
|
|
6034
|
+
`);
|
|
6035
|
+
try {
|
|
6036
|
+
let ceilingId = null;
|
|
6037
|
+
while (true) {
|
|
6038
|
+
if (shouldStop()) return deletedTotal;
|
|
6039
|
+
const batch = db.transaction(() => {
|
|
6040
|
+
sqlite2.query(`DELETE FROM ${candidateTable}`).run();
|
|
6041
|
+
const ceilingPredicate = ceilingId == null ? "" : "AND candidate.id < ?";
|
|
6042
|
+
const rawCandidateStatement = sqlite2.query(`
|
|
6043
|
+
INSERT INTO ${candidateTable}(id)
|
|
6044
|
+
SELECT candidate.id
|
|
6045
|
+
FROM tasks AS candidate NOT INDEXED
|
|
6046
|
+
WHERE candidate.status IN ('done', 'dead_letter', 'cancelled')
|
|
6047
|
+
AND candidate.finished_at IS NOT NULL
|
|
6048
|
+
AND candidate.finished_at < ?
|
|
6049
|
+
${ceilingPredicate}
|
|
6050
|
+
AND NOT EXISTS (
|
|
6051
|
+
SELECT 1 FROM task_runs AS active_run
|
|
6052
|
+
WHERE active_run.task_id = candidate.id
|
|
6053
|
+
AND active_run.status = 'running'
|
|
6054
|
+
)
|
|
6055
|
+
ORDER BY candidate.id DESC
|
|
6056
|
+
LIMIT ?
|
|
6057
|
+
`);
|
|
6058
|
+
const rawCount = ceilingId == null ? rawCandidateStatement.run(cutoffSec, batchSize).changes : rawCandidateStatement.run(cutoffSec, ceilingId, batchSize).changes;
|
|
6059
|
+
if (rawCount === 0) return { deleted: 0, nextCeilingId: null };
|
|
6060
|
+
const rawPage = sqlite2.query(`
|
|
6061
|
+
SELECT min(id) AS nextCeilingId FROM ${candidateTable}
|
|
6062
|
+
`).get();
|
|
6063
|
+
sqlite2.query(`
|
|
6064
|
+
DELETE FROM ${candidateTable}
|
|
6065
|
+
WHERE EXISTS (
|
|
6066
|
+
SELECT 1 FROM tasks AS anomalous
|
|
6067
|
+
WHERE anomalous.id = ${candidateTable}.id
|
|
6068
|
+
AND anomalous.depends_on IS NOT NULL
|
|
6069
|
+
AND anomalous.depends_on >= anomalous.id
|
|
6070
|
+
)
|
|
6071
|
+
`).run();
|
|
6072
|
+
while (true) {
|
|
6073
|
+
const pruned = sqlite2.query(`
|
|
6074
|
+
DELETE FROM ${candidateTable}
|
|
6075
|
+
WHERE EXISTS (
|
|
6076
|
+
SELECT 1 FROM tasks AS dependent_task
|
|
6077
|
+
WHERE dependent_task.depends_on = ${candidateTable}.id
|
|
6078
|
+
AND NOT EXISTS (
|
|
6079
|
+
SELECT 1 FROM ${candidateTable} AS selected_dependent
|
|
6080
|
+
WHERE selected_dependent.id = dependent_task.id
|
|
6081
|
+
)
|
|
6082
|
+
)
|
|
6083
|
+
`).run().changes;
|
|
6084
|
+
if (pruned === 0) break;
|
|
6085
|
+
}
|
|
6086
|
+
const selected = sqlite2.query(`
|
|
6087
|
+
SELECT count(*) AS count FROM ${candidateTable}
|
|
6088
|
+
`).get();
|
|
6089
|
+
if (selected.count === 0) {
|
|
6090
|
+
return { deleted: 0, nextCeilingId: rawPage.nextCeilingId };
|
|
6091
|
+
}
|
|
6092
|
+
sqlite2.query(`
|
|
6093
|
+
DELETE FROM tasks
|
|
6094
|
+
WHERE id IN (SELECT id FROM ${candidateTable})
|
|
6095
|
+
`).run();
|
|
6096
|
+
const remaining = sqlite2.query(`
|
|
6097
|
+
SELECT count(*) AS count
|
|
6098
|
+
FROM tasks
|
|
6099
|
+
WHERE id IN (SELECT id FROM ${candidateTable})
|
|
6100
|
+
`).get();
|
|
6101
|
+
if (remaining.count !== 0) {
|
|
6102
|
+
throw new Error("\u5386\u53F2\u6E05\u7406\u5019\u9009\u5728\u540C\u4E00\u5199\u4E8B\u52A1\u5185\u53D1\u751F\u6F02\u79FB\uFF0C\u5DF2\u56DE\u6EDA\u672C\u6279\u5220\u9664");
|
|
6103
|
+
}
|
|
6104
|
+
return { deleted: selected.count, nextCeilingId: rawPage.nextCeilingId };
|
|
6105
|
+
}, { behavior: "immediate" });
|
|
6106
|
+
if (batch.nextCeilingId == null) break;
|
|
6107
|
+
ceilingId = batch.nextCeilingId;
|
|
6108
|
+
deletedTotal += batch.deleted;
|
|
6109
|
+
await Bun.sleep(0);
|
|
6110
|
+
if (shouldStop()) return deletedTotal;
|
|
6111
|
+
}
|
|
6112
|
+
return deletedTotal;
|
|
6113
|
+
} finally {
|
|
6114
|
+
sqlite2.exec(`DROP TABLE IF EXISTS ${candidateTable}`);
|
|
6115
|
+
}
|
|
5650
6116
|
}
|
|
5651
6117
|
};
|
|
5652
6118
|
|
|
6119
|
+
// src/core/process-control.ts
|
|
6120
|
+
import { spawnSync } from "child_process";
|
|
6121
|
+
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
6122
|
+
import { basename, dirname as dirname2, resolve } from "path";
|
|
6123
|
+
|
|
6124
|
+
// src/core/launch-protocol.ts
|
|
6125
|
+
var TOKEN_GUARDIAN_LAUNCH_PROTOCOL = "gated-v3-token-guardian";
|
|
6126
|
+
var LAUNCH_IDENTITY_ARGUMENT = "--supertask-launch-identity";
|
|
6127
|
+
var DRAIN_PROOF_MESSAGE_TYPE = "supertask-drained";
|
|
6128
|
+
var MANAGED_RUN_ENV = "SUPERTASK_MANAGED_RUN";
|
|
6129
|
+
var MANAGED_RUN_ENV_VALUE = "1";
|
|
6130
|
+
function isLaunchIdentity(value) {
|
|
6131
|
+
return value != null && /^gateway-[1-9]\d*:launch:[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(value);
|
|
6132
|
+
}
|
|
6133
|
+
function isMatchingDrainProof(message, launchIdentity) {
|
|
6134
|
+
if (typeof message !== "object" || message == null) return false;
|
|
6135
|
+
const candidate = message;
|
|
6136
|
+
return candidate.type === DRAIN_PROOF_MESSAGE_TYPE && candidate.identity === launchIdentity;
|
|
6137
|
+
}
|
|
6138
|
+
|
|
6139
|
+
// src/core/process-control.ts
|
|
6140
|
+
var OS_COMMAND_TIMEOUT_MS = 2e3;
|
|
6141
|
+
function isSafePid(pid) {
|
|
6142
|
+
return Number.isInteger(pid) && pid > 1 && pid !== process.pid;
|
|
6143
|
+
}
|
|
6144
|
+
function absentLeaderResult(pid) {
|
|
6145
|
+
return inspectSpawnedProcessTreePresence(pid) === "not-running" ? "not-running" : "inspect-failed";
|
|
6146
|
+
}
|
|
6147
|
+
function isProcessAlive(pid) {
|
|
6148
|
+
if (!Number.isInteger(pid) || pid <= 0) return false;
|
|
6149
|
+
try {
|
|
6150
|
+
process.kill(pid, 0);
|
|
6151
|
+
return true;
|
|
6152
|
+
} catch (error) {
|
|
6153
|
+
return error instanceof Error && "code" in error && error.code === "EPERM";
|
|
6154
|
+
}
|
|
6155
|
+
}
|
|
6156
|
+
function inspectSpawnedProcessTreePresence(pid) {
|
|
6157
|
+
if (!Number.isInteger(pid) || pid <= 0) return "unknown";
|
|
6158
|
+
if (process.platform === "win32") {
|
|
6159
|
+
const processIds = inspectWindowsProcessTree(pid);
|
|
6160
|
+
if (processIds == null) return "unknown";
|
|
6161
|
+
return processIds.length > 0 ? "running" : "not-running";
|
|
6162
|
+
}
|
|
6163
|
+
try {
|
|
6164
|
+
process.kill(-pid, 0);
|
|
6165
|
+
return "running";
|
|
6166
|
+
} catch (error) {
|
|
6167
|
+
if (!(error instanceof Error) || !("code" in error)) return "unknown";
|
|
6168
|
+
if (error.code === "EPERM") return "running";
|
|
6169
|
+
if (error.code === "ESRCH") return "not-running";
|
|
6170
|
+
return "unknown";
|
|
6171
|
+
}
|
|
6172
|
+
}
|
|
6173
|
+
function inspectUnixProcess(pid) {
|
|
6174
|
+
const result = spawnSync("ps", ["-o", "pgid=", "-o", "command=", "-p", String(pid)], {
|
|
6175
|
+
encoding: "utf8",
|
|
6176
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
6177
|
+
timeout: OS_COMMAND_TIMEOUT_MS,
|
|
6178
|
+
killSignal: "SIGKILL"
|
|
6179
|
+
});
|
|
6180
|
+
if (result.status !== 0) return null;
|
|
6181
|
+
const match = result.stdout.trim().match(/^(\d+)\s+(.+)$/s);
|
|
6182
|
+
if (!match) return null;
|
|
6183
|
+
return { processGroupId: Number(match[1]), command: match[2] };
|
|
6184
|
+
}
|
|
6185
|
+
function inspectWindowsCommand(pid) {
|
|
6186
|
+
const script = `(Get-CimInstance Win32_Process -Filter "ProcessId = ${pid}").CommandLine`;
|
|
6187
|
+
const result = spawnSync(
|
|
6188
|
+
"powershell.exe",
|
|
6189
|
+
["-NoProfile", "-NonInteractive", "-Command", script],
|
|
6190
|
+
{
|
|
6191
|
+
encoding: "utf8",
|
|
6192
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
6193
|
+
timeout: OS_COMMAND_TIMEOUT_MS,
|
|
6194
|
+
killSignal: "SIGKILL"
|
|
6195
|
+
}
|
|
6196
|
+
);
|
|
6197
|
+
if (result.status !== 0) return null;
|
|
6198
|
+
return result.stdout.trim() || null;
|
|
6199
|
+
}
|
|
6200
|
+
function inspectWindowsProcessTree(rootPid) {
|
|
6201
|
+
const script = `$root=${rootPid}; $all=@(Get-CimInstance Win32_Process | Select-Object ProcessId,ParentProcessId); $ids=New-Object 'System.Collections.Generic.HashSet[int]'; [void]$ids.Add($root); do { $added=$false; foreach($p in $all) { if($ids.Contains([int]$p.ParentProcessId) -and $ids.Add([int]$p.ProcessId)) { $added=$true } } } while($added); $all | Where-Object { $ids.Contains([int]$_.ProcessId) } | Select-Object -ExpandProperty ProcessId`;
|
|
6202
|
+
const result = spawnSync(
|
|
6203
|
+
"powershell.exe",
|
|
6204
|
+
["-NoProfile", "-NonInteractive", "-Command", script],
|
|
6205
|
+
{
|
|
6206
|
+
encoding: "utf8",
|
|
6207
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
6208
|
+
timeout: OS_COMMAND_TIMEOUT_MS,
|
|
6209
|
+
killSignal: "SIGKILL"
|
|
6210
|
+
}
|
|
6211
|
+
);
|
|
6212
|
+
if (result.status !== 0) return null;
|
|
6213
|
+
return result.stdout.split(/\s+/).filter(Boolean).map(Number).filter((pid) => Number.isInteger(pid) && pid > 0);
|
|
6214
|
+
}
|
|
6215
|
+
function executableName(value) {
|
|
6216
|
+
return basename(value).trim().toLowerCase().replace(/\.(?:exe|cmd|bat)$/i, "");
|
|
6217
|
+
}
|
|
6218
|
+
function commandTokens(command) {
|
|
6219
|
+
return command.trim().split(/\s+/).map((token) => token.replace(/^["']|["']$/g, ""));
|
|
6220
|
+
}
|
|
6221
|
+
function openCodeArgsMatch(args) {
|
|
6222
|
+
const agentIndex = args.indexOf("--agent");
|
|
6223
|
+
const formatIndex = args.indexOf("--format");
|
|
6224
|
+
return args[0] === "run" && agentIndex >= 0 && Boolean(args[agentIndex + 1]) && formatIndex >= 0 && args[formatIndex + 1] === "json";
|
|
6225
|
+
}
|
|
6226
|
+
function guardianLauncherPath() {
|
|
6227
|
+
const extension = import.meta.url.endsWith(".ts") ? "ts" : "js";
|
|
6228
|
+
return resolve(dirname2(fileURLToPath2(import.meta.url)), `../worker/launcher.${extension}`).replaceAll("\\", "/");
|
|
6229
|
+
}
|
|
6230
|
+
function commandMatches(command, expectedExecutable, launchProtocol, expectedLaunchIdentity) {
|
|
6231
|
+
const expectedName = executableName(expectedExecutable);
|
|
6232
|
+
if (!expectedName) return false;
|
|
6233
|
+
const tokens = commandTokens(command);
|
|
6234
|
+
if (launchProtocol === TOKEN_GUARDIAN_LAUNCH_PROTOCOL) {
|
|
6235
|
+
if (!isLaunchIdentity(expectedLaunchIdentity)) return false;
|
|
6236
|
+
const expectedLauncher = guardianLauncherPath();
|
|
6237
|
+
const launcherIndex = tokens.findIndex((token, index2) => index2 > 0 && index2 <= 3 && token.replaceAll("\\", "/") === expectedLauncher);
|
|
6238
|
+
if (launcherIndex < 0) return false;
|
|
6239
|
+
const runtimeName = executableName(process.execPath);
|
|
6240
|
+
const hasRuntime = tokens.slice(0, launcherIndex).some((token) => executableName(token) === runtimeName || executableName(token) === "bun");
|
|
6241
|
+
const identityArgumentIndex = launcherIndex + 1;
|
|
6242
|
+
const launchIdentityIndex = launcherIndex + 2;
|
|
6243
|
+
const executableIndex2 = launcherIndex + 3;
|
|
6244
|
+
return hasRuntime && tokens[identityArgumentIndex] === LAUNCH_IDENTITY_ARGUMENT && tokens[launchIdentityIndex] === expectedLaunchIdentity && executableName(tokens[executableIndex2] ?? "") === expectedName && openCodeArgsMatch(tokens.slice(executableIndex2 + 1));
|
|
6245
|
+
}
|
|
6246
|
+
if (launchProtocol != null) return false;
|
|
6247
|
+
const executableIndex = tokens.findIndex((token, index2) => index2 <= 3 && executableName(token) === expectedName);
|
|
6248
|
+
if (executableIndex < 0) return false;
|
|
6249
|
+
return openCodeArgsMatch(tokens.slice(executableIndex + 1, executableIndex + 12));
|
|
6250
|
+
}
|
|
6251
|
+
function signalRecordedProcessTreeWithResult(pid, signal, expectedExecutable, launchProtocol = null, expectedLaunchIdentity = null) {
|
|
6252
|
+
if (!isSafePid(pid)) return "identity-mismatch";
|
|
6253
|
+
if (!isProcessAlive(pid)) return absentLeaderResult(pid);
|
|
6254
|
+
if (process.platform === "win32") {
|
|
6255
|
+
const command = inspectWindowsCommand(pid);
|
|
6256
|
+
if (!command) return isProcessAlive(pid) ? "inspect-failed" : absentLeaderResult(pid);
|
|
6257
|
+
if (!commandMatches(
|
|
6258
|
+
command,
|
|
6259
|
+
expectedExecutable,
|
|
6260
|
+
launchProtocol,
|
|
6261
|
+
expectedLaunchIdentity
|
|
6262
|
+
)) {
|
|
6263
|
+
return "identity-mismatch";
|
|
6264
|
+
}
|
|
6265
|
+
const args = ["/PID", String(pid), "/T"];
|
|
6266
|
+
if (signal === "SIGKILL") args.push("/F");
|
|
6267
|
+
const status = spawnSync("taskkill", args, {
|
|
6268
|
+
stdio: "ignore",
|
|
6269
|
+
timeout: OS_COMMAND_TIMEOUT_MS,
|
|
6270
|
+
killSignal: "SIGKILL"
|
|
6271
|
+
}).status;
|
|
6272
|
+
if (status === 0) return "signalled";
|
|
6273
|
+
return isProcessAlive(pid) ? "signal-failed" : absentLeaderResult(pid);
|
|
6274
|
+
}
|
|
6275
|
+
const info = inspectUnixProcess(pid);
|
|
6276
|
+
if (!info) return isProcessAlive(pid) ? "inspect-failed" : absentLeaderResult(pid);
|
|
6277
|
+
if (!commandMatches(
|
|
6278
|
+
info.command,
|
|
6279
|
+
expectedExecutable,
|
|
6280
|
+
launchProtocol,
|
|
6281
|
+
expectedLaunchIdentity
|
|
6282
|
+
)) {
|
|
6283
|
+
return "identity-mismatch";
|
|
6284
|
+
}
|
|
6285
|
+
try {
|
|
6286
|
+
if (info.processGroupId === pid) process.kill(-pid, signal);
|
|
6287
|
+
else process.kill(pid, signal);
|
|
6288
|
+
return "signalled";
|
|
6289
|
+
} catch {
|
|
6290
|
+
return isProcessAlive(pid) ? "signal-failed" : absentLeaderResult(pid);
|
|
6291
|
+
}
|
|
6292
|
+
}
|
|
6293
|
+
async function waitForSpawnedProcessTreeExit(pid, timeoutMs = 5e3) {
|
|
6294
|
+
const deadline = Date.now() + timeoutMs;
|
|
6295
|
+
while (inspectSpawnedProcessTreePresence(pid) !== "not-running" && Date.now() < deadline) {
|
|
6296
|
+
await Bun.sleep(Math.min(50, deadline - Date.now()));
|
|
6297
|
+
}
|
|
6298
|
+
return inspectSpawnedProcessTreePresence(pid) === "not-running";
|
|
6299
|
+
}
|
|
6300
|
+
|
|
5653
6301
|
// src/core/services/task-run.service.ts
|
|
5654
|
-
var { taskRuns: taskRuns3 } = schema_exports;
|
|
6302
|
+
var { tasks: tasks3, taskRuns: taskRuns3 } = schema_exports;
|
|
6303
|
+
var LegacyRunAbandonConflictError = class extends Error {
|
|
6304
|
+
constructor(message) {
|
|
6305
|
+
super(message);
|
|
6306
|
+
this.name = "LegacyRunAbandonConflictError";
|
|
6307
|
+
}
|
|
6308
|
+
};
|
|
5655
6309
|
var TaskRunService = class {
|
|
5656
6310
|
static async create(data) {
|
|
5657
6311
|
const result = await db.insert(taskRuns3).values(data).returning();
|
|
@@ -5681,12 +6335,12 @@ var TaskRunService = class {
|
|
|
5681
6335
|
const result = await db.update(taskRuns3).set({ heartbeatAt: Date.now() }).where(and(eq(taskRuns3.id, id), eq(taskRuns3.status, "running"))).returning();
|
|
5682
6336
|
return result[0] || null;
|
|
5683
6337
|
}
|
|
5684
|
-
static async updatePid(id, workerPid, childPid) {
|
|
6338
|
+
static async updatePid(id, workerPid, childPid, lockedBy = `gateway-${process.pid}`) {
|
|
5685
6339
|
const result = await db.update(taskRuns3).set({
|
|
5686
6340
|
workerPid,
|
|
5687
6341
|
childPid,
|
|
5688
6342
|
lockedAt: Date.now(),
|
|
5689
|
-
lockedBy
|
|
6343
|
+
lockedBy
|
|
5690
6344
|
}).where(and(eq(taskRuns3.id, id), eq(taskRuns3.status, "running"))).returning();
|
|
5691
6345
|
return result[0] || null;
|
|
5692
6346
|
}
|
|
@@ -5714,39 +6368,108 @@ var TaskRunService = class {
|
|
|
5714
6368
|
}
|
|
5715
6369
|
static async getStaleRuns(heartbeatTimeoutMs) {
|
|
5716
6370
|
const cutoffMs = Date.now() - heartbeatTimeoutMs;
|
|
5717
|
-
const cutoffSec = Math.floor(cutoffMs / 1e3);
|
|
5718
6371
|
const { tasks: tasksTable } = schema_exports;
|
|
5719
6372
|
const result = await db.select({
|
|
5720
6373
|
runId: taskRuns3.id,
|
|
5721
6374
|
taskId: taskRuns3.taskId,
|
|
5722
6375
|
childPid: taskRuns3.childPid,
|
|
6376
|
+
workerPid: taskRuns3.workerPid,
|
|
6377
|
+
launchProtocol: taskRuns3.launchProtocol,
|
|
6378
|
+
lockedBy: taskRuns3.lockedBy,
|
|
6379
|
+
startedAt: taskRuns3.startedAt,
|
|
6380
|
+
heartbeatAt: taskRuns3.heartbeatAt,
|
|
5723
6381
|
taskRetryCount: tasksTable.retryCount,
|
|
5724
6382
|
taskMaxRetries: tasksTable.maxRetries,
|
|
5725
|
-
taskRetryBackoffMs: tasksTable.retryBackoffMs
|
|
5726
|
-
|
|
5727
|
-
|
|
5728
|
-
|
|
5729
|
-
|
|
5730
|
-
|
|
5731
|
-
|
|
5732
|
-
|
|
5733
|
-
|
|
5734
|
-
and(
|
|
5735
|
-
sql`${taskRuns3.heartbeatAt} IS NOT NULL`,
|
|
5736
|
-
sql`${taskRuns3.heartbeatAt} < ${cutoffMs}`
|
|
5737
|
-
)
|
|
5738
|
-
)
|
|
5739
|
-
)
|
|
5740
|
-
);
|
|
5741
|
-
return result.map((row) => ({
|
|
6383
|
+
taskRetryBackoffMs: tasksTable.retryBackoffMs,
|
|
6384
|
+
taskStatus: tasksTable.status,
|
|
6385
|
+
taskCwd: tasksTable.cwd
|
|
6386
|
+
}).from(taskRuns3).innerJoin(tasksTable, eq(taskRuns3.taskId, tasksTable.id)).where(eq(taskRuns3.status, "running"));
|
|
6387
|
+
return result.filter((row) => {
|
|
6388
|
+
const heartbeatExpired = row.heartbeatAt == null ? row.startedAt == null || row.startedAt.getTime() < cutoffMs : row.heartbeatAt < cutoffMs;
|
|
6389
|
+
const ownerExited = row.workerPid != null && row.workerPid > 0 && !isProcessAlive(row.workerPid);
|
|
6390
|
+
return heartbeatExpired || ownerExited;
|
|
6391
|
+
}).map((row) => ({
|
|
5742
6392
|
runId: row.runId,
|
|
5743
6393
|
taskId: row.taskId,
|
|
5744
6394
|
childPid: row.childPid,
|
|
6395
|
+
workerPid: row.workerPid,
|
|
6396
|
+
launchProtocol: row.launchProtocol,
|
|
6397
|
+
lockedBy: row.lockedBy,
|
|
5745
6398
|
taskRetryCount: row.taskRetryCount ?? 0,
|
|
5746
6399
|
taskMaxRetries: row.taskMaxRetries ?? 3,
|
|
5747
|
-
taskRetryBackoffMs: row.taskRetryBackoffMs ?? 3e4
|
|
6400
|
+
taskRetryBackoffMs: row.taskRetryBackoffMs ?? 3e4,
|
|
6401
|
+
taskStatus: row.taskStatus,
|
|
6402
|
+
taskCwd: row.taskCwd,
|
|
6403
|
+
ownerAlive: row.workerPid != null && row.workerPid > 0 && isProcessAlive(row.workerPid)
|
|
6404
|
+
}));
|
|
6405
|
+
}
|
|
6406
|
+
static async listLegacyQuarantinedRuns(heartbeatTimeoutMs = 0) {
|
|
6407
|
+
const staleRuns = await this.getStaleRuns(heartbeatTimeoutMs);
|
|
6408
|
+
return staleRuns.filter(
|
|
6409
|
+
(row) => row.launchProtocol == null && row.childPid == null
|
|
6410
|
+
).map((row) => ({
|
|
6411
|
+
runId: row.runId,
|
|
6412
|
+
taskId: row.taskId,
|
|
6413
|
+
taskStatus: row.taskStatus,
|
|
6414
|
+
taskCwd: row.taskCwd,
|
|
6415
|
+
workerPid: row.workerPid,
|
|
6416
|
+
ownerAlive: row.ownerAlive
|
|
5748
6417
|
}));
|
|
5749
6418
|
}
|
|
6419
|
+
static async abandonLegacyRun(runId) {
|
|
6420
|
+
return db.transaction((tx) => {
|
|
6421
|
+
const current = tx.select({
|
|
6422
|
+
runId: taskRuns3.id,
|
|
6423
|
+
taskId: taskRuns3.taskId,
|
|
6424
|
+
runStatus: taskRuns3.status,
|
|
6425
|
+
taskStatus: tasks3.status,
|
|
6426
|
+
workerPid: taskRuns3.workerPid,
|
|
6427
|
+
childPid: taskRuns3.childPid,
|
|
6428
|
+
launchProtocol: taskRuns3.launchProtocol,
|
|
6429
|
+
log: taskRuns3.log
|
|
6430
|
+
}).from(taskRuns3).innerJoin(tasks3, eq(taskRuns3.taskId, tasks3.id)).where(eq(taskRuns3.id, runId)).get();
|
|
6431
|
+
if (!current) return null;
|
|
6432
|
+
if (current.runStatus !== "running") {
|
|
6433
|
+
throw new LegacyRunAbandonConflictError(`run #${runId} \u5DF2\u4E0D\u662F running \u72B6\u6001`);
|
|
6434
|
+
}
|
|
6435
|
+
if (current.launchProtocol != null) {
|
|
6436
|
+
throw new LegacyRunAbandonConflictError(
|
|
6437
|
+
`run #${runId} \u4F7F\u7528\u672A\u77E5\u6216\u53D7\u7BA1\u534F\u8BAE ${current.launchProtocol}\uFF0C\u7981\u6B62\u4EBA\u5DE5 abandon`
|
|
6438
|
+
);
|
|
6439
|
+
}
|
|
6440
|
+
if (current.childPid != null) {
|
|
6441
|
+
throw new LegacyRunAbandonConflictError(
|
|
6442
|
+
`run #${runId} \u5DF2\u8BB0\u5F55 child PID ${current.childPid}\uFF0C\u5FC5\u987B\u7531 Worker/Watchdog \u786E\u8BA4\u8FDB\u7A0B\u6811\u9000\u51FA`
|
|
6443
|
+
);
|
|
6444
|
+
}
|
|
6445
|
+
if (current.workerPid != null && isProcessAlive(current.workerPid)) {
|
|
6446
|
+
throw new LegacyRunAbandonConflictError(
|
|
6447
|
+
`run #${runId} \u7684 owner PID ${current.workerPid} \u4ECD\u5B58\u6D3B`
|
|
6448
|
+
);
|
|
6449
|
+
}
|
|
6450
|
+
if (current.taskStatus !== "cancelled") {
|
|
6451
|
+
throw new LegacyRunAbandonConflictError(
|
|
6452
|
+
`\u4EFB\u52A1 #${current.taskId} \u5FC5\u987B\u5148\u53D6\u6D88\u5E76\u4FDD\u6301 cancelled \u72B6\u6001`
|
|
6453
|
+
);
|
|
6454
|
+
}
|
|
6455
|
+
const note = "\u64CD\u4F5C\u5458\u5DF2\u786E\u8BA4\u65E7\u7248\u65E0 PID \u6267\u884C\u4E0D\u5B58\u5728\uFF0C\u5E76\u901A\u8FC7 run abandon \u5173\u95ED\u9694\u79BB\u8BB0\u5F55";
|
|
6456
|
+
const updated = tx.update(taskRuns3).set({
|
|
6457
|
+
status: "failed",
|
|
6458
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
6459
|
+
log: current.log ? `${current.log}
|
|
6460
|
+
${note}` : note
|
|
6461
|
+
}).where(and(eq(taskRuns3.id, runId), eq(taskRuns3.status, "running"))).returning({ id: taskRuns3.id }).get();
|
|
6462
|
+
if (!updated) {
|
|
6463
|
+
throw new LegacyRunAbandonConflictError(`run #${runId} \u72B6\u6001\u5DF2\u88AB\u5176\u4ED6\u64CD\u4F5C\u6539\u53D8`);
|
|
6464
|
+
}
|
|
6465
|
+
return {
|
|
6466
|
+
runId,
|
|
6467
|
+
taskId: current.taskId,
|
|
6468
|
+
runStatus: "failed",
|
|
6469
|
+
taskStatus: "cancelled"
|
|
6470
|
+
};
|
|
6471
|
+
}, { behavior: "immediate" });
|
|
6472
|
+
}
|
|
5750
6473
|
static async getRunningRunByTaskId(taskId) {
|
|
5751
6474
|
const result = await db.select().from(taskRuns3).where(and(eq(taskRuns3.taskId, taskId), eq(taskRuns3.status, "running"))).orderBy(desc(taskRuns3.startedAt), desc(taskRuns3.id)).limit(1);
|
|
5752
6475
|
return result[0] || null;
|
|
@@ -5769,31 +6492,38 @@ var state = null;
|
|
|
5769
6492
|
function markGatewayActivity(component) {
|
|
5770
6493
|
if (state) state.lastActivityAt[component] = Date.now();
|
|
5771
6494
|
}
|
|
5772
|
-
|
|
5773
|
-
|
|
5774
|
-
|
|
5775
|
-
|
|
6495
|
+
function markGatewaySuccess(component) {
|
|
6496
|
+
if (!state) return;
|
|
6497
|
+
const now = Date.now();
|
|
6498
|
+
state.lastActivityAt[component] = now;
|
|
6499
|
+
state.lastSuccessAt[component] = now;
|
|
6500
|
+
state.consecutiveFailures[component] = 0;
|
|
5776
6501
|
}
|
|
5777
|
-
function
|
|
5778
|
-
if (!
|
|
5779
|
-
|
|
5780
|
-
|
|
5781
|
-
|
|
5782
|
-
|
|
5783
|
-
|
|
5784
|
-
|
|
5785
|
-
}
|
|
5786
|
-
try {
|
|
5787
|
-
process.kill(pid, signal);
|
|
5788
|
-
return true;
|
|
5789
|
-
} catch {
|
|
5790
|
-
return false;
|
|
5791
|
-
}
|
|
6502
|
+
function markGatewayFailure(component, error) {
|
|
6503
|
+
if (!state) return;
|
|
6504
|
+
const now = Date.now();
|
|
6505
|
+
state.lastActivityAt[component] = now;
|
|
6506
|
+
state.consecutiveFailures[component] += 1;
|
|
6507
|
+
state.lastError[component] = {
|
|
6508
|
+
at: now,
|
|
6509
|
+
message: error instanceof Error ? error.message : String(error)
|
|
6510
|
+
};
|
|
5792
6511
|
}
|
|
5793
6512
|
|
|
5794
6513
|
// src/worker/index.ts
|
|
6514
|
+
import { fileURLToPath as fileURLToPath3 } from "url";
|
|
6515
|
+
import { existsSync as existsSync2 } from "fs";
|
|
6516
|
+
import { dirname as dirname3, join as join2 } from "path";
|
|
6517
|
+
import { randomUUID } from "crypto";
|
|
5795
6518
|
var DEFAULT_MAX_OUTPUT_CHARS = 64 * 1024;
|
|
5796
6519
|
var FORBIDDEN_AGENT = "supertask-runner";
|
|
6520
|
+
function assertWorkerProcessIsolationSupported(platform = process.platform) {
|
|
6521
|
+
if (platform === "win32") {
|
|
6522
|
+
throw new Error(
|
|
6523
|
+
"Windows Worker \u5DF2\u5B89\u5168\u7981\u7528\uFF1A\u5F53\u524D\u8FD0\u884C\u65F6\u65E0\u6CD5\u7528 Job Object \u8BC1\u660E\u6574\u4E2A OpenCode \u8FDB\u7A0B\u6811\u5DF2\u9000\u51FA"
|
|
6524
|
+
);
|
|
6525
|
+
}
|
|
6526
|
+
}
|
|
5797
6527
|
var WorkerEngine = class {
|
|
5798
6528
|
activeBatchIds = /* @__PURE__ */ new Set();
|
|
5799
6529
|
runningTasks = /* @__PURE__ */ new Map();
|
|
@@ -5810,10 +6540,13 @@ var WorkerEngine = class {
|
|
|
5810
6540
|
this.maxOutputChars = options.maxOutputChars ?? DEFAULT_MAX_OUTPUT_CHARS;
|
|
5811
6541
|
}
|
|
5812
6542
|
start() {
|
|
6543
|
+
assertWorkerProcessIsolationSupported();
|
|
5813
6544
|
this.stopped = false;
|
|
5814
6545
|
markGatewayActivity("worker");
|
|
5815
6546
|
this.poll();
|
|
5816
|
-
this.heartbeatTimer = setInterval(() =>
|
|
6547
|
+
this.heartbeatTimer = setInterval(() => {
|
|
6548
|
+
this.runDetached(this.updateHeartbeats(), "worker heartbeat cycle failed");
|
|
6549
|
+
}, this.cfg.heartbeatIntervalMs);
|
|
5817
6550
|
}
|
|
5818
6551
|
async stop(gracePeriodMs = 0) {
|
|
5819
6552
|
this.stopped = true;
|
|
@@ -5832,14 +6565,23 @@ var WorkerEngine = class {
|
|
|
5832
6565
|
clearInterval(this.heartbeatTimer);
|
|
5833
6566
|
this.heartbeatTimer = null;
|
|
5834
6567
|
}
|
|
5835
|
-
const
|
|
5836
|
-
const
|
|
5837
|
-
|
|
5838
|
-
entry.
|
|
5839
|
-
|
|
5840
|
-
|
|
5841
|
-
|
|
5842
|
-
|
|
6568
|
+
const interrupted = [];
|
|
6569
|
+
const entries = [...this.runningTasks.values()];
|
|
6570
|
+
await Promise.all(entries.map(async (entry) => {
|
|
6571
|
+
if (entry.settled) {
|
|
6572
|
+
if (entry.settlementPromise) await entry.settlementPromise;
|
|
6573
|
+
return;
|
|
6574
|
+
}
|
|
6575
|
+
const termination = entry.termination ?? {
|
|
6576
|
+
kind: "shutdown",
|
|
6577
|
+
message: "Gateway shutdown"
|
|
6578
|
+
};
|
|
6579
|
+
const terminated = await this.terminateEntry(entry, termination);
|
|
6580
|
+
if (terminated && termination.kind === "shutdown") {
|
|
6581
|
+
interrupted.push({ taskId: entry.task.id, runId: entry.runId });
|
|
6582
|
+
}
|
|
6583
|
+
}));
|
|
6584
|
+
return interrupted;
|
|
5843
6585
|
}
|
|
5844
6586
|
getRunningTaskIds() {
|
|
5845
6587
|
return [...this.runningTasks.keys()];
|
|
@@ -5847,24 +6589,40 @@ var WorkerEngine = class {
|
|
|
5847
6589
|
getRunningCount() {
|
|
5848
6590
|
return this.runningTasks.size;
|
|
5849
6591
|
}
|
|
6592
|
+
ownsRun(taskId, runId) {
|
|
6593
|
+
return this.runningTasks.get(taskId)?.runId === runId;
|
|
6594
|
+
}
|
|
5850
6595
|
poll() {
|
|
5851
6596
|
if (this.stopped) return;
|
|
5852
6597
|
markGatewayActivity("worker");
|
|
5853
|
-
this.pollCyclePromise = this.tryDispatch().
|
|
6598
|
+
this.pollCyclePromise = this.tryDispatch().then((healthy) => {
|
|
6599
|
+
if (healthy) markGatewaySuccess("worker");
|
|
6600
|
+
}).catch((err) => {
|
|
6601
|
+
markGatewayFailure("worker", err);
|
|
6602
|
+
this.logError("worker poll failed", err);
|
|
6603
|
+
}).finally(() => {
|
|
5854
6604
|
this.pollCyclePromise = null;
|
|
5855
6605
|
if (this.stopped) return;
|
|
5856
6606
|
this.pollTimer = setTimeout(() => this.poll(), this.cfg.pollIntervalMs);
|
|
5857
6607
|
});
|
|
5858
6608
|
}
|
|
5859
6609
|
async tryDispatch() {
|
|
6610
|
+
await TaskService.resetOrphanRunningToPending();
|
|
5860
6611
|
await this.reconcileCancelledTasks();
|
|
6612
|
+
await this.reconcileQuarantinedTasks();
|
|
6613
|
+
const quarantined = [...this.runningTasks.values()].filter((entry) => entry.quarantined);
|
|
6614
|
+
if (quarantined.length > 0) {
|
|
6615
|
+
return false;
|
|
6616
|
+
}
|
|
5861
6617
|
while (!this.stopped && this.runningTasks.size < this.cfg.maxConcurrency) {
|
|
6618
|
+
const databaseRunningCount = await TaskService.countRunning();
|
|
6619
|
+
if (databaseRunningCount >= this.cfg.maxConcurrency) break;
|
|
5862
6620
|
let task;
|
|
5863
6621
|
try {
|
|
5864
6622
|
task = await TaskService.next({ excludedBatchIds: [...this.activeBatchIds] });
|
|
5865
6623
|
} catch (err) {
|
|
5866
6624
|
this.logError("task claim failed", err);
|
|
5867
|
-
|
|
6625
|
+
throw err;
|
|
5868
6626
|
}
|
|
5869
6627
|
if (!task) break;
|
|
5870
6628
|
if (this.stopped) break;
|
|
@@ -5875,12 +6633,19 @@ var WorkerEngine = class {
|
|
|
5875
6633
|
this.releaseBatch(task);
|
|
5876
6634
|
break;
|
|
5877
6635
|
}
|
|
6636
|
+
let runId = null;
|
|
5878
6637
|
try {
|
|
6638
|
+
const launchIdentity = `gateway-${process.pid}:launch:${randomUUID()}`;
|
|
5879
6639
|
const run = await TaskRunService.create({
|
|
5880
6640
|
taskId: task.id,
|
|
5881
6641
|
model: this.resolveModel(task.model),
|
|
5882
|
-
status: "running"
|
|
6642
|
+
status: "running",
|
|
6643
|
+
workerPid: process.pid,
|
|
6644
|
+
lockedAt: Date.now(),
|
|
6645
|
+
lockedBy: launchIdentity,
|
|
6646
|
+
launchProtocol: TOKEN_GUARDIAN_LAUNCH_PROTOCOL
|
|
5883
6647
|
});
|
|
6648
|
+
runId = run.id;
|
|
5884
6649
|
if (this.stopped) {
|
|
5885
6650
|
await TaskRunService.fail(run.id, "Gateway shutdown before spawn");
|
|
5886
6651
|
await TaskService.resetRunningToPending([task.id]);
|
|
@@ -5889,49 +6654,82 @@ var WorkerEngine = class {
|
|
|
5889
6654
|
}
|
|
5890
6655
|
if (task.agent === FORBIDDEN_AGENT) {
|
|
5891
6656
|
const message = `\u7981\u6B62\u6267\u884C\u9012\u5F52 Agent: ${FORBIDDEN_AGENT}`;
|
|
5892
|
-
await
|
|
5893
|
-
await TaskService.fail(task.id, message, {}, { setDeadLetter: true });
|
|
6657
|
+
await TaskService.failRun(task.id, run.id, message, { setDeadLetter: true });
|
|
5894
6658
|
this.releaseBatch(task);
|
|
5895
6659
|
continue;
|
|
5896
6660
|
}
|
|
5897
|
-
this.spawnTask(task, run.id);
|
|
6661
|
+
await this.spawnTask(task, run.id, launchIdentity);
|
|
5898
6662
|
} catch (err) {
|
|
5899
6663
|
this.releaseBatch(task);
|
|
5900
6664
|
const message = `Worker \u542F\u52A8\u4EFB\u52A1\u5931\u8D25\uFF1A${err instanceof Error ? err.message : String(err)}`;
|
|
5901
6665
|
try {
|
|
5902
|
-
|
|
6666
|
+
if (runId == null) {
|
|
6667
|
+
await TaskService.fail(task.id, message);
|
|
6668
|
+
} else {
|
|
6669
|
+
const failed = await TaskService.failRun(task.id, runId, message);
|
|
6670
|
+
if (!failed) {
|
|
6671
|
+
await TaskRunService.fail(runId, `${message}
|
|
6672
|
+
\u4EFB\u52A1\u72B6\u6001\u5DF2\u88AB\u5176\u4ED6\u64CD\u4F5C\u6539\u53D8`);
|
|
6673
|
+
}
|
|
6674
|
+
}
|
|
5903
6675
|
} catch (failErr) {
|
|
5904
6676
|
this.logError("failed to compensate task startup", failErr, task.id);
|
|
5905
6677
|
}
|
|
5906
6678
|
this.logError("task dispatch failed", err, task.id);
|
|
5907
6679
|
}
|
|
5908
6680
|
}
|
|
6681
|
+
return ![...this.runningTasks.values()].some((entry) => entry.quarantined);
|
|
6682
|
+
}
|
|
6683
|
+
launcherEntry() {
|
|
6684
|
+
const extension = import.meta.url.endsWith(".ts") ? "ts" : "js";
|
|
6685
|
+
const moduleDir = dirname3(fileURLToPath3(import.meta.url));
|
|
6686
|
+
const candidates = [
|
|
6687
|
+
join2(moduleDir, `launcher.${extension}`),
|
|
6688
|
+
join2(moduleDir, `../worker/launcher.${extension}`)
|
|
6689
|
+
];
|
|
6690
|
+
const entry = candidates.find((candidate) => existsSync2(candidate));
|
|
6691
|
+
if (!entry) throw new Error(`Worker launcher \u4E0D\u5B58\u5728\uFF1A${candidates.join(", ")}`);
|
|
6692
|
+
return entry;
|
|
5909
6693
|
}
|
|
5910
|
-
spawnTask(task, runId) {
|
|
6694
|
+
async spawnTask(task, runId, launchIdentity) {
|
|
5911
6695
|
const model = this.resolveModel(task.model);
|
|
5912
6696
|
const args = ["run", "--agent", task.agent, "--format", "json"];
|
|
5913
6697
|
if (model) args.push("-m", model);
|
|
5914
6698
|
args.push(task.prompt);
|
|
5915
|
-
const child = spawn(
|
|
6699
|
+
const child = spawn(process.execPath, [
|
|
6700
|
+
this.launcherEntry(),
|
|
6701
|
+
LAUNCH_IDENTITY_ARGUMENT,
|
|
6702
|
+
launchIdentity,
|
|
6703
|
+
this.opencodeBin,
|
|
6704
|
+
...args
|
|
6705
|
+
], {
|
|
5916
6706
|
cwd: task.cwd || process.cwd(),
|
|
5917
|
-
|
|
6707
|
+
env: {
|
|
6708
|
+
...process.env,
|
|
6709
|
+
[MANAGED_RUN_ENV]: MANAGED_RUN_ENV_VALUE
|
|
6710
|
+
},
|
|
6711
|
+
stdio: ["pipe", "pipe", "pipe", "ipc"],
|
|
5918
6712
|
detached: process.platform !== "win32"
|
|
5919
6713
|
});
|
|
5920
6714
|
const entry = {
|
|
5921
6715
|
task,
|
|
5922
6716
|
runId,
|
|
6717
|
+
launchIdentity,
|
|
5923
6718
|
child,
|
|
5924
6719
|
output: "",
|
|
5925
6720
|
sessionId: null,
|
|
5926
6721
|
timeoutTimer: null,
|
|
5927
|
-
|
|
6722
|
+
termination: null,
|
|
6723
|
+
terminationPromise: null,
|
|
6724
|
+
settlementPromise: null,
|
|
6725
|
+
guardianDrained: false,
|
|
6726
|
+
quarantined: false,
|
|
5928
6727
|
settled: false
|
|
5929
6728
|
};
|
|
5930
6729
|
this.runningTasks.set(task.id, entry);
|
|
5931
6730
|
const handleData = (data) => {
|
|
5932
6731
|
const text2 = data.toString();
|
|
5933
6732
|
entry.output = (entry.output + text2).slice(-this.maxOutputChars);
|
|
5934
|
-
process.stdout.write(text2);
|
|
5935
6733
|
const match = entry.output.match(/"sessionID"\s*:\s*"(ses_[^"]+)"/);
|
|
5936
6734
|
if (match?.[1] && match[1] !== entry.sessionId) {
|
|
5937
6735
|
entry.sessionId = match[1];
|
|
@@ -5942,70 +6740,187 @@ var WorkerEngine = class {
|
|
|
5942
6740
|
};
|
|
5943
6741
|
child.stdout?.on("data", handleData);
|
|
5944
6742
|
child.stderr?.on("data", handleData);
|
|
5945
|
-
child.
|
|
5946
|
-
|
|
6743
|
+
child.on("message", (message) => {
|
|
6744
|
+
if (isMatchingDrainProof(message, entry.launchIdentity)) {
|
|
6745
|
+
entry.guardianDrained = true;
|
|
6746
|
+
}
|
|
6747
|
+
});
|
|
6748
|
+
let spawnError = null;
|
|
6749
|
+
const spawned = new Promise((resolve2, reject) => {
|
|
6750
|
+
child.once("spawn", resolve2);
|
|
6751
|
+
child.once("error", (error) => {
|
|
6752
|
+
spawnError = error;
|
|
6753
|
+
reject(error);
|
|
6754
|
+
});
|
|
5947
6755
|
});
|
|
5948
6756
|
child.once("close", (code, signal) => {
|
|
5949
|
-
|
|
5950
|
-
void this.finishEntry(entry, code, failure);
|
|
6757
|
+
this.handleChildClose(entry, code, signal, spawnError);
|
|
5951
6758
|
});
|
|
6759
|
+
try {
|
|
6760
|
+
await spawned;
|
|
6761
|
+
} catch (error) {
|
|
6762
|
+
entry.settled = true;
|
|
6763
|
+
this.runningTasks.delete(task.id);
|
|
6764
|
+
this.releaseBatch(task);
|
|
6765
|
+
throw error;
|
|
6766
|
+
}
|
|
6767
|
+
const childPid = child.pid;
|
|
6768
|
+
if (!childPid) {
|
|
6769
|
+
entry.settled = true;
|
|
6770
|
+
this.runningTasks.delete(task.id);
|
|
6771
|
+
this.releaseBatch(task);
|
|
6772
|
+
throw new Error("launcher \u672A\u8FD4\u56DE PID");
|
|
6773
|
+
}
|
|
6774
|
+
let pidRecorded = false;
|
|
6775
|
+
try {
|
|
6776
|
+
pidRecorded = await TaskRunService.updatePid(
|
|
6777
|
+
runId,
|
|
6778
|
+
process.pid,
|
|
6779
|
+
childPid,
|
|
6780
|
+
launchIdentity
|
|
6781
|
+
) !== null;
|
|
6782
|
+
} catch (error) {
|
|
6783
|
+
await this.terminateForFailure(
|
|
6784
|
+
entry,
|
|
6785
|
+
`\u8BB0\u5F55 Worker PID \u5931\u8D25\uFF1A${error instanceof Error ? error.message : String(error)}`
|
|
6786
|
+
);
|
|
6787
|
+
return;
|
|
6788
|
+
}
|
|
6789
|
+
if (!pidRecorded) {
|
|
6790
|
+
await this.terminateForFailure(entry, "\u8BB0\u5F55 Worker PID \u5931\u8D25\uFF1Arun \u5DF2\u4E0D\u518D\u5904\u4E8E running \u72B6\u6001");
|
|
6791
|
+
return;
|
|
6792
|
+
}
|
|
6793
|
+
if (!child.stdin) {
|
|
6794
|
+
await this.terminateForFailure(entry, "\u653E\u884C OpenCode \u5931\u8D25\uFF1Alauncher stdin \u4E0D\u53EF\u7528");
|
|
6795
|
+
return;
|
|
6796
|
+
}
|
|
6797
|
+
try {
|
|
6798
|
+
await new Promise((resolve2, reject) => {
|
|
6799
|
+
child.stdin.end("START\n", (error) => {
|
|
6800
|
+
if (error) reject(error);
|
|
6801
|
+
else resolve2();
|
|
6802
|
+
});
|
|
6803
|
+
});
|
|
6804
|
+
} catch (error) {
|
|
6805
|
+
await this.terminateForFailure(
|
|
6806
|
+
entry,
|
|
6807
|
+
`\u653E\u884C OpenCode \u5931\u8D25\uFF1A${error instanceof Error ? error.message : String(error)}`
|
|
6808
|
+
);
|
|
6809
|
+
return;
|
|
6810
|
+
}
|
|
5952
6811
|
const timeoutMs = task.timeoutMs ?? this.cfg.taskTimeoutMs;
|
|
5953
6812
|
if (timeoutMs > 0) {
|
|
5954
6813
|
entry.timeoutTimer = setTimeout(() => {
|
|
5955
|
-
this.
|
|
5956
|
-
|
|
6814
|
+
this.runDetached(
|
|
6815
|
+
this.terminateForFailure(entry, `\u4EFB\u52A1\u8D85\u65F6\uFF08${timeoutMs}ms\uFF09`, "SIGKILL"),
|
|
6816
|
+
"timeout termination failed",
|
|
6817
|
+
entry.task.id
|
|
6818
|
+
);
|
|
5957
6819
|
}, timeoutMs);
|
|
5958
6820
|
}
|
|
5959
|
-
TaskRunService.updatePid(runId, process.pid, child.pid ?? 0).catch((err) => {
|
|
5960
|
-
this.signalEntry(entry, "SIGKILL");
|
|
5961
|
-
void this.finishEntry(entry, null, `\u8BB0\u5F55 Worker PID \u5931\u8D25\uFF1A${err instanceof Error ? err.message : String(err)}`);
|
|
5962
|
-
});
|
|
5963
6821
|
}
|
|
5964
|
-
|
|
5965
|
-
if (entry.settled) return;
|
|
6822
|
+
handleChildClose(entry, code, signal, spawnError) {
|
|
6823
|
+
if (entry.settled || entry.termination) return;
|
|
6824
|
+
if (!entry.guardianDrained) {
|
|
6825
|
+
const pid = entry.child.pid;
|
|
6826
|
+
const presence = pid == null ? spawnError == null ? "unknown" : "not-running" : inspectSpawnedProcessTreePresence(pid);
|
|
6827
|
+
const message = "guardian \u672A\u63D0\u4F9B\u8FDB\u7A0B\u6811\u6392\u7A7A\u8BC1\u660E";
|
|
6828
|
+
if (presence !== "not-running") {
|
|
6829
|
+
if (entry.timeoutTimer) {
|
|
6830
|
+
clearTimeout(entry.timeoutTimer);
|
|
6831
|
+
entry.timeoutTimer = null;
|
|
6832
|
+
}
|
|
6833
|
+
entry.termination = { kind: "failure", message };
|
|
6834
|
+
entry.quarantined = true;
|
|
6835
|
+
markGatewayFailure(
|
|
6836
|
+
"worker",
|
|
6837
|
+
new Error(`${message}\uFF1B\u4EFB\u52A1 #${entry.task.id} \u4FDD\u6301\u9694\u79BB`)
|
|
6838
|
+
);
|
|
6839
|
+
return;
|
|
6840
|
+
}
|
|
6841
|
+
this.runDetached(
|
|
6842
|
+
this.settleEntry(entry, null, message),
|
|
6843
|
+
"task settlement failed",
|
|
6844
|
+
entry.task.id
|
|
6845
|
+
);
|
|
6846
|
+
return;
|
|
6847
|
+
}
|
|
6848
|
+
const failure = code === 0 ? void 0 : `${spawnError ? "\u65E0\u6CD5\u542F\u52A8 opencode" : "opencode \u9000\u51FA\u7801"} ${spawnError?.message ?? code ?? "null"}${signal ? `\uFF0C\u4FE1\u53F7 ${signal}` : ""}`;
|
|
6849
|
+
this.runDetached(
|
|
6850
|
+
this.settleEntry(entry, code, failure),
|
|
6851
|
+
"task settlement failed",
|
|
6852
|
+
entry.task.id
|
|
6853
|
+
);
|
|
6854
|
+
}
|
|
6855
|
+
settleEntry(entry, code, failure) {
|
|
6856
|
+
if (entry.settlementPromise) return entry.settlementPromise;
|
|
6857
|
+
if (entry.settled) return Promise.resolve(false);
|
|
5966
6858
|
entry.settled = true;
|
|
5967
6859
|
if (entry.timeoutTimer) {
|
|
5968
6860
|
clearTimeout(entry.timeoutTimer);
|
|
5969
6861
|
entry.timeoutTimer = null;
|
|
5970
6862
|
}
|
|
5971
|
-
|
|
5972
|
-
|
|
5973
|
-
|
|
5974
|
-
|
|
5975
|
-
|
|
5976
|
-
|
|
6863
|
+
const settlement = this.commitEntry(entry, code, failure).then(() => true).catch((error) => {
|
|
6864
|
+
markGatewayFailure("worker", error);
|
|
6865
|
+
this.logError("task settlement failed", error, entry.task.id);
|
|
6866
|
+
return false;
|
|
6867
|
+
}).finally(() => {
|
|
6868
|
+
this.runningTasks.delete(entry.task.id);
|
|
6869
|
+
this.releaseBatch(entry.task);
|
|
6870
|
+
});
|
|
6871
|
+
entry.settlementPromise = settlement;
|
|
6872
|
+
return settlement;
|
|
6873
|
+
}
|
|
6874
|
+
async commitEntry(entry, code, failure) {
|
|
6875
|
+
const termination = entry.termination;
|
|
6876
|
+
if (termination?.kind === "shutdown") return;
|
|
6877
|
+
if (termination?.kind === "cancel") {
|
|
6878
|
+
const output2 = entry.output.trim();
|
|
6879
|
+
const log2 = `${termination.message}${output2 ? `
|
|
6880
|
+
${output2}` : ""}`;
|
|
6881
|
+
await TaskRunService.fail(entry.runId, log2);
|
|
6882
|
+
console.log(JSON.stringify({
|
|
6883
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
6884
|
+
level: "info",
|
|
6885
|
+
msg: "running task cancelled",
|
|
6886
|
+
taskId: entry.task.id
|
|
6887
|
+
}));
|
|
6888
|
+
return;
|
|
6889
|
+
}
|
|
6890
|
+
const currentRun = await TaskRunService.getById(entry.runId);
|
|
6891
|
+
if (!currentRun || currentRun.status !== "running") return;
|
|
6892
|
+
const output = entry.output.trim();
|
|
6893
|
+
const log = failure ? `${failure}${output ? `
|
|
5977
6894
|
${output}` : ""}` : output;
|
|
5978
|
-
|
|
5979
|
-
|
|
5980
|
-
|
|
5981
|
-
|
|
5982
|
-
|
|
5983
|
-
|
|
5984
|
-
|
|
5985
|
-
|
|
5986
|
-
|
|
5987
|
-
}));
|
|
5988
|
-
return;
|
|
5989
|
-
}
|
|
5990
|
-
await TaskRunService.fail(entry.runId, "\u4EFB\u52A1\u72B6\u6001\u5DF2\u88AB\u5176\u4ED6\u64CD\u4F5C\u6539\u53D8");
|
|
6895
|
+
if (code === 0 && !failure) {
|
|
6896
|
+
const completed = await TaskService.completeRun(entry.task.id, entry.runId, log);
|
|
6897
|
+
if (completed) {
|
|
6898
|
+
console.log(JSON.stringify({
|
|
6899
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
6900
|
+
level: "info",
|
|
6901
|
+
msg: "task done",
|
|
6902
|
+
taskId: entry.task.id
|
|
6903
|
+
}));
|
|
5991
6904
|
return;
|
|
5992
6905
|
}
|
|
5993
|
-
await TaskRunService.fail(entry.runId,
|
|
5994
|
-
|
|
5995
|
-
if (!failed) {
|
|
5996
|
-
this.logError("task failure state transition rejected", failure ?? "unknown failure", entry.task.id);
|
|
5997
|
-
}
|
|
5998
|
-
console.error(JSON.stringify({
|
|
5999
|
-
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
6000
|
-
level: "error",
|
|
6001
|
-
msg: "task failed",
|
|
6002
|
-
taskId: entry.task.id,
|
|
6003
|
-
error: failure
|
|
6004
|
-
}));
|
|
6005
|
-
} finally {
|
|
6006
|
-
this.runningTasks.delete(entry.task.id);
|
|
6007
|
-
this.releaseBatch(entry.task);
|
|
6906
|
+
await TaskRunService.fail(entry.runId, "\u4EFB\u52A1\u6216\u6267\u884C\u8BB0\u5F55\u72B6\u6001\u5DF2\u88AB\u5176\u4ED6\u64CD\u4F5C\u6539\u53D8");
|
|
6907
|
+
return;
|
|
6008
6908
|
}
|
|
6909
|
+
const failed = await TaskService.failRun(entry.task.id, entry.runId, log);
|
|
6910
|
+
if (!failed) {
|
|
6911
|
+
await TaskRunService.fail(
|
|
6912
|
+
entry.runId,
|
|
6913
|
+
`${log}${log ? "\n" : ""}\u4EFB\u52A1\u72B6\u6001\u5DF2\u88AB\u5176\u4ED6\u64CD\u4F5C\u6539\u53D8`
|
|
6914
|
+
);
|
|
6915
|
+
this.logError("task failure state transition rejected", failure ?? "unknown failure", entry.task.id);
|
|
6916
|
+
}
|
|
6917
|
+
console.error(JSON.stringify({
|
|
6918
|
+
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
6919
|
+
level: "error",
|
|
6920
|
+
msg: "task failed",
|
|
6921
|
+
taskId: entry.task.id,
|
|
6922
|
+
error: failure
|
|
6923
|
+
}));
|
|
6009
6924
|
}
|
|
6010
6925
|
async reconcileCancelledTasks() {
|
|
6011
6926
|
for (const entry of [...this.runningTasks.values()]) {
|
|
@@ -6017,32 +6932,18 @@ ${output}` : ""}` : output;
|
|
|
6017
6932
|
}
|
|
6018
6933
|
}
|
|
6019
6934
|
}
|
|
6020
|
-
async
|
|
6021
|
-
|
|
6022
|
-
|
|
6023
|
-
|
|
6024
|
-
clearTimeout(entry.timeoutTimer);
|
|
6025
|
-
entry.timeoutTimer = null;
|
|
6026
|
-
}
|
|
6027
|
-
try {
|
|
6028
|
-
await this.killEntry(entry);
|
|
6029
|
-
const output = entry.output.trim();
|
|
6030
|
-
const log = `\u4EFB\u52A1\u5DF2\u53D6\u6D88${output ? `
|
|
6031
|
-
${output}` : ""}`;
|
|
6032
|
-
await TaskRunService.fail(entry.runId, log);
|
|
6033
|
-
console.log(JSON.stringify({
|
|
6034
|
-
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
6035
|
-
level: "info",
|
|
6036
|
-
msg: "running task cancelled",
|
|
6037
|
-
taskId: entry.task.id
|
|
6038
|
-
}));
|
|
6039
|
-
} finally {
|
|
6040
|
-
this.runningTasks.delete(entry.task.id);
|
|
6041
|
-
this.releaseBatch(entry.task);
|
|
6935
|
+
async reconcileQuarantinedTasks() {
|
|
6936
|
+
for (const entry of [...this.runningTasks.values()]) {
|
|
6937
|
+
if (!entry.quarantined || !entry.termination) continue;
|
|
6938
|
+
await this.terminateEntry(entry, entry.termination);
|
|
6042
6939
|
}
|
|
6043
6940
|
}
|
|
6941
|
+
async cancelEntry(entry) {
|
|
6942
|
+
await this.terminateEntry(entry, { kind: "cancel", message: "\u4EFB\u52A1\u5DF2\u53D6\u6D88" });
|
|
6943
|
+
}
|
|
6044
6944
|
async updateHeartbeats() {
|
|
6045
6945
|
for (const entry of this.runningTasks.values()) {
|
|
6946
|
+
if (entry.quarantined) continue;
|
|
6046
6947
|
try {
|
|
6047
6948
|
await TaskRunService.heartbeat(entry.runId);
|
|
6048
6949
|
} catch (err) {
|
|
@@ -6050,26 +6951,75 @@ ${output}` : ""}`;
|
|
|
6050
6951
|
}
|
|
6051
6952
|
}
|
|
6052
6953
|
}
|
|
6053
|
-
|
|
6054
|
-
|
|
6055
|
-
|
|
6954
|
+
async terminateForFailure(entry, message, initialSignal = "SIGTERM") {
|
|
6955
|
+
return this.terminateEntry(entry, { kind: "failure", message }, initialSignal);
|
|
6956
|
+
}
|
|
6957
|
+
terminateEntry(entry, termination, initialSignal = "SIGTERM") {
|
|
6958
|
+
if (entry.terminationPromise) return entry.terminationPromise;
|
|
6959
|
+
if (entry.settled) return entry.settlementPromise ?? Promise.resolve(false);
|
|
6960
|
+
entry.termination ??= termination;
|
|
6961
|
+
if (entry.timeoutTimer) {
|
|
6962
|
+
clearTimeout(entry.timeoutTimer);
|
|
6963
|
+
entry.timeoutTimer = null;
|
|
6056
6964
|
}
|
|
6057
|
-
|
|
6058
|
-
|
|
6059
|
-
|
|
6060
|
-
|
|
6061
|
-
}
|
|
6062
|
-
entry.child.once("close", () => {
|
|
6063
|
-
clearTimeout(timeout);
|
|
6064
|
-
resolve();
|
|
6065
|
-
});
|
|
6066
|
-
this.signalEntry(entry, "SIGTERM");
|
|
6965
|
+
entry.quarantined = true;
|
|
6966
|
+
const terminationPromise = this.completeTermination(entry, initialSignal).finally(() => {
|
|
6967
|
+
if (!entry.settled && entry.terminationPromise === terminationPromise) {
|
|
6968
|
+
entry.terminationPromise = null;
|
|
6969
|
+
}
|
|
6067
6970
|
});
|
|
6971
|
+
entry.terminationPromise = terminationPromise;
|
|
6972
|
+
return terminationPromise;
|
|
6973
|
+
}
|
|
6974
|
+
async completeTermination(entry, initialSignal) {
|
|
6975
|
+
const termination = entry.termination;
|
|
6976
|
+
if (!termination) return false;
|
|
6977
|
+
try {
|
|
6978
|
+
const exited = await this.killEntry(entry, initialSignal);
|
|
6979
|
+
if (!exited) {
|
|
6980
|
+
markGatewayFailure(
|
|
6981
|
+
"worker",
|
|
6982
|
+
new Error(`${termination.message}\uFF1B\u4EFB\u52A1 #${entry.task.id} \u7684\u8FDB\u7A0B\u65E0\u6CD5\u786E\u8BA4\u9000\u51FA`)
|
|
6983
|
+
);
|
|
6984
|
+
return false;
|
|
6985
|
+
}
|
|
6986
|
+
entry.quarantined = false;
|
|
6987
|
+
return await this.settleEntry(
|
|
6988
|
+
entry,
|
|
6989
|
+
null,
|
|
6990
|
+
termination.kind === "failure" ? termination.message : void 0
|
|
6991
|
+
);
|
|
6992
|
+
} catch (error) {
|
|
6993
|
+
markGatewayFailure("worker", error);
|
|
6994
|
+
this.logError("task termination failed", error, entry.task.id);
|
|
6995
|
+
return false;
|
|
6996
|
+
}
|
|
6997
|
+
}
|
|
6998
|
+
async killEntry(entry, initialSignal = "SIGTERM") {
|
|
6999
|
+
if (entry.guardianDrained && (entry.child.exitCode !== null || entry.child.signalCode !== null)) return true;
|
|
7000
|
+
const pid = entry.child.pid;
|
|
7001
|
+
if (!pid) return false;
|
|
7002
|
+
const initialResult = this.signalEntry(entry, initialSignal);
|
|
7003
|
+
if (initialResult === "not-running") return true;
|
|
7004
|
+
if (initialResult !== "signalled") return false;
|
|
7005
|
+
if (await waitForSpawnedProcessTreeExit(pid, 2500)) return true;
|
|
7006
|
+
if (initialSignal === "SIGKILL") return false;
|
|
7007
|
+
if (entry.guardianDrained && (entry.child.exitCode !== null || entry.child.signalCode !== null)) return true;
|
|
7008
|
+
const forcedResult = this.signalEntry(entry, "SIGKILL");
|
|
7009
|
+
if (forcedResult === "not-running") return true;
|
|
7010
|
+
if (forcedResult !== "signalled") return false;
|
|
7011
|
+
return waitForSpawnedProcessTreeExit(pid, 2500);
|
|
6068
7012
|
}
|
|
6069
7013
|
signalEntry(entry, signal) {
|
|
6070
7014
|
const pid = entry.child.pid;
|
|
6071
|
-
if (!pid) return;
|
|
6072
|
-
|
|
7015
|
+
if (!pid) return "not-running";
|
|
7016
|
+
return signalRecordedProcessTreeWithResult(
|
|
7017
|
+
pid,
|
|
7018
|
+
signal,
|
|
7019
|
+
this.opencodeBin,
|
|
7020
|
+
TOKEN_GUARDIAN_LAUNCH_PROTOCOL,
|
|
7021
|
+
entry.launchIdentity
|
|
7022
|
+
);
|
|
6073
7023
|
}
|
|
6074
7024
|
releaseBatch(task) {
|
|
6075
7025
|
if (task.batchId) this.activeBatchIds.delete(task.batchId);
|
|
@@ -6078,6 +7028,12 @@ ${output}` : ""}`;
|
|
|
6078
7028
|
if (!taskModel || taskModel === "default") return null;
|
|
6079
7029
|
return taskModel;
|
|
6080
7030
|
}
|
|
7031
|
+
runDetached(operation, message, taskId) {
|
|
7032
|
+
operation.catch((error) => {
|
|
7033
|
+
markGatewayFailure("worker", error);
|
|
7034
|
+
this.logError(message, error, taskId);
|
|
7035
|
+
});
|
|
7036
|
+
}
|
|
6081
7037
|
logError(message, error, taskId) {
|
|
6082
7038
|
console.error(JSON.stringify({
|
|
6083
7039
|
ts: (/* @__PURE__ */ new Date()).toISOString(),
|
|
@@ -6089,6 +7045,7 @@ ${output}` : ""}`;
|
|
|
6089
7045
|
}
|
|
6090
7046
|
};
|
|
6091
7047
|
export {
|
|
6092
|
-
WorkerEngine
|
|
7048
|
+
WorkerEngine,
|
|
7049
|
+
assertWorkerProcessIsolationSupported
|
|
6093
7050
|
};
|
|
6094
7051
|
//# sourceMappingURL=index.js.map
|