opencode-supertask 0.1.25 → 0.1.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +55 -17
- package/dist/cli/index.js +3564 -1090
- package/dist/cli/index.js.map +1 -1
- package/dist/daemon/pm2-supervisor.d.ts +3 -0
- package/dist/daemon/pm2-supervisor.js +145 -0
- package/dist/daemon/pm2-supervisor.js.map +1 -0
- package/dist/gateway/index.d.ts +8 -1
- package/dist/gateway/index.js +2131 -681
- package/dist/gateway/index.js.map +1 -1
- package/dist/plugin/supertask.d.ts +2 -1
- package/dist/plugin/supertask.js +1352 -345
- package/dist/plugin/supertask.js.map +1 -1
- package/dist/web/index.d.ts +1 -0
- package/dist/web/index.js +1042 -264
- package/dist/web/index.js.map +1 -1
- package/dist/worker/index.d.ts +17 -3
- package/dist/worker/index.js +1204 -247
- package/dist/worker/index.js.map +1 -1
- package/dist/worker/launcher.d.ts +9 -0
- package/dist/worker/launcher.js +164 -0
- package/dist/worker/launcher.js.map +1 -0
- package/drizzle/0005_natural_ma_gnuci.sql +1 -0
- package/drizzle/0006_worried_agent_zero.sql +2 -0
- package/drizzle/0007_lean_titanium_man.sql +1 -0
- package/drizzle/meta/0005_snapshot.json +559 -0
- package/drizzle/meta/0006_snapshot.json +575 -0
- package/drizzle/meta/0007_snapshot.json +585 -0
- package/drizzle/meta/_journal.json +21 -0
- package/package.json +1 -1
package/dist/web/index.js
CHANGED
|
@@ -16521,7 +16521,9 @@ var tasks = sqliteTable("tasks", {
|
|
|
16521
16521
|
}, (table) => [
|
|
16522
16522
|
index("tasks_queue_idx").on(table.status, table.retryAfter, table.urgency, table.importance, table.createdAt, table.id),
|
|
16523
16523
|
index("tasks_batch_status_idx").on(table.batchId, table.status),
|
|
16524
|
-
index("tasks_template_status_idx").on(table.templateId, table.status)
|
|
16524
|
+
index("tasks_template_status_idx").on(table.templateId, table.status),
|
|
16525
|
+
index("tasks_depends_on_status_idx").on(table.dependsOn, table.status),
|
|
16526
|
+
index("tasks_cleanup_idx").on(table.finishedAt, table.id, table.status)
|
|
16525
16527
|
]);
|
|
16526
16528
|
var TASK_CATEGORIES = [
|
|
16527
16529
|
"translate",
|
|
@@ -16544,7 +16546,8 @@ var taskRuns = sqliteTable("task_runs", {
|
|
|
16544
16546
|
lockedBy: text("locked_by"),
|
|
16545
16547
|
heartbeatAt: integer("heartbeat_at"),
|
|
16546
16548
|
workerPid: integer("worker_pid"),
|
|
16547
|
-
childPid: integer("child_pid")
|
|
16549
|
+
childPid: integer("child_pid"),
|
|
16550
|
+
launchProtocol: text("launch_protocol")
|
|
16548
16551
|
}, (table) => [
|
|
16549
16552
|
index("task_runs_task_started_idx").on(table.taskId, table.startedAt, table.id),
|
|
16550
16553
|
index("task_runs_status_heartbeat_idx").on(table.status, table.heartbeatAt)
|
|
@@ -16574,7 +16577,13 @@ var taskTemplates = sqliteTable("task_templates", {
|
|
|
16574
16577
|
createdAt: integer("created_at").default(0),
|
|
16575
16578
|
updatedAt: integer("updated_at").default(0)
|
|
16576
16579
|
}, (table) => [
|
|
16577
|
-
index("task_templates_due_idx").on(table.enabled, table.nextRunAt, table.id)
|
|
16580
|
+
index("task_templates_due_idx").on(table.enabled, table.nextRunAt, table.id),
|
|
16581
|
+
index("task_templates_retention_idx").on(
|
|
16582
|
+
table.scheduleType,
|
|
16583
|
+
table.enabled,
|
|
16584
|
+
table.lastRunAt,
|
|
16585
|
+
table.id
|
|
16586
|
+
)
|
|
16578
16587
|
]);
|
|
16579
16588
|
|
|
16580
16589
|
// src/core/db/index.ts
|
|
@@ -16599,36 +16608,47 @@ function getMigrationsFolder() {
|
|
|
16599
16608
|
}
|
|
16600
16609
|
return join(__dirname, "../../drizzle");
|
|
16601
16610
|
}
|
|
16602
|
-
function
|
|
16603
|
-
|
|
16604
|
-
if (!existsSync(dataDir)) {
|
|
16605
|
-
mkdirSync(dataDir, { recursive: true });
|
|
16606
|
-
}
|
|
16607
|
-
_sqlite = new Database2(DB_FILE_PATH);
|
|
16608
|
-
_sqlite.exec("PRAGMA journal_mode = WAL;");
|
|
16609
|
-
_sqlite.exec("PRAGMA busy_timeout = 5000;");
|
|
16610
|
-
_sqlite.exec(`
|
|
16611
|
+
function ensureGatewayLock(sqliteDb) {
|
|
16612
|
+
sqliteDb.exec(`
|
|
16611
16613
|
CREATE TABLE IF NOT EXISTS gateway_lock (
|
|
16612
16614
|
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
16613
16615
|
pid INTEGER NOT NULL,
|
|
16614
16616
|
acquired_at INTEGER NOT NULL,
|
|
16615
16617
|
heartbeat_at INTEGER NOT NULL,
|
|
16616
|
-
ready_at INTEGER
|
|
16618
|
+
ready_at INTEGER,
|
|
16619
|
+
version TEXT
|
|
16617
16620
|
);
|
|
16618
16621
|
`);
|
|
16619
|
-
const
|
|
16620
|
-
if (!
|
|
16621
|
-
|
|
16622
|
+
const columns = sqliteDb.query("PRAGMA table_info(gateway_lock)").all();
|
|
16623
|
+
if (!columns.some((column) => column.name === "ready_at")) {
|
|
16624
|
+
sqliteDb.exec("ALTER TABLE gateway_lock ADD COLUMN ready_at INTEGER;");
|
|
16625
|
+
}
|
|
16626
|
+
if (!columns.some((column) => column.name === "version")) {
|
|
16627
|
+
sqliteDb.exec("ALTER TABLE gateway_lock ADD COLUMN version TEXT;");
|
|
16622
16628
|
}
|
|
16623
|
-
|
|
16629
|
+
}
|
|
16630
|
+
function migrateSqliteDatabase(sqliteDb) {
|
|
16631
|
+
ensureGatewayLock(sqliteDb);
|
|
16632
|
+
const drizzleDb = drizzle(sqliteDb, { schema: schema_exports });
|
|
16633
|
+
migrate(drizzleDb, { migrationsFolder: getMigrationsFolder() });
|
|
16634
|
+
sqliteDb.exec("PRAGMA foreign_keys = ON;");
|
|
16635
|
+
const violations = sqliteDb.query("PRAGMA foreign_key_check;").all();
|
|
16636
|
+
if (violations.length > 0) {
|
|
16637
|
+
throw new Error(`\u68C0\u6D4B\u5230 ${violations.length} \u6761\u5B64\u7ACB\u5173\u8054\u8BB0\u5F55\uFF0C\u8BF7\u5148\u4FEE\u590D\u6570\u636E\u518D\u542F\u52A8`);
|
|
16638
|
+
}
|
|
16639
|
+
return drizzleDb;
|
|
16640
|
+
}
|
|
16641
|
+
function initDb() {
|
|
16642
|
+
const dataDir = dirname(DB_FILE_PATH);
|
|
16643
|
+
if (!existsSync(dataDir)) {
|
|
16644
|
+
mkdirSync(dataDir, { recursive: true });
|
|
16645
|
+
}
|
|
16646
|
+
_sqlite = new Database2(DB_FILE_PATH);
|
|
16647
|
+
_sqlite.exec("PRAGMA journal_mode = WAL;");
|
|
16648
|
+
_sqlite.exec("PRAGMA busy_timeout = 5000;");
|
|
16624
16649
|
if (!_migrationRan) {
|
|
16625
16650
|
try {
|
|
16626
|
-
|
|
16627
|
-
_sqlite.exec("PRAGMA foreign_keys = ON;");
|
|
16628
|
-
const violations = _sqlite.query("PRAGMA foreign_key_check;").all();
|
|
16629
|
-
if (violations.length > 0) {
|
|
16630
|
-
throw new Error(`\u68C0\u6D4B\u5230 ${violations.length} \u6761\u5B64\u7ACB\u5173\u8054\u8BB0\u5F55\uFF0C\u8BF7\u5148\u4FEE\u590D\u6570\u636E\u518D\u542F\u52A8`);
|
|
16631
|
-
}
|
|
16651
|
+
_db = migrateSqliteDatabase(_sqlite);
|
|
16632
16652
|
_migrationRan = true;
|
|
16633
16653
|
} catch (err) {
|
|
16634
16654
|
const msg = err instanceof Error ? err.message : String(err);
|
|
@@ -16639,6 +16659,8 @@ function initDb() {
|
|
|
16639
16659
|
console.error(`[supertask] migration failed: ${msg}`);
|
|
16640
16660
|
throw new Error(`[supertask] DB migration failed: ${msg}`);
|
|
16641
16661
|
}
|
|
16662
|
+
} else {
|
|
16663
|
+
_db = drizzle(_sqlite, { schema: schema_exports });
|
|
16642
16664
|
}
|
|
16643
16665
|
return _db;
|
|
16644
16666
|
}
|
|
@@ -16650,14 +16672,6 @@ function getSqlite() {
|
|
|
16650
16672
|
if (!_sqlite) initDb();
|
|
16651
16673
|
return _sqlite;
|
|
16652
16674
|
}
|
|
16653
|
-
function closeDb() {
|
|
16654
|
-
if (_sqlite) {
|
|
16655
|
-
_sqlite.close();
|
|
16656
|
-
_sqlite = null;
|
|
16657
|
-
_db = null;
|
|
16658
|
-
_migrationRan = false;
|
|
16659
|
-
}
|
|
16660
|
-
}
|
|
16661
16675
|
var db = new Proxy({}, {
|
|
16662
16676
|
get(_, prop) {
|
|
16663
16677
|
const target = getDb();
|
|
@@ -16683,6 +16697,58 @@ function computeBackoff(retryCount, baseMs = 3e4, maxMs = MAX_BACKOFF_MS) {
|
|
|
16683
16697
|
|
|
16684
16698
|
// src/core/services/task.service.ts
|
|
16685
16699
|
var { tasks: tasks2, taskRuns: taskRuns2 } = schema_exports;
|
|
16700
|
+
var cleanupInvocation = 0;
|
|
16701
|
+
var TaskDeletionConflictError = class extends Error {
|
|
16702
|
+
constructor(message) {
|
|
16703
|
+
super(message);
|
|
16704
|
+
this.name = "TaskDeletionConflictError";
|
|
16705
|
+
}
|
|
16706
|
+
};
|
|
16707
|
+
function hasNoExecutableDependents() {
|
|
16708
|
+
return sql`NOT EXISTS (
|
|
16709
|
+
SELECT 1 FROM tasks AS dependent_task
|
|
16710
|
+
WHERE dependent_task.depends_on = ${tasks2.id}
|
|
16711
|
+
AND dependent_task.status IN ('pending', 'running', 'failed', 'dead_letter')
|
|
16712
|
+
)`;
|
|
16713
|
+
}
|
|
16714
|
+
function hasViableDependency() {
|
|
16715
|
+
return or(
|
|
16716
|
+
isNull(tasks2.dependsOn),
|
|
16717
|
+
sql`EXISTS (
|
|
16718
|
+
SELECT 1 FROM tasks AS dependency_task
|
|
16719
|
+
WHERE dependency_task.id = ${tasks2.dependsOn}
|
|
16720
|
+
AND dependency_task.cwd IS ${tasks2.cwd}
|
|
16721
|
+
AND (
|
|
16722
|
+
dependency_task.status IN ('pending', 'running', 'done')
|
|
16723
|
+
OR (
|
|
16724
|
+
dependency_task.status = 'failed'
|
|
16725
|
+
AND dependency_task.retry_count <= dependency_task.max_retries
|
|
16726
|
+
)
|
|
16727
|
+
)
|
|
16728
|
+
)`
|
|
16729
|
+
);
|
|
16730
|
+
}
|
|
16731
|
+
function blockedDependentsOf(prerequisiteId) {
|
|
16732
|
+
return sql`${tasks2.id} IN (
|
|
16733
|
+
WITH RECURSIVE blocked_task(id) AS (
|
|
16734
|
+
SELECT direct_dependent.id
|
|
16735
|
+
FROM tasks AS direct_dependent
|
|
16736
|
+
WHERE direct_dependent.depends_on = ${prerequisiteId}
|
|
16737
|
+
AND direct_dependent.status IN ('pending', 'failed')
|
|
16738
|
+
AND EXISTS (
|
|
16739
|
+
SELECT 1 FROM tasks AS prerequisite
|
|
16740
|
+
WHERE prerequisite.id = ${prerequisiteId}
|
|
16741
|
+
AND prerequisite.status IN ('dead_letter', 'cancelled')
|
|
16742
|
+
)
|
|
16743
|
+
UNION
|
|
16744
|
+
SELECT descendant.id
|
|
16745
|
+
FROM tasks AS descendant
|
|
16746
|
+
INNER JOIN blocked_task ON descendant.depends_on = blocked_task.id
|
|
16747
|
+
WHERE descendant.status IN ('pending', 'failed')
|
|
16748
|
+
)
|
|
16749
|
+
SELECT id FROM blocked_task
|
|
16750
|
+
)`;
|
|
16751
|
+
}
|
|
16686
16752
|
var TaskService = class {
|
|
16687
16753
|
static buildScopeWhere(scope) {
|
|
16688
16754
|
const conditions = [];
|
|
@@ -16693,8 +16759,28 @@ var TaskService = class {
|
|
|
16693
16759
|
}
|
|
16694
16760
|
static async add(data) {
|
|
16695
16761
|
this.validateNewTask(data);
|
|
16696
|
-
|
|
16697
|
-
|
|
16762
|
+
return db.transaction((tx) => {
|
|
16763
|
+
if (data.dependsOn != null) {
|
|
16764
|
+
const dependency = tx.select({
|
|
16765
|
+
id: tasks2.id,
|
|
16766
|
+
cwd: tasks2.cwd,
|
|
16767
|
+
status: tasks2.status,
|
|
16768
|
+
retryCount: tasks2.retryCount,
|
|
16769
|
+
maxRetries: tasks2.maxRetries
|
|
16770
|
+
}).from(tasks2).where(eq(tasks2.id, data.dependsOn)).get();
|
|
16771
|
+
if (!dependency) {
|
|
16772
|
+
throw new Error(`dependsOn \u6307\u5411\u7684\u4EFB\u52A1 #${data.dependsOn} \u4E0D\u5B58\u5728`);
|
|
16773
|
+
}
|
|
16774
|
+
if ((dependency.cwd ?? null) !== (data.cwd ?? null)) {
|
|
16775
|
+
throw new Error("dependsOn \u5FC5\u987B\u6307\u5411\u540C\u4E00 cwd \u7684\u4EFB\u52A1");
|
|
16776
|
+
}
|
|
16777
|
+
const dependencyIsRecoverable = dependency.status === "pending" || dependency.status === "running" || dependency.status === "done" || dependency.status === "failed" && (dependency.retryCount ?? 0) <= (dependency.maxRetries ?? 3);
|
|
16778
|
+
if (!dependencyIsRecoverable) {
|
|
16779
|
+
throw new Error(`dependsOn \u6307\u5411\u7684\u4EFB\u52A1 #${data.dependsOn} \u5DF2\u8FDB\u5165\u4E0D\u53EF\u6062\u590D\u7EC8\u6001`);
|
|
16780
|
+
}
|
|
16781
|
+
}
|
|
16782
|
+
return tx.insert(tasks2).values(data).returning().get();
|
|
16783
|
+
}, { behavior: "immediate" });
|
|
16698
16784
|
}
|
|
16699
16785
|
static validateNewTask(data) {
|
|
16700
16786
|
if (!data.name.trim()) throw new Error("name \u4E0D\u80FD\u4E3A\u7A7A");
|
|
@@ -16746,23 +16832,54 @@ var TaskService = class {
|
|
|
16746
16832
|
if (batchFilter) {
|
|
16747
16833
|
conditions.push(batchFilter);
|
|
16748
16834
|
}
|
|
16749
|
-
const
|
|
16835
|
+
const result = await db.select().from(tasks2).where(and(
|
|
16836
|
+
...conditions,
|
|
16837
|
+
or(
|
|
16838
|
+
isNull(tasks2.dependsOn),
|
|
16839
|
+
sql`EXISTS (
|
|
16840
|
+
SELECT 1 FROM tasks AS dependency_task
|
|
16841
|
+
WHERE dependency_task.id = ${tasks2.dependsOn}
|
|
16842
|
+
AND dependency_task.status = 'done'
|
|
16843
|
+
AND dependency_task.cwd IS ${tasks2.cwd}
|
|
16844
|
+
)`
|
|
16845
|
+
),
|
|
16846
|
+
or(
|
|
16847
|
+
isNull(tasks2.batchId),
|
|
16848
|
+
sql`NOT EXISTS (
|
|
16849
|
+
SELECT 1 FROM tasks AS running_batch_task
|
|
16850
|
+
WHERE running_batch_task.batch_id = ${tasks2.batchId}
|
|
16851
|
+
AND (
|
|
16852
|
+
running_batch_task.status = 'running'
|
|
16853
|
+
OR EXISTS (
|
|
16854
|
+
SELECT 1 FROM task_runs AS running_batch_run
|
|
16855
|
+
WHERE running_batch_run.task_id = running_batch_task.id
|
|
16856
|
+
AND running_batch_run.status = 'running'
|
|
16857
|
+
)
|
|
16858
|
+
)
|
|
16859
|
+
)`
|
|
16860
|
+
),
|
|
16861
|
+
sql`NOT EXISTS (
|
|
16862
|
+
SELECT 1 FROM task_runs AS candidate_active_run
|
|
16863
|
+
WHERE candidate_active_run.task_id = ${tasks2.id}
|
|
16864
|
+
AND candidate_active_run.status = 'running'
|
|
16865
|
+
)`
|
|
16866
|
+
)).orderBy(
|
|
16750
16867
|
desc(tasks2.urgency),
|
|
16751
16868
|
desc(tasks2.importance),
|
|
16752
16869
|
asc(tasks2.createdAt),
|
|
16753
16870
|
asc(tasks2.id)
|
|
16754
|
-
);
|
|
16755
|
-
|
|
16756
|
-
|
|
16757
|
-
|
|
16758
|
-
|
|
16759
|
-
|
|
16760
|
-
|
|
16761
|
-
|
|
16762
|
-
|
|
16763
|
-
|
|
16764
|
-
|
|
16765
|
-
|
|
16871
|
+
).limit(1);
|
|
16872
|
+
return result[0] ?? null;
|
|
16873
|
+
}
|
|
16874
|
+
static async countRunning() {
|
|
16875
|
+
return db.transaction((tx) => {
|
|
16876
|
+
const runningTasks = tx.select({ count: sql`count(*)` }).from(tasks2).where(eq(tasks2.status, "running")).get();
|
|
16877
|
+
const runsWithoutRunningTask = tx.select({ count: sql`count(DISTINCT ${taskRuns2.taskId})` }).from(taskRuns2).innerJoin(tasks2, eq(tasks2.id, taskRuns2.taskId)).where(and(
|
|
16878
|
+
eq(taskRuns2.status, "running"),
|
|
16879
|
+
sql`${tasks2.status} <> 'running'`
|
|
16880
|
+
)).get();
|
|
16881
|
+
return Number(runningTasks?.count ?? 0) + Number(runsWithoutRunningTask?.count ?? 0);
|
|
16882
|
+
}, { behavior: "deferred" });
|
|
16766
16883
|
}
|
|
16767
16884
|
static async start(id, scope = {}) {
|
|
16768
16885
|
const conditions = [
|
|
@@ -16798,27 +16915,202 @@ var TaskService = class {
|
|
|
16798
16915
|
return result[0] || null;
|
|
16799
16916
|
}
|
|
16800
16917
|
static async fail(id, log, scope = {}, options) {
|
|
16801
|
-
|
|
16802
|
-
|
|
16803
|
-
|
|
16804
|
-
|
|
16805
|
-
|
|
16806
|
-
|
|
16807
|
-
|
|
16808
|
-
|
|
16809
|
-
|
|
16810
|
-
|
|
16918
|
+
return db.transaction((tx) => {
|
|
16919
|
+
const current = tx.select().from(tasks2).where(and(
|
|
16920
|
+
eq(tasks2.id, id),
|
|
16921
|
+
eq(tasks2.status, "running"),
|
|
16922
|
+
...this.buildScopeWhere(scope)
|
|
16923
|
+
)).get();
|
|
16924
|
+
if (!current) return null;
|
|
16925
|
+
const newRetryCount = (current.retryCount ?? 0) + 1;
|
|
16926
|
+
const maxRetries = current.maxRetries ?? 3;
|
|
16927
|
+
const isDeadLetter = options?.setDeadLetter ?? newRetryCount > maxRetries;
|
|
16928
|
+
const failed = tx.update(tasks2).set({
|
|
16929
|
+
status: isDeadLetter ? "dead_letter" : "failed",
|
|
16930
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
16931
|
+
resultLog: log,
|
|
16932
|
+
retryCount: newRetryCount,
|
|
16933
|
+
retryAfter: isDeadLetter ? null : options?.retryAfterMs ?? Date.now() + computeBackoff(
|
|
16934
|
+
newRetryCount,
|
|
16935
|
+
current.retryBackoffMs ?? 3e4
|
|
16936
|
+
)
|
|
16937
|
+
}).where(and(eq(tasks2.id, id), eq(tasks2.status, "running"))).returning().get();
|
|
16938
|
+
if (failed?.status === "dead_letter") {
|
|
16939
|
+
tx.update(tasks2).set({
|
|
16940
|
+
status: "dead_letter",
|
|
16941
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
16942
|
+
retryAfter: null,
|
|
16943
|
+
resultLog: `\u4F9D\u8D56\u4EFB\u52A1 #${id} \u5DF2\u8FDB\u5165\u4E0D\u53EF\u6062\u590D\u7EC8\u6001`
|
|
16944
|
+
}).where(blockedDependentsOf(id)).run();
|
|
16945
|
+
}
|
|
16946
|
+
return failed ?? null;
|
|
16947
|
+
}, { behavior: "immediate" });
|
|
16948
|
+
}
|
|
16949
|
+
static async completeRun(taskId, runId, log) {
|
|
16950
|
+
return db.transaction((tx) => {
|
|
16951
|
+
const currentTask = tx.select().from(tasks2).where(and(eq(tasks2.id, taskId), eq(tasks2.status, "running"))).get();
|
|
16952
|
+
const currentRun = tx.select({ id: taskRuns2.id }).from(taskRuns2).where(and(
|
|
16953
|
+
eq(taskRuns2.id, runId),
|
|
16954
|
+
eq(taskRuns2.taskId, taskId),
|
|
16955
|
+
eq(taskRuns2.status, "running")
|
|
16956
|
+
)).get();
|
|
16957
|
+
if (!currentTask || !currentRun) return null;
|
|
16958
|
+
const finishedAt = /* @__PURE__ */ new Date();
|
|
16959
|
+
const completed = tx.update(tasks2).set({
|
|
16960
|
+
status: "done",
|
|
16961
|
+
finishedAt,
|
|
16962
|
+
resultLog: log,
|
|
16963
|
+
retryAfter: null
|
|
16964
|
+
}).where(and(eq(tasks2.id, taskId), eq(tasks2.status, "running"))).returning().get();
|
|
16965
|
+
if (!completed) return null;
|
|
16966
|
+
tx.update(taskRuns2).set({ status: "done", finishedAt, log }).where(and(eq(taskRuns2.id, runId), eq(taskRuns2.status, "running"))).run();
|
|
16967
|
+
return completed;
|
|
16968
|
+
}, { behavior: "immediate" });
|
|
16969
|
+
}
|
|
16970
|
+
static async failRun(taskId, runId, log, options) {
|
|
16971
|
+
const failed = db.transaction((tx) => {
|
|
16972
|
+
const currentTask = tx.select().from(tasks2).where(and(eq(tasks2.id, taskId), eq(tasks2.status, "running"))).get();
|
|
16973
|
+
const currentRun = tx.select({ id: taskRuns2.id }).from(taskRuns2).where(and(
|
|
16974
|
+
eq(taskRuns2.id, runId),
|
|
16975
|
+
eq(taskRuns2.taskId, taskId),
|
|
16976
|
+
eq(taskRuns2.status, "running")
|
|
16977
|
+
)).get();
|
|
16978
|
+
if (!currentTask || !currentRun) return null;
|
|
16979
|
+
const newRetryCount = (currentTask.retryCount ?? 0) + 1;
|
|
16980
|
+
const maxRetries = currentTask.maxRetries ?? 3;
|
|
16981
|
+
const isDeadLetter = options?.setDeadLetter ?? newRetryCount > maxRetries;
|
|
16982
|
+
const finishedAt = /* @__PURE__ */ new Date();
|
|
16983
|
+
const retryAfter = isDeadLetter ? null : options?.retryAfterMs ?? Date.now() + computeBackoff(
|
|
16984
|
+
newRetryCount,
|
|
16985
|
+
currentTask.retryBackoffMs ?? 3e4
|
|
16986
|
+
);
|
|
16987
|
+
const failed2 = tx.update(tasks2).set({
|
|
16988
|
+
status: isDeadLetter ? "dead_letter" : "failed",
|
|
16989
|
+
finishedAt,
|
|
16990
|
+
resultLog: log,
|
|
16991
|
+
retryCount: newRetryCount,
|
|
16992
|
+
retryAfter
|
|
16993
|
+
}).where(and(eq(tasks2.id, taskId), eq(tasks2.status, "running"))).returning().get();
|
|
16994
|
+
if (!failed2) return null;
|
|
16995
|
+
tx.update(taskRuns2).set({ status: "failed", finishedAt, log }).where(and(eq(taskRuns2.id, runId), eq(taskRuns2.status, "running"))).run();
|
|
16996
|
+
if (failed2.status === "dead_letter") {
|
|
16997
|
+
tx.update(tasks2).set({
|
|
16998
|
+
status: "dead_letter",
|
|
16999
|
+
finishedAt,
|
|
17000
|
+
retryAfter: null,
|
|
17001
|
+
resultLog: `\u4F9D\u8D56\u4EFB\u52A1 #${taskId} \u5DF2\u8FDB\u5165\u4E0D\u53EF\u6062\u590D\u7EC8\u6001`
|
|
17002
|
+
}).where(blockedDependentsOf(taskId)).run();
|
|
17003
|
+
}
|
|
17004
|
+
return failed2;
|
|
17005
|
+
}, { behavior: "immediate" });
|
|
17006
|
+
return failed;
|
|
17007
|
+
}
|
|
17008
|
+
static async recoverRun(taskId, runId, log) {
|
|
17009
|
+
const recovery = db.transaction((tx) => {
|
|
17010
|
+
const currentTask = tx.select().from(tasks2).where(and(eq(tasks2.id, taskId), eq(tasks2.status, "running"))).get();
|
|
17011
|
+
const currentRun = tx.select({ id: taskRuns2.id }).from(taskRuns2).where(and(
|
|
17012
|
+
eq(taskRuns2.id, runId),
|
|
17013
|
+
eq(taskRuns2.taskId, taskId),
|
|
17014
|
+
eq(taskRuns2.status, "running")
|
|
17015
|
+
)).get();
|
|
17016
|
+
if (!currentRun) return null;
|
|
17017
|
+
const finishedAt = /* @__PURE__ */ new Date();
|
|
17018
|
+
tx.update(taskRuns2).set({ status: "failed", finishedAt, log }).where(and(eq(taskRuns2.id, runId), eq(taskRuns2.status, "running"))).run();
|
|
17019
|
+
if (!currentTask) return null;
|
|
17020
|
+
const retryCount = (currentTask.retryCount ?? 0) + 1;
|
|
17021
|
+
const maxRetries = currentTask.maxRetries ?? 3;
|
|
17022
|
+
const isDeadLetter = retryCount > maxRetries;
|
|
17023
|
+
const recoveryStatus = isDeadLetter ? "dead_letter" : "pending";
|
|
17024
|
+
const retryAfterMs = isDeadLetter ? null : Date.now() + computeBackoff(
|
|
17025
|
+
retryCount,
|
|
17026
|
+
currentTask.retryBackoffMs ?? 3e4
|
|
17027
|
+
);
|
|
17028
|
+
tx.update(tasks2).set({
|
|
17029
|
+
status: recoveryStatus,
|
|
17030
|
+
startedAt: null,
|
|
17031
|
+
finishedAt: isDeadLetter ? finishedAt : null,
|
|
17032
|
+
retryCount,
|
|
17033
|
+
retryAfter: retryAfterMs,
|
|
17034
|
+
resultLog: log
|
|
17035
|
+
}).where(and(eq(tasks2.id, taskId), eq(tasks2.status, "running"))).run();
|
|
17036
|
+
if (recoveryStatus === "dead_letter") {
|
|
17037
|
+
tx.update(tasks2).set({
|
|
17038
|
+
status: "dead_letter",
|
|
17039
|
+
finishedAt,
|
|
17040
|
+
retryAfter: null,
|
|
17041
|
+
resultLog: `\u4F9D\u8D56\u4EFB\u52A1 #${taskId} \u5DF2\u8FDB\u5165\u4E0D\u53EF\u6062\u590D\u7EC8\u6001`
|
|
17042
|
+
}).where(blockedDependentsOf(taskId)).run();
|
|
17043
|
+
}
|
|
17044
|
+
return {
|
|
17045
|
+
status: recoveryStatus,
|
|
17046
|
+
retryCount,
|
|
17047
|
+
retryAfterMs
|
|
17048
|
+
};
|
|
17049
|
+
}, { behavior: "immediate" });
|
|
17050
|
+
return recovery;
|
|
17051
|
+
}
|
|
17052
|
+
static async interruptRun(taskId, runId, log) {
|
|
17053
|
+
return db.transaction((tx) => {
|
|
17054
|
+
const currentRun = tx.select({ id: taskRuns2.id }).from(taskRuns2).where(and(
|
|
17055
|
+
eq(taskRuns2.id, runId),
|
|
17056
|
+
eq(taskRuns2.taskId, taskId),
|
|
17057
|
+
eq(taskRuns2.status, "running")
|
|
17058
|
+
)).get();
|
|
17059
|
+
if (!currentRun) return false;
|
|
17060
|
+
const updatedRun = tx.update(taskRuns2).set({ status: "failed", finishedAt: /* @__PURE__ */ new Date(), log }).where(and(eq(taskRuns2.id, runId), eq(taskRuns2.status, "running"))).returning({ id: taskRuns2.id }).get();
|
|
17061
|
+
tx.update(tasks2).set({
|
|
17062
|
+
status: "pending",
|
|
17063
|
+
startedAt: null,
|
|
17064
|
+
finishedAt: null,
|
|
17065
|
+
retryAfter: null,
|
|
17066
|
+
resultLog: log
|
|
17067
|
+
}).where(and(eq(tasks2.id, taskId), eq(tasks2.status, "running"))).run();
|
|
17068
|
+
return updatedRun != null;
|
|
17069
|
+
}, { behavior: "immediate" });
|
|
17070
|
+
}
|
|
17071
|
+
static async resolveBlockedDependencies() {
|
|
17072
|
+
const finishedAt = /* @__PURE__ */ new Date();
|
|
17073
|
+
const result = await db.update(tasks2).set({
|
|
17074
|
+
status: "dead_letter",
|
|
17075
|
+
finishedAt,
|
|
17076
|
+
retryAfter: null,
|
|
17077
|
+
resultLog: "\u4F9D\u8D56\u4EFB\u52A1\u4E0D\u5B58\u5728\u3001\u8DE8\u9879\u76EE\u6216\u5DF2\u8FDB\u5165\u4E0D\u53EF\u6062\u590D\u7EC8\u6001"
|
|
17078
|
+
}).where(sql`${tasks2.id} IN (
|
|
17079
|
+
WITH RECURSIVE blocked_task(id) AS (
|
|
17080
|
+
SELECT candidate.id
|
|
17081
|
+
FROM tasks AS candidate
|
|
17082
|
+
WHERE candidate.status IN ('pending', 'failed')
|
|
17083
|
+
AND candidate.depends_on IS NOT NULL
|
|
17084
|
+
AND NOT EXISTS (
|
|
17085
|
+
SELECT 1 FROM tasks AS viable_dependency
|
|
17086
|
+
WHERE viable_dependency.id = candidate.depends_on
|
|
17087
|
+
AND viable_dependency.cwd IS candidate.cwd
|
|
17088
|
+
AND (
|
|
17089
|
+
viable_dependency.status IN ('pending', 'running', 'done')
|
|
17090
|
+
OR (
|
|
17091
|
+
viable_dependency.status = 'failed'
|
|
17092
|
+
AND viable_dependency.retry_count <= viable_dependency.max_retries
|
|
17093
|
+
)
|
|
17094
|
+
)
|
|
17095
|
+
)
|
|
17096
|
+
UNION
|
|
17097
|
+
SELECT descendant.id
|
|
17098
|
+
FROM tasks AS descendant
|
|
17099
|
+
INNER JOIN blocked_task ON descendant.depends_on = blocked_task.id
|
|
17100
|
+
WHERE descendant.status IN ('pending', 'failed')
|
|
17101
|
+
)
|
|
17102
|
+
SELECT id FROM blocked_task
|
|
17103
|
+
)`).returning();
|
|
17104
|
+
return result.length;
|
|
17105
|
+
}
|
|
17106
|
+
static async rejectBlockedDependents(prerequisiteId) {
|
|
16811
17107
|
const result = await db.update(tasks2).set({
|
|
16812
|
-
status:
|
|
17108
|
+
status: "dead_letter",
|
|
16813
17109
|
finishedAt: /* @__PURE__ */ new Date(),
|
|
16814
|
-
|
|
16815
|
-
|
|
16816
|
-
|
|
16817
|
-
|
|
16818
|
-
current.retryBackoffMs ?? 3e4
|
|
16819
|
-
)
|
|
16820
|
-
}).where(and(...conditions)).returning();
|
|
16821
|
-
return result[0] || null;
|
|
17110
|
+
retryAfter: null,
|
|
17111
|
+
resultLog: `\u4F9D\u8D56\u4EFB\u52A1 #${prerequisiteId} \u5DF2\u8FDB\u5165\u4E0D\u53EF\u6062\u590D\u7EC8\u6001`
|
|
17112
|
+
}).where(blockedDependentsOf(prerequisiteId)).returning({ id: tasks2.id });
|
|
17113
|
+
return result.length;
|
|
16822
17114
|
}
|
|
16823
17115
|
static async markPendingForRetry(id, retryAfterMs, retryCount) {
|
|
16824
17116
|
const result = await db.update(tasks2).set({
|
|
@@ -16831,8 +17123,18 @@ var TaskService = class {
|
|
|
16831
17123
|
return result[0] || null;
|
|
16832
17124
|
}
|
|
16833
17125
|
static async markDeadLetter(id, retryCount) {
|
|
16834
|
-
|
|
16835
|
-
|
|
17126
|
+
return db.transaction((tx) => {
|
|
17127
|
+
const finishedAt = /* @__PURE__ */ new Date();
|
|
17128
|
+
const task = tx.update(tasks2).set({ status: "dead_letter", finishedAt, retryCount }).where(eq(tasks2.id, id)).returning().get();
|
|
17129
|
+
if (!task) return null;
|
|
17130
|
+
tx.update(tasks2).set({
|
|
17131
|
+
status: "dead_letter",
|
|
17132
|
+
finishedAt,
|
|
17133
|
+
retryAfter: null,
|
|
17134
|
+
resultLog: `\u4F9D\u8D56\u4EFB\u52A1 #${id} \u5DF2\u8FDB\u5165\u4E0D\u53EF\u6062\u590D\u7EC8\u6001`
|
|
17135
|
+
}).where(blockedDependentsOf(id)).run();
|
|
17136
|
+
return task;
|
|
17137
|
+
}, { behavior: "immediate" });
|
|
16836
17138
|
}
|
|
16837
17139
|
static async resetRunningToPending(ids) {
|
|
16838
17140
|
if (ids.length === 0) return 0;
|
|
@@ -16875,41 +17177,91 @@ var TaskService = class {
|
|
|
16875
17177
|
),
|
|
16876
17178
|
...this.buildScopeWhere(scope)
|
|
16877
17179
|
];
|
|
16878
|
-
|
|
16879
|
-
|
|
16880
|
-
|
|
16881
|
-
|
|
16882
|
-
|
|
16883
|
-
|
|
17180
|
+
return db.transaction((tx) => {
|
|
17181
|
+
const finishedAt = /* @__PURE__ */ new Date();
|
|
17182
|
+
const task = tx.update(tasks2).set({
|
|
17183
|
+
status: "cancelled",
|
|
17184
|
+
finishedAt,
|
|
17185
|
+
retryAfter: null
|
|
17186
|
+
}).where(and(...conditions)).returning().get();
|
|
17187
|
+
if (!task) return null;
|
|
17188
|
+
tx.update(tasks2).set({
|
|
17189
|
+
status: "dead_letter",
|
|
17190
|
+
finishedAt,
|
|
17191
|
+
retryAfter: null,
|
|
17192
|
+
resultLog: `\u4F9D\u8D56\u4EFB\u52A1 #${id} \u5DF2\u8FDB\u5165\u4E0D\u53EF\u6062\u590D\u7EC8\u6001`
|
|
17193
|
+
}).where(blockedDependentsOf(id)).run();
|
|
17194
|
+
return task;
|
|
17195
|
+
}, { behavior: "immediate" });
|
|
16884
17196
|
}
|
|
16885
17197
|
static async retry(id, scope = {}) {
|
|
16886
|
-
const current = await this.getById(id, scope);
|
|
16887
|
-
if (!current) return null;
|
|
16888
|
-
if (current.status !== "failed" && current.status !== "dead_letter") return null;
|
|
16889
|
-
const conditions = [eq(tasks2.id, id), ...this.buildScopeWhere(scope)];
|
|
16890
|
-
const result = await db.update(tasks2).set({
|
|
16891
|
-
status: "pending",
|
|
16892
|
-
startedAt: null,
|
|
16893
|
-
finishedAt: null,
|
|
16894
|
-
retryAfter: null,
|
|
16895
|
-
retryCount: 0
|
|
16896
|
-
}).where(and(...conditions)).returning();
|
|
16897
|
-
return result[0] || null;
|
|
16898
|
-
}
|
|
16899
|
-
static async retryBatch(batchId, scope = {}) {
|
|
16900
17198
|
const conditions = [
|
|
16901
|
-
eq(tasks2.
|
|
17199
|
+
eq(tasks2.id, id),
|
|
16902
17200
|
or(eq(tasks2.status, "failed"), eq(tasks2.status, "dead_letter")),
|
|
16903
17201
|
...this.buildScopeWhere(scope)
|
|
16904
17202
|
];
|
|
16905
|
-
|
|
17203
|
+
return db.transaction((tx) => tx.update(tasks2).set({
|
|
16906
17204
|
status: "pending",
|
|
16907
17205
|
startedAt: null,
|
|
16908
17206
|
finishedAt: null,
|
|
16909
17207
|
retryAfter: null,
|
|
16910
17208
|
retryCount: 0
|
|
16911
|
-
}).where(and(...conditions)).returning();
|
|
16912
|
-
|
|
17209
|
+
}).where(and(...conditions, hasViableDependency())).returning().get() ?? null, { behavior: "immediate" });
|
|
17210
|
+
}
|
|
17211
|
+
static async retryBatch(batchId, scope = {}) {
|
|
17212
|
+
const sqlite2 = getSqlite();
|
|
17213
|
+
const scopeFilter = scope.cwd === void 0 ? "" : "AND candidate.cwd = ?";
|
|
17214
|
+
const parameters = scope.cwd === void 0 ? [batchId] : [batchId, scope.cwd];
|
|
17215
|
+
return db.transaction(() => sqlite2.query(`
|
|
17216
|
+
WITH RECURSIVE
|
|
17217
|
+
candidate(id, cwd, depends_on) AS MATERIALIZED (
|
|
17218
|
+
SELECT candidate.id, candidate.cwd, candidate.depends_on
|
|
17219
|
+
FROM tasks AS candidate
|
|
17220
|
+
WHERE candidate.batch_id = ?
|
|
17221
|
+
AND candidate.status IN ('failed', 'dead_letter')
|
|
17222
|
+
${scopeFilter}
|
|
17223
|
+
),
|
|
17224
|
+
retryable(id, cwd, depends_on) AS (
|
|
17225
|
+
SELECT candidate.id,
|
|
17226
|
+
candidate.cwd,
|
|
17227
|
+
candidate.depends_on
|
|
17228
|
+
FROM candidate
|
|
17229
|
+
WHERE candidate.depends_on IS NULL
|
|
17230
|
+
OR (
|
|
17231
|
+
NOT EXISTS (
|
|
17232
|
+
SELECT 1 FROM candidate AS internal_parent
|
|
17233
|
+
WHERE internal_parent.id = candidate.depends_on
|
|
17234
|
+
)
|
|
17235
|
+
AND EXISTS (
|
|
17236
|
+
SELECT 1 FROM tasks AS external_parent
|
|
17237
|
+
WHERE external_parent.id = candidate.depends_on
|
|
17238
|
+
AND external_parent.cwd IS candidate.cwd
|
|
17239
|
+
AND (
|
|
17240
|
+
external_parent.status IN ('pending', 'running', 'done')
|
|
17241
|
+
OR (
|
|
17242
|
+
external_parent.status = 'failed'
|
|
17243
|
+
AND external_parent.retry_count <= external_parent.max_retries
|
|
17244
|
+
)
|
|
17245
|
+
)
|
|
17246
|
+
)
|
|
17247
|
+
)
|
|
17248
|
+
UNION
|
|
17249
|
+
SELECT child.id,
|
|
17250
|
+
child.cwd,
|
|
17251
|
+
child.depends_on
|
|
17252
|
+
FROM candidate AS child
|
|
17253
|
+
INNER JOIN retryable AS parent
|
|
17254
|
+
ON child.depends_on = parent.id
|
|
17255
|
+
AND child.cwd IS parent.cwd
|
|
17256
|
+
)
|
|
17257
|
+
UPDATE tasks
|
|
17258
|
+
SET status = 'pending',
|
|
17259
|
+
started_at = NULL,
|
|
17260
|
+
finished_at = NULL,
|
|
17261
|
+
retry_after = NULL,
|
|
17262
|
+
retry_count = 0
|
|
17263
|
+
WHERE id IN (SELECT id FROM retryable)
|
|
17264
|
+
`).run(...parameters).changes, { behavior: "immediate" });
|
|
16913
17265
|
}
|
|
16914
17266
|
static async getById(id, scope = {}) {
|
|
16915
17267
|
const conditions = [eq(tasks2.id, id), ...this.buildScopeWhere(scope)];
|
|
@@ -16974,28 +17326,151 @@ var TaskService = class {
|
|
|
16974
17326
|
return stats;
|
|
16975
17327
|
}
|
|
16976
17328
|
static async delete(id, scope = {}) {
|
|
16977
|
-
const conditions = [
|
|
16978
|
-
|
|
16979
|
-
|
|
16980
|
-
|
|
17329
|
+
const conditions = [
|
|
17330
|
+
eq(tasks2.id, id),
|
|
17331
|
+
sql`${tasks2.status} <> 'running'`,
|
|
17332
|
+
sql`NOT EXISTS (
|
|
17333
|
+
SELECT 1 FROM ${taskRuns2}
|
|
17334
|
+
WHERE ${taskRuns2.taskId} = ${tasks2.id}
|
|
17335
|
+
AND ${taskRuns2.status} = 'running'
|
|
17336
|
+
)`,
|
|
17337
|
+
hasNoExecutableDependents(),
|
|
17338
|
+
...this.buildScopeWhere(scope)
|
|
17339
|
+
];
|
|
16981
17340
|
const result = await db.delete(tasks2).where(and(...conditions)).returning();
|
|
16982
|
-
|
|
17341
|
+
if (result.length > 0) {
|
|
17342
|
+
await db.delete(taskRuns2).where(eq(taskRuns2.taskId, id));
|
|
17343
|
+
return true;
|
|
17344
|
+
}
|
|
17345
|
+
if (!await this.getById(id, scope)) return false;
|
|
17346
|
+
const dependent = await db.select({ id: tasks2.id }).from(tasks2).where(and(
|
|
17347
|
+
eq(tasks2.dependsOn, id),
|
|
17348
|
+
sql`${tasks2.status} IN ('pending', 'running', 'failed', 'dead_letter')`
|
|
17349
|
+
)).orderBy(asc(tasks2.id)).limit(1);
|
|
17350
|
+
if (dependent[0]) {
|
|
17351
|
+
throw new TaskDeletionConflictError(
|
|
17352
|
+
`\u4EFB\u52A1 #${id} \u4ECD\u88AB\u53EF\u6267\u884C\u4EFB\u52A1 #${dependent[0].id} \u4F9D\u8D56\uFF0C\u8BF7\u5148\u5904\u7406\u4F9D\u8D56\u4EFB\u52A1`
|
|
17353
|
+
);
|
|
17354
|
+
}
|
|
17355
|
+
throw new TaskDeletionConflictError(
|
|
17356
|
+
`\u4EFB\u52A1 #${id} \u6B63\u5728\u8FD0\u884C\uFF0C\u8BF7\u5148\u53D6\u6D88\u4EFB\u52A1\u5E76\u7B49\u5F85\u6267\u884C\u8FDB\u7A0B\u9000\u51FA`
|
|
17357
|
+
);
|
|
16983
17358
|
}
|
|
16984
|
-
static async deleteOlderThan(retentionDays) {
|
|
17359
|
+
static async deleteOlderThan(retentionDays, shouldStop = () => false) {
|
|
16985
17360
|
const cutoffSec = Math.floor(Date.now() / 1e3) - retentionDays * 86400;
|
|
16986
|
-
const
|
|
16987
|
-
|
|
16988
|
-
|
|
16989
|
-
|
|
16990
|
-
|
|
16991
|
-
|
|
16992
|
-
|
|
16993
|
-
|
|
17361
|
+
const batchSize = 500;
|
|
17362
|
+
const sqlite2 = getSqlite();
|
|
17363
|
+
cleanupInvocation += 1;
|
|
17364
|
+
const candidateTable = `cleanup_candidates_${process.pid}_${cleanupInvocation}`;
|
|
17365
|
+
let deletedTotal = 0;
|
|
17366
|
+
sqlite2.exec(`
|
|
17367
|
+
CREATE TEMP TABLE ${candidateTable} (
|
|
17368
|
+
id INTEGER NOT NULL PRIMARY KEY
|
|
17369
|
+
) WITHOUT ROWID;
|
|
17370
|
+
`);
|
|
17371
|
+
try {
|
|
17372
|
+
let ceilingId = null;
|
|
17373
|
+
while (true) {
|
|
17374
|
+
if (shouldStop()) return deletedTotal;
|
|
17375
|
+
const batch = db.transaction(() => {
|
|
17376
|
+
sqlite2.query(`DELETE FROM ${candidateTable}`).run();
|
|
17377
|
+
const ceilingPredicate = ceilingId == null ? "" : "AND candidate.id < ?";
|
|
17378
|
+
const rawCandidateStatement = sqlite2.query(`
|
|
17379
|
+
INSERT INTO ${candidateTable}(id)
|
|
17380
|
+
SELECT candidate.id
|
|
17381
|
+
FROM tasks AS candidate NOT INDEXED
|
|
17382
|
+
WHERE candidate.status IN ('done', 'dead_letter', 'cancelled')
|
|
17383
|
+
AND candidate.finished_at IS NOT NULL
|
|
17384
|
+
AND candidate.finished_at < ?
|
|
17385
|
+
${ceilingPredicate}
|
|
17386
|
+
AND NOT EXISTS (
|
|
17387
|
+
SELECT 1 FROM task_runs AS active_run
|
|
17388
|
+
WHERE active_run.task_id = candidate.id
|
|
17389
|
+
AND active_run.status = 'running'
|
|
17390
|
+
)
|
|
17391
|
+
ORDER BY candidate.id DESC
|
|
17392
|
+
LIMIT ?
|
|
17393
|
+
`);
|
|
17394
|
+
const rawCount = ceilingId == null ? rawCandidateStatement.run(cutoffSec, batchSize).changes : rawCandidateStatement.run(cutoffSec, ceilingId, batchSize).changes;
|
|
17395
|
+
if (rawCount === 0) return { deleted: 0, nextCeilingId: null };
|
|
17396
|
+
const rawPage = sqlite2.query(`
|
|
17397
|
+
SELECT min(id) AS nextCeilingId FROM ${candidateTable}
|
|
17398
|
+
`).get();
|
|
17399
|
+
sqlite2.query(`
|
|
17400
|
+
DELETE FROM ${candidateTable}
|
|
17401
|
+
WHERE EXISTS (
|
|
17402
|
+
SELECT 1 FROM tasks AS anomalous
|
|
17403
|
+
WHERE anomalous.id = ${candidateTable}.id
|
|
17404
|
+
AND anomalous.depends_on IS NOT NULL
|
|
17405
|
+
AND anomalous.depends_on >= anomalous.id
|
|
17406
|
+
)
|
|
17407
|
+
`).run();
|
|
17408
|
+
while (true) {
|
|
17409
|
+
const pruned = sqlite2.query(`
|
|
17410
|
+
DELETE FROM ${candidateTable}
|
|
17411
|
+
WHERE EXISTS (
|
|
17412
|
+
SELECT 1 FROM tasks AS dependent_task
|
|
17413
|
+
WHERE dependent_task.depends_on = ${candidateTable}.id
|
|
17414
|
+
AND NOT EXISTS (
|
|
17415
|
+
SELECT 1 FROM ${candidateTable} AS selected_dependent
|
|
17416
|
+
WHERE selected_dependent.id = dependent_task.id
|
|
17417
|
+
)
|
|
17418
|
+
)
|
|
17419
|
+
`).run().changes;
|
|
17420
|
+
if (pruned === 0) break;
|
|
17421
|
+
}
|
|
17422
|
+
const selected = sqlite2.query(`
|
|
17423
|
+
SELECT count(*) AS count FROM ${candidateTable}
|
|
17424
|
+
`).get();
|
|
17425
|
+
if (selected.count === 0) {
|
|
17426
|
+
return { deleted: 0, nextCeilingId: rawPage.nextCeilingId };
|
|
17427
|
+
}
|
|
17428
|
+
sqlite2.query(`
|
|
17429
|
+
DELETE FROM tasks
|
|
17430
|
+
WHERE id IN (SELECT id FROM ${candidateTable})
|
|
17431
|
+
`).run();
|
|
17432
|
+
const remaining = sqlite2.query(`
|
|
17433
|
+
SELECT count(*) AS count
|
|
17434
|
+
FROM tasks
|
|
17435
|
+
WHERE id IN (SELECT id FROM ${candidateTable})
|
|
17436
|
+
`).get();
|
|
17437
|
+
if (remaining.count !== 0) {
|
|
17438
|
+
throw new Error("\u5386\u53F2\u6E05\u7406\u5019\u9009\u5728\u540C\u4E00\u5199\u4E8B\u52A1\u5185\u53D1\u751F\u6F02\u79FB\uFF0C\u5DF2\u56DE\u6EDA\u672C\u6279\u5220\u9664");
|
|
17439
|
+
}
|
|
17440
|
+
return { deleted: selected.count, nextCeilingId: rawPage.nextCeilingId };
|
|
17441
|
+
}, { behavior: "immediate" });
|
|
17442
|
+
if (batch.nextCeilingId == null) break;
|
|
17443
|
+
ceilingId = batch.nextCeilingId;
|
|
17444
|
+
deletedTotal += batch.deleted;
|
|
17445
|
+
await Bun.sleep(0);
|
|
17446
|
+
if (shouldStop()) return deletedTotal;
|
|
17447
|
+
}
|
|
17448
|
+
return deletedTotal;
|
|
17449
|
+
} finally {
|
|
17450
|
+
sqlite2.exec(`DROP TABLE IF EXISTS ${candidateTable}`);
|
|
17451
|
+
}
|
|
16994
17452
|
}
|
|
16995
17453
|
};
|
|
16996
17454
|
|
|
17455
|
+
// src/core/process-control.ts
|
|
17456
|
+
function isProcessAlive(pid) {
|
|
17457
|
+
if (!Number.isInteger(pid) || pid <= 0) return false;
|
|
17458
|
+
try {
|
|
17459
|
+
process.kill(pid, 0);
|
|
17460
|
+
return true;
|
|
17461
|
+
} catch (error) {
|
|
17462
|
+
return error instanceof Error && "code" in error && error.code === "EPERM";
|
|
17463
|
+
}
|
|
17464
|
+
}
|
|
17465
|
+
|
|
16997
17466
|
// src/core/services/task-run.service.ts
|
|
16998
|
-
var { taskRuns: taskRuns3 } = schema_exports;
|
|
17467
|
+
var { tasks: tasks3, taskRuns: taskRuns3 } = schema_exports;
|
|
17468
|
+
var LegacyRunAbandonConflictError = class extends Error {
|
|
17469
|
+
constructor(message) {
|
|
17470
|
+
super(message);
|
|
17471
|
+
this.name = "LegacyRunAbandonConflictError";
|
|
17472
|
+
}
|
|
17473
|
+
};
|
|
16999
17474
|
var TaskRunService = class {
|
|
17000
17475
|
static async create(data) {
|
|
17001
17476
|
const result = await db.insert(taskRuns3).values(data).returning();
|
|
@@ -17025,12 +17500,12 @@ var TaskRunService = class {
|
|
|
17025
17500
|
const result = await db.update(taskRuns3).set({ heartbeatAt: Date.now() }).where(and(eq(taskRuns3.id, id), eq(taskRuns3.status, "running"))).returning();
|
|
17026
17501
|
return result[0] || null;
|
|
17027
17502
|
}
|
|
17028
|
-
static async updatePid(id, workerPid, childPid) {
|
|
17503
|
+
static async updatePid(id, workerPid, childPid, lockedBy = `gateway-${process.pid}`) {
|
|
17029
17504
|
const result = await db.update(taskRuns3).set({
|
|
17030
17505
|
workerPid,
|
|
17031
17506
|
childPid,
|
|
17032
17507
|
lockedAt: Date.now(),
|
|
17033
|
-
lockedBy
|
|
17508
|
+
lockedBy
|
|
17034
17509
|
}).where(and(eq(taskRuns3.id, id), eq(taskRuns3.status, "running"))).returning();
|
|
17035
17510
|
return result[0] || null;
|
|
17036
17511
|
}
|
|
@@ -17058,39 +17533,108 @@ var TaskRunService = class {
|
|
|
17058
17533
|
}
|
|
17059
17534
|
static async getStaleRuns(heartbeatTimeoutMs) {
|
|
17060
17535
|
const cutoffMs = Date.now() - heartbeatTimeoutMs;
|
|
17061
|
-
const cutoffSec = Math.floor(cutoffMs / 1e3);
|
|
17062
17536
|
const { tasks: tasksTable } = schema_exports;
|
|
17063
17537
|
const result = await db.select({
|
|
17064
17538
|
runId: taskRuns3.id,
|
|
17065
17539
|
taskId: taskRuns3.taskId,
|
|
17066
17540
|
childPid: taskRuns3.childPid,
|
|
17541
|
+
workerPid: taskRuns3.workerPid,
|
|
17542
|
+
launchProtocol: taskRuns3.launchProtocol,
|
|
17543
|
+
lockedBy: taskRuns3.lockedBy,
|
|
17544
|
+
startedAt: taskRuns3.startedAt,
|
|
17545
|
+
heartbeatAt: taskRuns3.heartbeatAt,
|
|
17067
17546
|
taskRetryCount: tasksTable.retryCount,
|
|
17068
17547
|
taskMaxRetries: tasksTable.maxRetries,
|
|
17069
|
-
taskRetryBackoffMs: tasksTable.retryBackoffMs
|
|
17070
|
-
|
|
17071
|
-
|
|
17072
|
-
|
|
17073
|
-
|
|
17074
|
-
|
|
17075
|
-
|
|
17076
|
-
|
|
17077
|
-
|
|
17078
|
-
and(
|
|
17079
|
-
sql`${taskRuns3.heartbeatAt} IS NOT NULL`,
|
|
17080
|
-
sql`${taskRuns3.heartbeatAt} < ${cutoffMs}`
|
|
17081
|
-
)
|
|
17082
|
-
)
|
|
17083
|
-
)
|
|
17084
|
-
);
|
|
17085
|
-
return result.map((row) => ({
|
|
17548
|
+
taskRetryBackoffMs: tasksTable.retryBackoffMs,
|
|
17549
|
+
taskStatus: tasksTable.status,
|
|
17550
|
+
taskCwd: tasksTable.cwd
|
|
17551
|
+
}).from(taskRuns3).innerJoin(tasksTable, eq(taskRuns3.taskId, tasksTable.id)).where(eq(taskRuns3.status, "running"));
|
|
17552
|
+
return result.filter((row) => {
|
|
17553
|
+
const heartbeatExpired = row.heartbeatAt == null ? row.startedAt == null || row.startedAt.getTime() < cutoffMs : row.heartbeatAt < cutoffMs;
|
|
17554
|
+
const ownerExited = row.workerPid != null && row.workerPid > 0 && !isProcessAlive(row.workerPid);
|
|
17555
|
+
return heartbeatExpired || ownerExited;
|
|
17556
|
+
}).map((row) => ({
|
|
17086
17557
|
runId: row.runId,
|
|
17087
17558
|
taskId: row.taskId,
|
|
17088
17559
|
childPid: row.childPid,
|
|
17560
|
+
workerPid: row.workerPid,
|
|
17561
|
+
launchProtocol: row.launchProtocol,
|
|
17562
|
+
lockedBy: row.lockedBy,
|
|
17089
17563
|
taskRetryCount: row.taskRetryCount ?? 0,
|
|
17090
17564
|
taskMaxRetries: row.taskMaxRetries ?? 3,
|
|
17091
|
-
taskRetryBackoffMs: row.taskRetryBackoffMs ?? 3e4
|
|
17565
|
+
taskRetryBackoffMs: row.taskRetryBackoffMs ?? 3e4,
|
|
17566
|
+
taskStatus: row.taskStatus,
|
|
17567
|
+
taskCwd: row.taskCwd,
|
|
17568
|
+
ownerAlive: row.workerPid != null && row.workerPid > 0 && isProcessAlive(row.workerPid)
|
|
17569
|
+
}));
|
|
17570
|
+
}
|
|
17571
|
+
static async listLegacyQuarantinedRuns(heartbeatTimeoutMs = 0) {
|
|
17572
|
+
const staleRuns = await this.getStaleRuns(heartbeatTimeoutMs);
|
|
17573
|
+
return staleRuns.filter(
|
|
17574
|
+
(row) => row.launchProtocol == null && row.childPid == null
|
|
17575
|
+
).map((row) => ({
|
|
17576
|
+
runId: row.runId,
|
|
17577
|
+
taskId: row.taskId,
|
|
17578
|
+
taskStatus: row.taskStatus,
|
|
17579
|
+
taskCwd: row.taskCwd,
|
|
17580
|
+
workerPid: row.workerPid,
|
|
17581
|
+
ownerAlive: row.ownerAlive
|
|
17092
17582
|
}));
|
|
17093
17583
|
}
|
|
17584
|
+
static async abandonLegacyRun(runId) {
|
|
17585
|
+
return db.transaction((tx) => {
|
|
17586
|
+
const current = tx.select({
|
|
17587
|
+
runId: taskRuns3.id,
|
|
17588
|
+
taskId: taskRuns3.taskId,
|
|
17589
|
+
runStatus: taskRuns3.status,
|
|
17590
|
+
taskStatus: tasks3.status,
|
|
17591
|
+
workerPid: taskRuns3.workerPid,
|
|
17592
|
+
childPid: taskRuns3.childPid,
|
|
17593
|
+
launchProtocol: taskRuns3.launchProtocol,
|
|
17594
|
+
log: taskRuns3.log
|
|
17595
|
+
}).from(taskRuns3).innerJoin(tasks3, eq(taskRuns3.taskId, tasks3.id)).where(eq(taskRuns3.id, runId)).get();
|
|
17596
|
+
if (!current) return null;
|
|
17597
|
+
if (current.runStatus !== "running") {
|
|
17598
|
+
throw new LegacyRunAbandonConflictError(`run #${runId} \u5DF2\u4E0D\u662F running \u72B6\u6001`);
|
|
17599
|
+
}
|
|
17600
|
+
if (current.launchProtocol != null) {
|
|
17601
|
+
throw new LegacyRunAbandonConflictError(
|
|
17602
|
+
`run #${runId} \u4F7F\u7528\u672A\u77E5\u6216\u53D7\u7BA1\u534F\u8BAE ${current.launchProtocol}\uFF0C\u7981\u6B62\u4EBA\u5DE5 abandon`
|
|
17603
|
+
);
|
|
17604
|
+
}
|
|
17605
|
+
if (current.childPid != null) {
|
|
17606
|
+
throw new LegacyRunAbandonConflictError(
|
|
17607
|
+
`run #${runId} \u5DF2\u8BB0\u5F55 child PID ${current.childPid}\uFF0C\u5FC5\u987B\u7531 Worker/Watchdog \u786E\u8BA4\u8FDB\u7A0B\u6811\u9000\u51FA`
|
|
17608
|
+
);
|
|
17609
|
+
}
|
|
17610
|
+
if (current.workerPid != null && isProcessAlive(current.workerPid)) {
|
|
17611
|
+
throw new LegacyRunAbandonConflictError(
|
|
17612
|
+
`run #${runId} \u7684 owner PID ${current.workerPid} \u4ECD\u5B58\u6D3B`
|
|
17613
|
+
);
|
|
17614
|
+
}
|
|
17615
|
+
if (current.taskStatus !== "cancelled") {
|
|
17616
|
+
throw new LegacyRunAbandonConflictError(
|
|
17617
|
+
`\u4EFB\u52A1 #${current.taskId} \u5FC5\u987B\u5148\u53D6\u6D88\u5E76\u4FDD\u6301 cancelled \u72B6\u6001`
|
|
17618
|
+
);
|
|
17619
|
+
}
|
|
17620
|
+
const note = "\u64CD\u4F5C\u5458\u5DF2\u786E\u8BA4\u65E7\u7248\u65E0 PID \u6267\u884C\u4E0D\u5B58\u5728\uFF0C\u5E76\u901A\u8FC7 run abandon \u5173\u95ED\u9694\u79BB\u8BB0\u5F55";
|
|
17621
|
+
const updated = tx.update(taskRuns3).set({
|
|
17622
|
+
status: "failed",
|
|
17623
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
17624
|
+
log: current.log ? `${current.log}
|
|
17625
|
+
${note}` : note
|
|
17626
|
+
}).where(and(eq(taskRuns3.id, runId), eq(taskRuns3.status, "running"))).returning({ id: taskRuns3.id }).get();
|
|
17627
|
+
if (!updated) {
|
|
17628
|
+
throw new LegacyRunAbandonConflictError(`run #${runId} \u72B6\u6001\u5DF2\u88AB\u5176\u4ED6\u64CD\u4F5C\u6539\u53D8`);
|
|
17629
|
+
}
|
|
17630
|
+
return {
|
|
17631
|
+
runId,
|
|
17632
|
+
taskId: current.taskId,
|
|
17633
|
+
runStatus: "failed",
|
|
17634
|
+
taskStatus: "cancelled"
|
|
17635
|
+
};
|
|
17636
|
+
}, { behavior: "immediate" });
|
|
17637
|
+
}
|
|
17094
17638
|
static async getRunningRunByTaskId(taskId) {
|
|
17095
17639
|
const result = await db.select().from(taskRuns3).where(and(eq(taskRuns3.taskId, taskId), eq(taskRuns3.status, "running"))).orderBy(desc(taskRuns3.startedAt), desc(taskRuns3.id)).limit(1);
|
|
17096
17640
|
return result[0] || null;
|
|
@@ -17132,19 +17676,17 @@ var TaskTemplateService = class {
|
|
|
17132
17676
|
static async create(data) {
|
|
17133
17677
|
this.validate(data);
|
|
17134
17678
|
const now = Date.now();
|
|
17135
|
-
const
|
|
17136
|
-
|
|
17137
|
-
|
|
17138
|
-
|
|
17139
|
-
|
|
17140
|
-
|
|
17141
|
-
|
|
17142
|
-
|
|
17143
|
-
|
|
17144
|
-
|
|
17145
|
-
|
|
17146
|
-
}
|
|
17147
|
-
return tmpl;
|
|
17679
|
+
const nextRunAt = data.nextRunAt ?? this.calculateNextRunAt(
|
|
17680
|
+
data.scheduleType,
|
|
17681
|
+
{
|
|
17682
|
+
cronExpr: data.cronExpr ?? null,
|
|
17683
|
+
intervalMs: data.intervalMs ?? null,
|
|
17684
|
+
runAt: data.runAt ?? null
|
|
17685
|
+
},
|
|
17686
|
+
now
|
|
17687
|
+
);
|
|
17688
|
+
const result = await db.insert(taskTemplates2).values({ ...data, nextRunAt, createdAt: now, updatedAt: now }).returning();
|
|
17689
|
+
return result[0];
|
|
17148
17690
|
}
|
|
17149
17691
|
static validate(data) {
|
|
17150
17692
|
if (!data.name.trim()) throw new Error("name \u4E0D\u80FD\u4E3A\u7A7A");
|
|
@@ -17184,8 +17726,18 @@ var TaskTemplateService = class {
|
|
|
17184
17726
|
return result[0] || null;
|
|
17185
17727
|
}
|
|
17186
17728
|
static async enable(id) {
|
|
17187
|
-
|
|
17188
|
-
|
|
17729
|
+
return db.transaction((tx) => {
|
|
17730
|
+
const template = tx.select().from(taskTemplates2).where(eq(taskTemplates2.id, id)).limit(1).get();
|
|
17731
|
+
if (!template) return null;
|
|
17732
|
+
const nextRunAt = template.nextRunAt ?? this.calculateNextRunAt(
|
|
17733
|
+
template.scheduleType,
|
|
17734
|
+
template
|
|
17735
|
+
);
|
|
17736
|
+
if (nextRunAt == null) {
|
|
17737
|
+
throw new Error(`\u6A21\u677F #${id} \u65E0\u6CD5\u8BA1\u7B97\u4E0B\u4E00\u6B21\u6267\u884C\u65F6\u95F4\uFF0C\u5DF2\u4FDD\u6301\u7981\u7528`);
|
|
17738
|
+
}
|
|
17739
|
+
return tx.update(taskTemplates2).set({ enabled: true, nextRunAt, updatedAt: Date.now() }).where(eq(taskTemplates2.id, id)).returning().get() ?? null;
|
|
17740
|
+
}, { behavior: "immediate" });
|
|
17189
17741
|
}
|
|
17190
17742
|
static async disable(id) {
|
|
17191
17743
|
const result = await db.update(taskTemplates2).set({ enabled: false, updatedAt: Date.now() }).where(eq(taskTemplates2.id, id)).returning();
|
|
@@ -17195,6 +17747,47 @@ var TaskTemplateService = class {
|
|
|
17195
17747
|
const result = await db.delete(taskTemplates2).where(eq(taskTemplates2.id, id)).returning();
|
|
17196
17748
|
return result.length > 0;
|
|
17197
17749
|
}
|
|
17750
|
+
static async deleteExpiredDelayed(retentionDays, shouldStop = () => false) {
|
|
17751
|
+
const cutoffMs = Date.now() - retentionDays * 864e5;
|
|
17752
|
+
const batchSize = 500;
|
|
17753
|
+
let deletedTotal = 0;
|
|
17754
|
+
while (!shouldStop()) {
|
|
17755
|
+
const deleted = db.transaction((tx) => tx.delete(taskTemplates2).where(and(
|
|
17756
|
+
sql`${taskTemplates2.id} IN (
|
|
17757
|
+
SELECT candidate.id
|
|
17758
|
+
FROM task_templates AS candidate
|
|
17759
|
+
WHERE candidate.schedule_type = 'delayed'
|
|
17760
|
+
AND candidate.enabled = 0
|
|
17761
|
+
AND candidate.last_run_at IS NOT NULL
|
|
17762
|
+
AND candidate.last_run_at < ${cutoffMs}
|
|
17763
|
+
AND NOT EXISTS (
|
|
17764
|
+
SELECT 1 FROM tasks AS active_task
|
|
17765
|
+
WHERE active_task.template_id = candidate.id
|
|
17766
|
+
AND (
|
|
17767
|
+
active_task.status IN ('pending', 'running')
|
|
17768
|
+
OR (
|
|
17769
|
+
active_task.status = 'failed'
|
|
17770
|
+
AND active_task.retry_count <= active_task.max_retries
|
|
17771
|
+
)
|
|
17772
|
+
)
|
|
17773
|
+
)
|
|
17774
|
+
AND NOT EXISTS (
|
|
17775
|
+
SELECT 1
|
|
17776
|
+
FROM task_runs AS active_run
|
|
17777
|
+
INNER JOIN tasks AS run_task ON run_task.id = active_run.task_id
|
|
17778
|
+
WHERE run_task.template_id = candidate.id
|
|
17779
|
+
AND active_run.status = 'running'
|
|
17780
|
+
)
|
|
17781
|
+
ORDER BY candidate.last_run_at, candidate.id
|
|
17782
|
+
LIMIT ${batchSize}
|
|
17783
|
+
)`
|
|
17784
|
+
)).returning({ id: taskTemplates2.id }).all().length, { behavior: "immediate" });
|
|
17785
|
+
if (deleted === 0) break;
|
|
17786
|
+
deletedTotal += deleted;
|
|
17787
|
+
await Bun.sleep(0);
|
|
17788
|
+
}
|
|
17789
|
+
return deletedTotal;
|
|
17790
|
+
}
|
|
17198
17791
|
static calculateNextRunAt(scheduleType, template, afterMs) {
|
|
17199
17792
|
const base = afterMs ?? Date.now();
|
|
17200
17793
|
switch (scheduleType) {
|
|
@@ -17220,8 +17813,6 @@ import { Database as Database3 } from "bun:sqlite";
|
|
|
17220
17813
|
import { randomUUID } from "crypto";
|
|
17221
17814
|
import {
|
|
17222
17815
|
chmodSync,
|
|
17223
|
-
constants,
|
|
17224
|
-
copyFileSync,
|
|
17225
17816
|
existsSync as existsSync2,
|
|
17226
17817
|
mkdirSync as mkdirSync2,
|
|
17227
17818
|
renameSync,
|
|
@@ -17231,19 +17822,6 @@ import {
|
|
|
17231
17822
|
} from "fs";
|
|
17232
17823
|
import { tmpdir } from "os";
|
|
17233
17824
|
import { basename, dirname as dirname2, resolve } from "path";
|
|
17234
|
-
|
|
17235
|
-
// src/core/process-control.ts
|
|
17236
|
-
function isProcessAlive(pid) {
|
|
17237
|
-
if (!Number.isInteger(pid) || pid <= 0) return false;
|
|
17238
|
-
try {
|
|
17239
|
-
process.kill(pid, 0);
|
|
17240
|
-
return true;
|
|
17241
|
-
} catch (error) {
|
|
17242
|
-
return error instanceof Error && "code" in error && error.code === "EPERM";
|
|
17243
|
-
}
|
|
17244
|
-
}
|
|
17245
|
-
|
|
17246
|
-
// src/core/services/database-maintenance.service.ts
|
|
17247
17825
|
var REQUIRED_TABLES = ["gateway_lock", "tasks", "task_runs", "task_templates"];
|
|
17248
17826
|
var DatabaseMaintenanceConflictError = class extends Error {
|
|
17249
17827
|
constructor(message) {
|
|
@@ -17268,6 +17846,12 @@ function safeUnlink(path) {
|
|
|
17268
17846
|
function errorMessage(error) {
|
|
17269
17847
|
return error instanceof Error ? error.message : String(error);
|
|
17270
17848
|
}
|
|
17849
|
+
function quoteIdentifier(value) {
|
|
17850
|
+
return `"${value.replaceAll('"', '""')}"`;
|
|
17851
|
+
}
|
|
17852
|
+
function columnDefinitionsMatch(left, right) {
|
|
17853
|
+
return left.type.trim().toUpperCase() === right.type.trim().toUpperCase() && left.notNull === right.notNull && left.defaultValue === right.defaultValue && left.primaryKeyOrder === right.primaryKeyOrder;
|
|
17854
|
+
}
|
|
17271
17855
|
var DatabaseMaintenanceService = class {
|
|
17272
17856
|
static check() {
|
|
17273
17857
|
return this.inspect(getSqlite(), DB_FILE_PATH);
|
|
@@ -17291,9 +17875,11 @@ var DatabaseMaintenanceService = class {
|
|
|
17291
17875
|
this.assertNoRunningWork(sqlite2);
|
|
17292
17876
|
const before = this.readCounts(sqlite2);
|
|
17293
17877
|
backup = this.writeSnapshot(sqlite2, this.createBackupPath("pre-clear"));
|
|
17294
|
-
|
|
17295
|
-
sqlite2.exec("
|
|
17296
|
-
|
|
17878
|
+
const businessTables = this.readRestoreTables(sqlite2, "main");
|
|
17879
|
+
sqlite2.exec("PRAGMA defer_foreign_keys = ON");
|
|
17880
|
+
for (const table of businessTables.values()) {
|
|
17881
|
+
sqlite2.exec(`DELETE FROM main.${quoteIdentifier(table.name)}`);
|
|
17882
|
+
}
|
|
17297
17883
|
sqlite2.exec("COMMIT");
|
|
17298
17884
|
return {
|
|
17299
17885
|
backupPath: backup.path,
|
|
@@ -17315,78 +17901,102 @@ var DatabaseMaintenanceService = class {
|
|
|
17315
17901
|
const livePath = normalizedPath(DB_FILE_PATH);
|
|
17316
17902
|
if (source === livePath) throw new Error("\u6062\u590D\u6765\u6E90\u4E0D\u80FD\u662F\u5F53\u524D\u6570\u636E\u5E93\u6587\u4EF6");
|
|
17317
17903
|
const current = getSqlite();
|
|
17318
|
-
this.assertGatewaySafe(current, false);
|
|
17319
|
-
this.assertNoRunningWork(current);
|
|
17320
|
-
if (!existsSync2(source) || !statSync(source).isFile()) {
|
|
17321
|
-
throw new Error(`\u5907\u4EFD\u6587\u4EF6\u4E0D\u5B58\u5728\uFF1A${source}`);
|
|
17322
|
-
}
|
|
17323
|
-
let sourceCheck;
|
|
17324
|
-
try {
|
|
17325
|
-
sourceCheck = this.inspectFile(source);
|
|
17326
|
-
} catch (error) {
|
|
17327
|
-
throw new Error(`\u5907\u4EFD\u6587\u4EF6\u65E0\u6548\uFF1A${errorMessage(error)}`);
|
|
17328
|
-
}
|
|
17329
|
-
if (!sourceCheck.ok) {
|
|
17330
|
-
throw new Error(`\u5907\u4EFD\u6587\u4EF6\u6821\u9A8C\u5931\u8D25\uFF1A${sourceCheck.integrityMessages.join("; ") || "schema/foreign key error"}`);
|
|
17331
|
-
}
|
|
17332
17904
|
const stagePath = `${livePath}.restore-${process.pid}-${randomUUID()}.tmp`;
|
|
17333
|
-
const rollbackPath = `${livePath}.rollback-${process.pid}-${randomUUID()}.tmp`;
|
|
17334
17905
|
let safetyBackup = null;
|
|
17335
|
-
let
|
|
17906
|
+
let attached = false;
|
|
17907
|
+
let committed = false;
|
|
17336
17908
|
try {
|
|
17337
|
-
|
|
17909
|
+
current.exec("BEGIN EXCLUSIVE");
|
|
17910
|
+
this.assertGatewaySafe(current, false);
|
|
17911
|
+
this.assertNoRunningWork(current);
|
|
17912
|
+
if (!existsSync2(source)) {
|
|
17913
|
+
throw new Error(`\u5907\u4EFD\u6587\u4EF6\u4E0D\u5B58\u5728\uFF1A${source}`);
|
|
17914
|
+
}
|
|
17915
|
+
const sourceStat = statSync(source);
|
|
17916
|
+
if (!sourceStat.isFile()) throw new Error(`\u5907\u4EFD\u8DEF\u5F84\u4E0D\u662F\u6587\u4EF6\uFF1A${source}`);
|
|
17917
|
+
const liveStat = statSync(livePath);
|
|
17918
|
+
if (sourceStat.dev === liveStat.dev && sourceStat.ino === liveStat.ino) {
|
|
17919
|
+
throw new Error("\u6062\u590D\u6765\u6E90\u4E0D\u80FD\u662F\u5F53\u524D\u6570\u636E\u5E93\u7684\u7B26\u53F7\u94FE\u63A5\u6216\u786C\u94FE\u63A5\u522B\u540D");
|
|
17920
|
+
}
|
|
17921
|
+
let sourceCheck;
|
|
17922
|
+
try {
|
|
17923
|
+
const sourceDatabase = new Database3(source, { readonly: true, strict: true });
|
|
17924
|
+
try {
|
|
17925
|
+
sourceDatabase.exec("BEGIN");
|
|
17926
|
+
sourceCheck = this.inspect(sourceDatabase, source);
|
|
17927
|
+
if (!sourceCheck.ok) {
|
|
17928
|
+
throw new Error(
|
|
17929
|
+
sourceCheck.integrityMessages.join("; ") || "schema/foreign key error"
|
|
17930
|
+
);
|
|
17931
|
+
}
|
|
17932
|
+
writeFileSync(stagePath, sourceDatabase.serialize(), {
|
|
17933
|
+
flag: "wx",
|
|
17934
|
+
mode: 384
|
|
17935
|
+
});
|
|
17936
|
+
} finally {
|
|
17937
|
+
if (sourceDatabase.inTransaction) sourceDatabase.exec("ROLLBACK");
|
|
17938
|
+
sourceDatabase.close();
|
|
17939
|
+
}
|
|
17940
|
+
} catch (error) {
|
|
17941
|
+
throw new Error(`\u5907\u4EFD\u6587\u4EF6\u65E0\u6548\uFF1A${errorMessage(error)}`);
|
|
17942
|
+
}
|
|
17338
17943
|
chmodSync(stagePath, 384);
|
|
17339
17944
|
const staged = new Database3(stagePath);
|
|
17340
17945
|
try {
|
|
17946
|
+
staged.exec("PRAGMA journal_mode = DELETE;");
|
|
17947
|
+
staged.exec("PRAGMA busy_timeout = 5000;");
|
|
17948
|
+
migrateSqliteDatabase(staged);
|
|
17341
17949
|
const stagedCheck = this.inspect(staged, stagePath);
|
|
17342
17950
|
if (!stagedCheck.ok) throw new Error("\u6682\u5B58\u6062\u590D\u6587\u4EF6\u6821\u9A8C\u5931\u8D25");
|
|
17343
|
-
const now = Date.now();
|
|
17344
|
-
staged.query(
|
|
17345
|
-
"INSERT OR REPLACE INTO gateway_lock (id, pid, acquired_at, heartbeat_at, ready_at) VALUES (1, ?, ?, ?, NULL)"
|
|
17346
|
-
).run(process.pid, now, now);
|
|
17347
17951
|
} finally {
|
|
17348
17952
|
staged.close();
|
|
17349
17953
|
}
|
|
17350
|
-
const sqlite2 =
|
|
17351
|
-
|
|
17954
|
+
const sqlite2 = current;
|
|
17955
|
+
let recoveredRunningTasks = 0;
|
|
17956
|
+
let closedRunningRuns = 0;
|
|
17352
17957
|
try {
|
|
17353
17958
|
this.assertGatewaySafe(sqlite2, false);
|
|
17354
17959
|
this.assertNoRunningWork(sqlite2);
|
|
17355
17960
|
safetyBackup = this.writeSnapshot(sqlite2, this.createBackupPath("pre-restore"));
|
|
17356
|
-
|
|
17357
|
-
|
|
17358
|
-
|
|
17359
|
-
).run(process.pid, now, now);
|
|
17360
|
-
sqlite2.exec("COMMIT");
|
|
17361
|
-
} catch (error) {
|
|
17362
|
-
if (sqlite2.inTransaction) sqlite2.exec("ROLLBACK");
|
|
17363
|
-
throw error;
|
|
17364
|
-
}
|
|
17365
|
-
const checkpoint = sqlite2.query("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
17366
|
-
if (checkpoint && checkpoint.busy !== 0) {
|
|
17367
|
-
throw new DatabaseMaintenanceConflictError("\u6570\u636E\u5E93\u4ECD\u88AB\u5176\u4ED6\u8FDE\u63A5\u5360\u7528\uFF0C\u65E0\u6CD5\u5B89\u5168\u6062\u590D");
|
|
17368
|
-
}
|
|
17369
|
-
closeDb();
|
|
17370
|
-
safeUnlink(`${livePath}-wal`);
|
|
17371
|
-
safeUnlink(`${livePath}-shm`);
|
|
17372
|
-
renameSync(livePath, rollbackPath);
|
|
17373
|
-
liveMoved = true;
|
|
17374
|
-
renameSync(stagePath, livePath);
|
|
17375
|
-
const restored = getSqlite();
|
|
17376
|
-
restored.exec("BEGIN IMMEDIATE");
|
|
17377
|
-
let recoveredRunningTasks = 0;
|
|
17378
|
-
let closedRunningRuns = 0;
|
|
17379
|
-
try {
|
|
17961
|
+
sqlite2.query("ATTACH DATABASE ? AS restore_source").run(stagePath);
|
|
17962
|
+
attached = true;
|
|
17963
|
+
const restorePlan = this.buildRestorePlan(sqlite2);
|
|
17380
17964
|
recoveredRunningTasks = this.scalar(
|
|
17381
|
-
|
|
17382
|
-
"SELECT COUNT(*) AS count FROM tasks WHERE status = 'running'"
|
|
17965
|
+
sqlite2,
|
|
17966
|
+
"SELECT COUNT(*) AS count FROM restore_source.tasks WHERE status = 'running'"
|
|
17383
17967
|
);
|
|
17384
17968
|
closedRunningRuns = this.scalar(
|
|
17385
|
-
|
|
17386
|
-
"SELECT COUNT(*) AS count FROM task_runs WHERE status = 'running'"
|
|
17969
|
+
sqlite2,
|
|
17970
|
+
"SELECT COUNT(*) AS count FROM restore_source.task_runs WHERE status = 'running'"
|
|
17387
17971
|
);
|
|
17972
|
+
sqlite2.exec("PRAGMA defer_foreign_keys = ON");
|
|
17973
|
+
for (const table of restorePlan) {
|
|
17974
|
+
sqlite2.exec(`DELETE FROM main.${quoteIdentifier(table.name)}`);
|
|
17975
|
+
}
|
|
17976
|
+
for (const table of restorePlan) {
|
|
17977
|
+
if (!table.sourceExists) continue;
|
|
17978
|
+
const columns = table.columns.map((column) => quoteIdentifier(column.name)).join(", ");
|
|
17979
|
+
sqlite2.exec(`
|
|
17980
|
+
INSERT INTO main.${quoteIdentifier(table.name)} (${columns})
|
|
17981
|
+
SELECT ${columns} FROM restore_source.${quoteIdentifier(table.name)}
|
|
17982
|
+
`);
|
|
17983
|
+
}
|
|
17984
|
+
const liveTableNames = restorePlan.map((table) => table.name);
|
|
17985
|
+
const sourceTableNames = restorePlan.filter((table) => table.sourceExists).map((table) => table.name);
|
|
17986
|
+
const livePlaceholders = liveTableNames.map(() => "?").join(", ");
|
|
17987
|
+
sqlite2.query(
|
|
17988
|
+
`DELETE FROM main.sqlite_sequence WHERE name IN (${livePlaceholders})`
|
|
17989
|
+
).run(...liveTableNames);
|
|
17990
|
+
if (sourceTableNames.length > 0) {
|
|
17991
|
+
const sourcePlaceholders = sourceTableNames.map(() => "?").join(", ");
|
|
17992
|
+
sqlite2.query(`
|
|
17993
|
+
INSERT INTO main.sqlite_sequence(name, seq)
|
|
17994
|
+
SELECT name, seq FROM restore_source.sqlite_sequence
|
|
17995
|
+
WHERE name IN (${sourcePlaceholders})
|
|
17996
|
+
`).run(...sourceTableNames);
|
|
17997
|
+
}
|
|
17388
17998
|
const finishedAt = Math.floor(Date.now() / 1e3);
|
|
17389
|
-
|
|
17999
|
+
sqlite2.query(`
|
|
17390
18000
|
UPDATE task_runs
|
|
17391
18001
|
SET status = 'failed', finished_at = ?,
|
|
17392
18002
|
log = CASE
|
|
@@ -17396,20 +18006,34 @@ var DatabaseMaintenanceService = class {
|
|
|
17396
18006
|
END
|
|
17397
18007
|
WHERE status = 'running'
|
|
17398
18008
|
`).run(finishedAt);
|
|
17399
|
-
|
|
18009
|
+
sqlite2.exec(`
|
|
17400
18010
|
UPDATE tasks
|
|
17401
18011
|
SET status = 'pending', started_at = NULL, finished_at = NULL
|
|
17402
18012
|
WHERE status = 'running'
|
|
17403
18013
|
`);
|
|
17404
|
-
|
|
17405
|
-
|
|
18014
|
+
sqlite2.exec("DELETE FROM gateway_lock");
|
|
18015
|
+
const violations = sqlite2.query("PRAGMA foreign_key_check").all();
|
|
18016
|
+
if (violations.length > 0) {
|
|
18017
|
+
throw new Error(`\u6062\u590D\u6570\u636E\u5305\u542B ${violations.length} \u6761\u5916\u952E\u8FDD\u89C4`);
|
|
18018
|
+
}
|
|
18019
|
+
const transactionCheck = this.inspect(sqlite2, livePath);
|
|
18020
|
+
if (!transactionCheck.ok) throw new Error("\u6062\u590D\u540E\u7684\u6570\u636E\u5E93\u672A\u901A\u8FC7\u5B8C\u6574\u6027\u6821\u9A8C");
|
|
18021
|
+
sqlite2.exec("COMMIT");
|
|
18022
|
+
committed = true;
|
|
17406
18023
|
} catch (error) {
|
|
17407
|
-
if (
|
|
18024
|
+
if (sqlite2.inTransaction) sqlite2.exec("ROLLBACK");
|
|
17408
18025
|
throw error;
|
|
18026
|
+
} finally {
|
|
18027
|
+
if (attached && !sqlite2.inTransaction) {
|
|
18028
|
+
try {
|
|
18029
|
+
sqlite2.exec("DETACH DATABASE restore_source");
|
|
18030
|
+
} catch {
|
|
18031
|
+
}
|
|
18032
|
+
attached = false;
|
|
18033
|
+
}
|
|
17409
18034
|
}
|
|
17410
|
-
const check = this.inspect(
|
|
18035
|
+
const check = this.inspect(sqlite2, livePath);
|
|
17411
18036
|
if (!check.ok) throw new Error("\u6062\u590D\u540E\u7684\u6570\u636E\u5E93\u672A\u901A\u8FC7\u5B8C\u6574\u6027\u6821\u9A8C");
|
|
17412
|
-
safeUnlink(rollbackPath);
|
|
17413
18037
|
return {
|
|
17414
18038
|
sourcePath: source,
|
|
17415
18039
|
safetyBackupPath: safetyBackup.path,
|
|
@@ -17418,31 +18042,19 @@ var DatabaseMaintenanceService = class {
|
|
|
17418
18042
|
check
|
|
17419
18043
|
};
|
|
17420
18044
|
} catch (error) {
|
|
17421
|
-
|
|
17422
|
-
safeUnlink(`${livePath}-wal`);
|
|
17423
|
-
safeUnlink(`${livePath}-shm`);
|
|
18045
|
+
if (current.inTransaction) current.exec("ROLLBACK");
|
|
17424
18046
|
safeUnlink(stagePath);
|
|
17425
18047
|
safeUnlink(`${stagePath}-journal`);
|
|
17426
18048
|
safeUnlink(`${stagePath}-wal`);
|
|
17427
18049
|
safeUnlink(`${stagePath}-shm`);
|
|
17428
|
-
if (liveMoved && existsSync2(rollbackPath)) {
|
|
17429
|
-
safeUnlink(livePath);
|
|
17430
|
-
renameSync(rollbackPath, livePath);
|
|
17431
|
-
try {
|
|
17432
|
-
const original = getSqlite();
|
|
17433
|
-
original.query("DELETE FROM gateway_lock WHERE pid = ?").run(process.pid);
|
|
17434
|
-
} catch {
|
|
17435
|
-
}
|
|
17436
|
-
} else if (existsSync2(rollbackPath)) {
|
|
17437
|
-
safeUnlink(rollbackPath);
|
|
17438
|
-
} else {
|
|
17439
|
-
try {
|
|
17440
|
-
getSqlite().query("DELETE FROM gateway_lock WHERE pid = ?").run(process.pid);
|
|
17441
|
-
} catch {
|
|
17442
|
-
}
|
|
17443
|
-
}
|
|
17444
18050
|
const backupHint = safetyBackup ? `\uFF1B\u5F53\u524D\u5E93\u5B89\u5168\u5907\u4EFD\uFF1A${safetyBackup.path}` : "";
|
|
17445
|
-
|
|
18051
|
+
const rollbackHint = committed ? "\uFF1B\u4E8B\u52A1\u5DF2\u63D0\u4EA4\uFF0C\u672A\u81EA\u52A8\u8986\u76D6\u540E\u7EED\u5199\u5165" : "\uFF1B\u4E8B\u52A1\u5DF2\u56DE\u6EDA";
|
|
18052
|
+
throw new Error(`\u6062\u590D\u6570\u636E\u5E93\u5931\u8D25${rollbackHint}${backupHint}\uFF1A${errorMessage(error)}`);
|
|
18053
|
+
} finally {
|
|
18054
|
+
safeUnlink(stagePath);
|
|
18055
|
+
safeUnlink(`${stagePath}-journal`);
|
|
18056
|
+
safeUnlink(`${stagePath}-wal`);
|
|
18057
|
+
safeUnlink(`${stagePath}-shm`);
|
|
17446
18058
|
}
|
|
17447
18059
|
}
|
|
17448
18060
|
static inspectFile(path) {
|
|
@@ -17458,6 +18070,78 @@ var DatabaseMaintenanceService = class {
|
|
|
17458
18070
|
sqlite2.close();
|
|
17459
18071
|
}
|
|
17460
18072
|
}
|
|
18073
|
+
static buildRestorePlan(sqlite2) {
|
|
18074
|
+
const liveTables = this.readRestoreTables(sqlite2, "main");
|
|
18075
|
+
const sourceTables = this.readRestoreTables(sqlite2, "restore_source");
|
|
18076
|
+
for (const [tableName, sourceTable] of sourceTables) {
|
|
18077
|
+
const liveTable = liveTables.get(tableName);
|
|
18078
|
+
if (!liveTable) {
|
|
18079
|
+
throw new Error(`\u6062\u590D\u6765\u6E90\u5305\u542B\u5F53\u524D\u6570\u636E\u5E93\u4E0D\u8BA4\u8BC6\u7684\u4E1A\u52A1\u8868\uFF1A${tableName}`);
|
|
18080
|
+
}
|
|
18081
|
+
const liveColumns = new Map(liveTable.columns.map((column) => [column.name, column]));
|
|
18082
|
+
for (const sourceColumn of sourceTable.columns) {
|
|
18083
|
+
const liveColumn = liveColumns.get(sourceColumn.name);
|
|
18084
|
+
if (!liveColumn) {
|
|
18085
|
+
throw new Error(
|
|
18086
|
+
`\u6062\u590D\u6765\u6E90\u7684\u8868 ${tableName} \u5305\u542B\u5F53\u524D\u6570\u636E\u5E93\u4E0D\u8BA4\u8BC6\u7684\u53EF\u5199\u5217\uFF1A${sourceColumn.name}`
|
|
18087
|
+
);
|
|
18088
|
+
}
|
|
18089
|
+
if (!columnDefinitionsMatch(liveColumn, sourceColumn)) {
|
|
18090
|
+
throw new Error(`\u6062\u590D\u6765\u6E90\u7684\u8868 ${tableName}.${sourceColumn.name} \u4E0E\u5F53\u524D\u6570\u636E\u5E93\u5B9A\u4E49\u4E0D\u517C\u5BB9`);
|
|
18091
|
+
}
|
|
18092
|
+
}
|
|
18093
|
+
}
|
|
18094
|
+
const plan = [];
|
|
18095
|
+
for (const [tableName, liveTable] of liveTables) {
|
|
18096
|
+
const sourceTable = sourceTables.get(tableName);
|
|
18097
|
+
if (!sourceTable) {
|
|
18098
|
+
plan.push({ ...liveTable, columns: [], sourceExists: false });
|
|
18099
|
+
continue;
|
|
18100
|
+
}
|
|
18101
|
+
const sourceColumns = new Map(sourceTable.columns.map((column) => [column.name, column]));
|
|
18102
|
+
for (const liveColumn of liveTable.columns) {
|
|
18103
|
+
if (sourceColumns.has(liveColumn.name)) continue;
|
|
18104
|
+
if (liveColumn.primaryKeyOrder > 0 || liveColumn.notNull && liveColumn.defaultValue == null) {
|
|
18105
|
+
throw new Error(
|
|
18106
|
+
`\u5F53\u524D\u6570\u636E\u5E93\u7684\u8868 ${tableName}.${liveColumn.name} \u65E0\u6CD5\u4ECE\u65E7\u5907\u4EFD\u5B89\u5168\u8865\u9ED8\u8BA4\u503C`
|
|
18107
|
+
);
|
|
18108
|
+
}
|
|
18109
|
+
}
|
|
18110
|
+
plan.push({
|
|
18111
|
+
name: tableName,
|
|
18112
|
+
columns: sourceTable.columns,
|
|
18113
|
+
sourceExists: true
|
|
18114
|
+
});
|
|
18115
|
+
}
|
|
18116
|
+
return plan;
|
|
18117
|
+
}
|
|
18118
|
+
static readRestoreTables(sqlite2, databaseName) {
|
|
18119
|
+
const tableRows = sqlite2.query(`
|
|
18120
|
+
SELECT name
|
|
18121
|
+
FROM ${databaseName}.sqlite_schema
|
|
18122
|
+
WHERE type = 'table'
|
|
18123
|
+
AND name NOT LIKE 'sqlite_%'
|
|
18124
|
+
AND name NOT IN ('gateway_lock', '__drizzle_migrations')
|
|
18125
|
+
ORDER BY name
|
|
18126
|
+
`).all();
|
|
18127
|
+
const result = /* @__PURE__ */ new Map();
|
|
18128
|
+
for (const row of tableRows) {
|
|
18129
|
+
const columns = sqlite2.query(
|
|
18130
|
+
`PRAGMA ${databaseName}.table_xinfo(${quoteIdentifier(row.name)})`
|
|
18131
|
+
).all();
|
|
18132
|
+
result.set(row.name, {
|
|
18133
|
+
name: row.name,
|
|
18134
|
+
columns: columns.filter((column) => column.hidden === 0).map((column) => ({
|
|
18135
|
+
name: column.name,
|
|
18136
|
+
type: column.type,
|
|
18137
|
+
notNull: column.notnull !== 0,
|
|
18138
|
+
defaultValue: column.dflt_value,
|
|
18139
|
+
primaryKeyOrder: column.pk
|
|
18140
|
+
}))
|
|
18141
|
+
});
|
|
18142
|
+
}
|
|
18143
|
+
return result;
|
|
18144
|
+
}
|
|
17461
18145
|
static inspect(sqlite2, path) {
|
|
17462
18146
|
const tableRows = sqlite2.query(
|
|
17463
18147
|
"SELECT name FROM sqlite_master WHERE type = 'table'"
|
|
@@ -17677,10 +18361,14 @@ var state = null;
|
|
|
17677
18361
|
function componentStatus(component, enabled, maxAgeMs, now) {
|
|
17678
18362
|
const lastActivityAt = state?.lastActivityAt[component] ?? null;
|
|
17679
18363
|
const ageMs = lastActivityAt == null ? null : Math.max(0, now - lastActivityAt);
|
|
18364
|
+
const consecutiveFailures = state?.consecutiveFailures[component] ?? 0;
|
|
17680
18365
|
return {
|
|
17681
18366
|
enabled,
|
|
17682
|
-
healthy: !enabled || ageMs != null && ageMs <= maxAgeMs,
|
|
18367
|
+
healthy: !enabled || ageMs != null && ageMs <= maxAgeMs && consecutiveFailures === 0,
|
|
17683
18368
|
lastActivityAt,
|
|
18369
|
+
lastSuccessAt: state?.lastSuccessAt[component] ?? null,
|
|
18370
|
+
consecutiveFailures,
|
|
18371
|
+
lastError: state?.lastError[component] ?? null,
|
|
17684
18372
|
ageMs,
|
|
17685
18373
|
maxAgeMs
|
|
17686
18374
|
};
|
|
@@ -17704,6 +18392,12 @@ function getGatewayHealth(now = Date.now()) {
|
|
|
17704
18392
|
Math.max((state?.config.watchdogCheckIntervalMs ?? 6e4) * 3, 5e3),
|
|
17705
18393
|
now
|
|
17706
18394
|
);
|
|
18395
|
+
const watchdogCleanup = componentStatus(
|
|
18396
|
+
"watchdogCleanup",
|
|
18397
|
+
true,
|
|
18398
|
+
Math.max((state?.config.watchdogCleanupIntervalMs ?? 864e5) * 2, 6e4),
|
|
18399
|
+
now
|
|
18400
|
+
);
|
|
17707
18401
|
let lock = { pid: null, heartbeatAt: null, readyAt: null, ageMs: null, healthy: false };
|
|
17708
18402
|
try {
|
|
17709
18403
|
const row = sqlite.prepare(
|
|
@@ -17721,16 +18415,95 @@ function getGatewayHealth(now = Date.now()) {
|
|
|
17721
18415
|
}
|
|
17722
18416
|
} catch {
|
|
17723
18417
|
}
|
|
17724
|
-
const healthy = state != null && lock.healthy && worker.healthy && scheduler.healthy && watchdog.healthy;
|
|
18418
|
+
const healthy = state != null && lock.healthy && worker.healthy && scheduler.healthy && watchdog.healthy && watchdogCleanup.healthy;
|
|
17725
18419
|
return {
|
|
17726
18420
|
status: healthy ? "ok" : "degraded",
|
|
17727
18421
|
pid: process.pid,
|
|
17728
18422
|
uptimeMs: state ? Math.max(0, now - state.startedAt) : 0,
|
|
17729
18423
|
lock,
|
|
17730
|
-
components: { worker, scheduler, watchdog }
|
|
18424
|
+
components: { worker, scheduler, watchdog, watchdogCleanup }
|
|
17731
18425
|
};
|
|
17732
18426
|
}
|
|
17733
18427
|
|
|
18428
|
+
// src/gateway/scheduler/job-templates.ts
|
|
18429
|
+
var { taskTemplates: taskTemplates3 } = schema_exports;
|
|
18430
|
+
async function triggerTaskFromTemplate(templateId) {
|
|
18431
|
+
return createTaskFromTemplate(templateId, {
|
|
18432
|
+
advanceSchedule: false,
|
|
18433
|
+
namePrefix: "[\u624B\u52A8\u89E6\u53D1] "
|
|
18434
|
+
});
|
|
18435
|
+
}
|
|
18436
|
+
function createTaskFromTemplate(templateId, options) {
|
|
18437
|
+
const nowMs = Date.now();
|
|
18438
|
+
return db.transaction((tx) => {
|
|
18439
|
+
const tmpl = tx.select().from(taskTemplates3).where(eq(taskTemplates3.id, templateId)).limit(1).get();
|
|
18440
|
+
if (!tmpl || options.advanceSchedule && !tmpl.enabled) return null;
|
|
18441
|
+
const activeTasks = tx.select({ count: sql`count(*)` }).from(schema_exports.tasks).where(and(
|
|
18442
|
+
eq(schema_exports.tasks.templateId, templateId),
|
|
18443
|
+
sql`${schema_exports.tasks.status} IN ('pending', 'running')`
|
|
18444
|
+
)).get();
|
|
18445
|
+
const retryableTasks = tx.select({ count: sql`count(*)` }).from(schema_exports.tasks).where(and(
|
|
18446
|
+
eq(schema_exports.tasks.templateId, templateId),
|
|
18447
|
+
eq(schema_exports.tasks.status, "failed"),
|
|
18448
|
+
sql`${schema_exports.tasks.retryCount} <= ${schema_exports.tasks.maxRetries}`
|
|
18449
|
+
)).get();
|
|
18450
|
+
const detachedRuns = tx.select({ count: sql`count(DISTINCT ${schema_exports.taskRuns.taskId})` }).from(schema_exports.taskRuns).innerJoin(schema_exports.tasks, eq(schema_exports.tasks.id, schema_exports.taskRuns.taskId)).where(and(
|
|
18451
|
+
eq(schema_exports.taskRuns.status, "running"),
|
|
18452
|
+
eq(schema_exports.tasks.templateId, templateId),
|
|
18453
|
+
sql`NOT (
|
|
18454
|
+
${schema_exports.tasks.status} IN ('pending', 'running')
|
|
18455
|
+
OR (
|
|
18456
|
+
${schema_exports.tasks.status} = 'failed'
|
|
18457
|
+
AND ${schema_exports.tasks.retryCount} <= ${schema_exports.tasks.maxRetries}
|
|
18458
|
+
)
|
|
18459
|
+
)`
|
|
18460
|
+
)).get();
|
|
18461
|
+
const activeCount = Number(activeTasks?.count ?? 0) + Number(retryableTasks?.count ?? 0) + Number(detachedRuns?.count ?? 0);
|
|
18462
|
+
if (activeCount >= (tmpl.maxInstances ?? 1)) {
|
|
18463
|
+
if (options.advanceSchedule && tmpl.scheduleType !== "delayed") {
|
|
18464
|
+
const nextRunAt2 = TaskTemplateService.calculateNextRunAt(
|
|
18465
|
+
tmpl.scheduleType,
|
|
18466
|
+
tmpl,
|
|
18467
|
+
nowMs
|
|
18468
|
+
);
|
|
18469
|
+
tx.update(taskTemplates3).set({ nextRunAt: nextRunAt2, updatedAt: nowMs }).where(and(eq(taskTemplates3.id, templateId), eq(taskTemplates3.enabled, true))).run();
|
|
18470
|
+
}
|
|
18471
|
+
return null;
|
|
18472
|
+
}
|
|
18473
|
+
const isDelayed = tmpl.scheduleType === "delayed";
|
|
18474
|
+
const nextRunAt = isDelayed ? null : TaskTemplateService.calculateNextRunAt(
|
|
18475
|
+
tmpl.scheduleType,
|
|
18476
|
+
tmpl,
|
|
18477
|
+
nowMs
|
|
18478
|
+
);
|
|
18479
|
+
const task = tx.insert(schema_exports.tasks).values({
|
|
18480
|
+
name: `${options.namePrefix ?? ""}${tmpl.name}`,
|
|
18481
|
+
agent: tmpl.agent,
|
|
18482
|
+
model: tmpl.model ?? "default",
|
|
18483
|
+
prompt: tmpl.prompt,
|
|
18484
|
+
cwd: tmpl.cwd ?? null,
|
|
18485
|
+
category: tmpl.category ?? "general",
|
|
18486
|
+
importance: tmpl.importance ?? 3,
|
|
18487
|
+
urgency: tmpl.urgency ?? 3,
|
|
18488
|
+
batchId: tmpl.batchId,
|
|
18489
|
+
maxRetries: tmpl.maxRetries ?? 3,
|
|
18490
|
+
retryBackoffMs: tmpl.retryBackoffMs ?? 3e4,
|
|
18491
|
+
timeoutMs: tmpl.timeoutMs,
|
|
18492
|
+
templateId: tmpl.id,
|
|
18493
|
+
scheduledAt: options.advanceSchedule ? tmpl.nextRunAt ?? nowMs : nowMs
|
|
18494
|
+
}).returning().get();
|
|
18495
|
+
if (options.advanceSchedule) {
|
|
18496
|
+
tx.update(taskTemplates3).set({
|
|
18497
|
+
lastRunAt: nowMs,
|
|
18498
|
+
nextRunAt,
|
|
18499
|
+
enabled: isDelayed ? false : tmpl.enabled,
|
|
18500
|
+
updatedAt: nowMs
|
|
18501
|
+
}).where(and(eq(taskTemplates3.id, templateId), eq(taskTemplates3.enabled, true))).run();
|
|
18502
|
+
}
|
|
18503
|
+
return task;
|
|
18504
|
+
}, { behavior: "immediate" });
|
|
18505
|
+
}
|
|
18506
|
+
|
|
17734
18507
|
// src/web/index.tsx
|
|
17735
18508
|
var app = new Hono2();
|
|
17736
18509
|
var TASK_STATUSES = /* @__PURE__ */ new Set([
|
|
@@ -17914,8 +18687,9 @@ var SHARED_STYLES = html`
|
|
|
17914
18687
|
function renderLayout(title, activeTab, body) {
|
|
17915
18688
|
return `<!DOCTYPE html><html><head><meta charset="UTF-8"><title>${title} - SuperTask</title>${SHARED_STYLES}
|
|
17916
18689
|
<script>
|
|
17917
|
-
async function retryTask(id){if(!confirm('\u786E\u5B9A\u91CD\u8BD5\u4EFB\u52A1 #'+id+'?'))return;await fetch('/api/tasks/'+id+'/retry',{method:'POST'});location.reload();}
|
|
17918
|
-
async function
|
|
18690
|
+
async function retryTask(id){if(!confirm('\u786E\u5B9A\u91CD\u8BD5\u4EFB\u52A1 #'+id+'?'))return;const r=await fetch('/api/tasks/'+id+'/retry',{method:'POST'});const d=await r.json();if(!r.ok){alert('\u91CD\u8BD5\u5931\u8D25: '+d.error);return;}location.reload();}
|
|
18691
|
+
async function cancelTask(id){if(!confirm('\u786E\u5B9A\u53D6\u6D88\u4EFB\u52A1 #'+id+'?'))return;const r=await fetch('/api/tasks/'+id+'/cancel',{method:'POST'});const d=await r.json();if(!r.ok){alert('\u53D6\u6D88\u5931\u8D25: '+d.error);return;}location.reload();}
|
|
18692
|
+
async function deleteTask(id){if(!confirm('\u786E\u5B9A\u5220\u9664\u4EFB\u52A1 #'+id+'?'))return;const r=await fetch('/api/tasks/'+id,{method:'DELETE'});const d=await r.json();if(!r.ok){alert('\u5220\u9664\u5931\u8D25: '+d.error);return;}location.reload();}
|
|
17919
18693
|
async function showDetail(id){try{const r=await fetch('/api/tasks/'+id);const t=await r.json();document.getElementById('dc').textContent=JSON.stringify(t,null,2);document.getElementById('dd').showModal();}catch(e){alert('\u83B7\u53D6\u8BE6\u60C5\u5931\u8D25');}}
|
|
17920
18694
|
async function showRunDetail(id){try{const r=await fetch('/api/runs/'+id);const t=await r.json();document.getElementById('dc').textContent=JSON.stringify(t,null,2);document.getElementById('dd').showModal();}catch(e){alert('\u83B7\u53D6\u8BE6\u60C5\u5931\u8D25');}}
|
|
17921
18695
|
async function showTemplateDetail(id){try{const r=await fetch('/api/templates/'+id);const t=await r.json();document.getElementById('dc').textContent=JSON.stringify(t,null,2);document.getElementById('dd').showModal();}catch(e){alert('\u83B7\u53D6\u8BE6\u60C5\u5931\u8D25');}}
|
|
@@ -17992,11 +18766,11 @@ app.get("/", async (c) => {
|
|
|
17992
18766
|
if (statusFilter && !parsedStatus) return c.text("invalid status", 400);
|
|
17993
18767
|
const limit = 50;
|
|
17994
18768
|
const offset = (page - 1) * limit;
|
|
17995
|
-
const [
|
|
18769
|
+
const [tasks4, statsData] = await Promise.all([
|
|
17996
18770
|
TaskService.list({ limit, offset, ...parsedStatus ? { status: parsedStatus } : {} }),
|
|
17997
18771
|
TaskService.stats({})
|
|
17998
18772
|
]);
|
|
17999
|
-
const taskIds =
|
|
18773
|
+
const taskIds = tasks4.map((t) => t.id);
|
|
18000
18774
|
const latestRuns = await TaskRunService.getLatestByTaskIds(taskIds);
|
|
18001
18775
|
const counts = {
|
|
18002
18776
|
pending: statsData.pending || 0,
|
|
@@ -18015,9 +18789,10 @@ app.get("/", async (c) => {
|
|
|
18015
18789
|
<a href="/?status=dead_letter" class="btn ${statusFilter === "dead_letter" ? "btn-primary" : ""}">Dead Letter</a>
|
|
18016
18790
|
</div>`;
|
|
18017
18791
|
let rows = "";
|
|
18018
|
-
for (const task of
|
|
18792
|
+
for (const task of tasks4) {
|
|
18019
18793
|
const status = safeStatus(task.status);
|
|
18020
18794
|
const st = status.toUpperCase();
|
|
18795
|
+
const executionActive = latestRuns.get(task.id)?.status === "running";
|
|
18021
18796
|
rows += `<tr>
|
|
18022
18797
|
<td class="mu">#${task.id}</td>
|
|
18023
18798
|
<td><div style="font-weight:500">${esc(task.name)}</div><div class="mu sm el">${esc(task.prompt.substring(0, 120))}</div></td>
|
|
@@ -18028,7 +18803,8 @@ app.get("/", async (c) => {
|
|
|
18028
18803
|
<td>
|
|
18029
18804
|
<button class="btn btn-sm" onclick="showDetail(${task.id})">\u8BE6\u60C5</button>
|
|
18030
18805
|
${task.status === "failed" || task.status === "dead_letter" ? `<button class="btn btn-sm btn-warn" onclick="retryTask(${task.id})">\u91CD\u8BD5</button>` : ""}
|
|
18031
|
-
|
|
18806
|
+
${["pending", "running", "failed"].includes(task.status ?? "") ? `<button class="btn btn-sm btn-warn" onclick="cancelTask(${task.id})">\u53D6\u6D88</button>` : ""}
|
|
18807
|
+
${task.status === "running" || executionActive ? "" : `<button class="btn btn-sm btn-danger" onclick="deleteTask(${task.id})">\u5220\u9664</button>`}
|
|
18032
18808
|
</td></tr>`;
|
|
18033
18809
|
}
|
|
18034
18810
|
const qp = statusFilter ? `&status=${statusFilter}` : "";
|
|
@@ -18280,11 +19056,25 @@ app.post("/api/tasks/:id/retry", async (c) => {
|
|
|
18280
19056
|
if (task) return c.json({ success: true });
|
|
18281
19057
|
return await TaskService.getById(id) ? c.json({ error: "task status does not allow retry" }, 409) : c.json({ error: "not found" }, 404);
|
|
18282
19058
|
});
|
|
19059
|
+
app.post("/api/tasks/:id/cancel", async (c) => {
|
|
19060
|
+
const id = parsePositiveInteger(c.req.param("id"));
|
|
19061
|
+
if (id === null) return c.json({ error: "invalid id" }, 400);
|
|
19062
|
+
const task = await TaskService.cancel(id);
|
|
19063
|
+
if (task) return c.json({ success: true });
|
|
19064
|
+
return await TaskService.getById(id) ? c.json({ error: "task status does not allow cancellation" }, 409) : c.json({ error: "not found" }, 404);
|
|
19065
|
+
});
|
|
18283
19066
|
app.delete("/api/tasks/:id", async (c) => {
|
|
18284
19067
|
const id = parsePositiveInteger(c.req.param("id"));
|
|
18285
19068
|
if (id === null) return c.json({ error: "invalid id" }, 400);
|
|
18286
|
-
|
|
18287
|
-
|
|
19069
|
+
try {
|
|
19070
|
+
const deleted = await TaskService.delete(id);
|
|
19071
|
+
return deleted ? c.json({ success: true }) : c.json({ error: "not found" }, 404);
|
|
19072
|
+
} catch (error) {
|
|
19073
|
+
if (error instanceof TaskDeletionConflictError) {
|
|
19074
|
+
return c.json({ error: error.message }, 409);
|
|
19075
|
+
}
|
|
19076
|
+
throw error;
|
|
19077
|
+
}
|
|
18288
19078
|
});
|
|
18289
19079
|
app.post("/api/templates/:id/enable", async (c) => {
|
|
18290
19080
|
const id = parsePositiveInteger(c.req.param("id"));
|
|
@@ -18309,21 +19099,8 @@ app.post("/api/templates/:id/trigger", async (c) => {
|
|
|
18309
19099
|
if (id === null) return c.json({ error: "invalid id" }, 400);
|
|
18310
19100
|
const tmpl = await TaskTemplateService.getById(id);
|
|
18311
19101
|
if (!tmpl) return c.json({ error: "not found" }, 404);
|
|
18312
|
-
const task = await
|
|
18313
|
-
|
|
18314
|
-
agent: tmpl.agent,
|
|
18315
|
-
model: tmpl.model,
|
|
18316
|
-
prompt: tmpl.prompt,
|
|
18317
|
-
cwd: tmpl.cwd,
|
|
18318
|
-
category: tmpl.category,
|
|
18319
|
-
importance: tmpl.importance,
|
|
18320
|
-
urgency: tmpl.urgency,
|
|
18321
|
-
batchId: tmpl.batchId,
|
|
18322
|
-
maxRetries: tmpl.maxRetries,
|
|
18323
|
-
retryBackoffMs: tmpl.retryBackoffMs,
|
|
18324
|
-
timeoutMs: tmpl.timeoutMs,
|
|
18325
|
-
templateId: tmpl.id
|
|
18326
|
-
});
|
|
19102
|
+
const task = await triggerTaskFromTemplate(id);
|
|
19103
|
+
if (!task) return c.json({ error: "maxInstances reached" }, 409);
|
|
18327
19104
|
return c.json({ success: true, taskId: task.id });
|
|
18328
19105
|
});
|
|
18329
19106
|
app.put("/api/config", async (c) => {
|
|
@@ -18365,6 +19142,7 @@ app.post("/api/database/clear", async (c) => {
|
|
|
18365
19142
|
});
|
|
18366
19143
|
var dashboardApp = app;
|
|
18367
19144
|
var web_default = {
|
|
19145
|
+
hostname: "127.0.0.1",
|
|
18368
19146
|
port: 4680,
|
|
18369
19147
|
fetch: app.fetch
|
|
18370
19148
|
};
|