agent-steward 0.0.3 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +154 -2
- package/CROSS_PLATFORM.md +19 -0
- package/README.md +54 -9
- package/ROADMAP-STRATEGY.md +23 -20
- package/ROADMAP.md +140 -143
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +351 -0
- package/dist/core/dlq-api.d.ts +44 -0
- package/dist/core/dlq-api.d.ts.map +1 -0
- package/dist/core/dlq-api.js +68 -0
- package/dist/core/dry-run.d.ts.map +1 -1
- package/dist/core/dry-run.js +18 -1
- package/dist/core/execution.d.ts +203 -0
- package/dist/core/execution.d.ts.map +1 -0
- package/dist/core/execution.js +309 -0
- package/dist/core/gate-integrity.d.ts +35 -0
- package/dist/core/gate-integrity.d.ts.map +1 -0
- package/dist/core/gate-integrity.js +132 -0
- package/dist/core/git-trigger.d.ts.map +1 -1
- package/dist/core/git-trigger.js +14 -2
- package/dist/core/guard-run.d.ts +4 -0
- package/dist/core/guard-run.d.ts.map +1 -0
- package/dist/core/guard-run.js +101 -0
- package/dist/core/guard.d.ts +52 -0
- package/dist/core/guard.d.ts.map +1 -0
- package/dist/core/guard.js +362 -0
- package/dist/core/lock.d.ts +61 -4
- package/dist/core/lock.d.ts.map +1 -1
- package/dist/core/lock.js +213 -26
- package/dist/core/matrix.d.ts.map +1 -1
- package/dist/core/matrix.js +75 -15
- package/dist/core/mission.d.ts +83 -0
- package/dist/core/mission.d.ts.map +1 -0
- package/dist/core/mission.js +334 -0
- package/dist/core/one-shot.d.ts +5 -1
- package/dist/core/one-shot.d.ts.map +1 -1
- package/dist/core/one-shot.js +33 -8
- package/dist/core/opencode-cli.d.ts +28 -0
- package/dist/core/opencode-cli.d.ts.map +1 -0
- package/dist/core/opencode-cli.js +74 -0
- package/dist/core/opencode-models.d.ts +62 -0
- package/dist/core/opencode-models.d.ts.map +1 -0
- package/dist/core/opencode-models.js +277 -0
- package/dist/core/pipeline-apply.d.ts +38 -0
- package/dist/core/pipeline-apply.d.ts.map +1 -0
- package/dist/core/pipeline-apply.js +117 -0
- package/dist/core/pipeline-io.d.ts +13 -0
- package/dist/core/pipeline-io.d.ts.map +1 -0
- package/dist/core/pipeline-io.js +64 -0
- package/dist/core/pipeline-map.d.ts +46 -0
- package/dist/core/pipeline-map.d.ts.map +1 -0
- package/dist/core/pipeline-map.js +196 -0
- package/dist/core/pipeline-spec.d.ts +109 -0
- package/dist/core/pipeline-spec.d.ts.map +1 -0
- package/dist/core/pipeline-spec.js +554 -0
- package/dist/core/ps-wrapper.d.ts +19 -2
- package/dist/core/ps-wrapper.d.ts.map +1 -1
- package/dist/core/ps-wrapper.js +261 -63
- package/dist/core/record-history.d.ts +15 -0
- package/dist/core/record-history.d.ts.map +1 -1
- package/dist/core/record-history.js +183 -58
- package/dist/core/router/dispatch.d.ts +36 -0
- package/dist/core/router/dispatch.d.ts.map +1 -0
- package/dist/core/router/dispatch.js +59 -0
- package/dist/core/router/escalation.d.ts +47 -0
- package/dist/core/router/escalation.d.ts.map +1 -0
- package/dist/core/router/escalation.js +87 -0
- package/dist/core/router/planner.d.ts +53 -0
- package/dist/core/router/planner.d.ts.map +1 -0
- package/dist/core/router/planner.js +113 -0
- package/dist/core/router/rules.d.ts +46 -0
- package/dist/core/router/rules.d.ts.map +1 -0
- package/dist/core/router/rules.js +91 -0
- package/dist/core/run-panel-api.d.ts +94 -0
- package/dist/core/run-panel-api.d.ts.map +1 -0
- package/dist/core/run-panel-api.js +169 -0
- package/dist/core/schedule-gate.js +5 -1
- package/dist/core/schedule-v2.d.ts +34 -0
- package/dist/core/schedule-v2.d.ts.map +1 -1
- package/dist/core/schedule-v2.js +91 -5
- package/dist/core/scheduler/at-fallback.d.ts +55 -2
- package/dist/core/scheduler/at-fallback.d.ts.map +1 -1
- package/dist/core/scheduler/at-fallback.js +135 -5
- package/dist/core/scheduler/linux.d.ts +18 -2
- package/dist/core/scheduler/linux.d.ts.map +1 -1
- package/dist/core/scheduler/linux.js +63 -20
- package/dist/core/scheduler/mac.d.ts +44 -2
- package/dist/core/scheduler/mac.d.ts.map +1 -1
- package/dist/core/scheduler/mac.js +126 -15
- package/dist/core/scheduler/posix.d.ts.map +1 -1
- package/dist/core/scheduler/posix.js +25 -1
- package/dist/core/scheduler/registry.d.ts +11 -0
- package/dist/core/scheduler/registry.d.ts.map +1 -1
- package/dist/core/scheduler/registry.js +83 -21
- package/dist/core/scheduler/run-now.d.ts +16 -0
- package/dist/core/scheduler/run-now.d.ts.map +1 -0
- package/dist/core/scheduler/run-now.js +28 -0
- package/dist/core/scheduler/windows.d.ts.map +1 -1
- package/dist/core/scheduler/windows.js +15 -12
- package/dist/core/secrets.d.ts.map +1 -1
- package/dist/core/secrets.js +125 -28
- package/dist/core/session.d.ts.map +1 -1
- package/dist/core/session.js +18 -11
- package/dist/core/structured-log.d.ts +7 -1
- package/dist/core/structured-log.d.ts.map +1 -1
- package/dist/core/task-name.d.ts +24 -0
- package/dist/core/task-name.d.ts.map +1 -0
- package/dist/core/task-name.js +40 -0
- package/dist/core/worktree-run.d.ts +4 -0
- package/dist/core/worktree-run.d.ts.map +1 -0
- package/dist/core/worktree-run.js +164 -0
- package/dist/core/worktree.d.ts +115 -0
- package/dist/core/worktree.d.ts.map +1 -0
- package/dist/core/worktree.js +462 -0
- package/dist/core/wrappers/bash.d.ts +17 -0
- package/dist/core/wrappers/bash.d.ts.map +1 -1
- package/dist/core/wrappers/bash.js +224 -28
- package/dist/db/capability-repo.d.ts +124 -0
- package/dist/db/capability-repo.d.ts.map +1 -0
- package/dist/db/capability-repo.js +267 -0
- package/dist/db/history-repo.d.ts +46 -5
- package/dist/db/history-repo.d.ts.map +1 -1
- package/dist/db/history-repo.js +107 -25
- package/dist/db/index.d.ts +4 -0
- package/dist/db/index.d.ts.map +1 -1
- package/dist/db/index.js +39 -1
- package/dist/db/metrics-repo.js +2 -2
- package/dist/db/migrate.d.ts.map +1 -1
- package/dist/db/migrate.js +3 -1
- package/dist/db/recovery.d.ts.map +1 -1
- package/dist/db/recovery.js +5 -4
- package/dist/db/run-context.d.ts +103 -0
- package/dist/db/run-context.d.ts.map +1 -0
- package/dist/db/run-context.js +224 -0
- package/dist/db/run-dlq.d.ts +100 -0
- package/dist/db/run-dlq.d.ts.map +1 -0
- package/dist/db/run-dlq.js +191 -0
- package/dist/db/run-journal.d.ts +33 -5
- package/dist/db/run-journal.d.ts.map +1 -1
- package/dist/db/run-journal.js +78 -14
- package/dist/db/run-resume.d.ts +10 -2
- package/dist/db/run-resume.d.ts.map +1 -1
- package/dist/db/run-resume.js +39 -13
- package/dist/db/runs-repo.d.ts +32 -7
- package/dist/db/runs-repo.d.ts.map +1 -1
- package/dist/db/runs-repo.js +50 -11
- package/dist/db/schema.d.ts +2 -2
- package/dist/db/schema.d.ts.map +1 -1
- package/dist/db/schema.js +216 -3
- package/dist/db/settings-repo.d.ts +11 -0
- package/dist/db/settings-repo.d.ts.map +1 -1
- package/dist/db/settings-repo.js +25 -0
- package/dist/db/step-idempotency.d.ts +51 -0
- package/dist/db/step-idempotency.d.ts.map +1 -0
- package/dist/db/step-idempotency.js +136 -0
- package/dist/history.d.ts +1 -1
- package/dist/history.d.ts.map +1 -1
- package/dist/i18n.d.ts +5 -0
- package/dist/i18n.d.ts.map +1 -1
- package/dist/i18n.js +11 -4
- package/dist/index.d.ts +140 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +480 -18
- package/dist/notifications.d.ts +22 -0
- package/dist/notifications.d.ts.map +1 -1
- package/dist/notifications.js +36 -8
- package/dist/profiles.d.ts.map +1 -1
- package/dist/profiles.js +25 -4
- package/dist/rpc.d.ts +924 -0
- package/dist/rpc.d.ts.map +1 -1
- package/dist/rpc.js +352 -0
- package/dist/server.d.ts +46 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +172 -53
- package/dist/settings.d.ts +2 -2
- package/dist/settings.d.ts.map +1 -1
- package/dist/settings.js +3 -3
- package/dist/shared/jobs-view.d.ts +1 -1
- package/dist/shared/jobs-view.d.ts.map +1 -1
- package/dist/shared/jobs-view.js +2 -0
- package/dist/task-transfer.d.ts +16 -0
- package/dist/task-transfer.d.ts.map +1 -1
- package/dist/task-transfer.js +16 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +7 -0
- package/dist/workspaces.d.ts +25 -3
- package/dist/workspaces.d.ts.map +1 -1
- package/dist/workspaces.js +43 -8
- package/docs/ARCHITECTURE.md +498 -58
- package/docs/COMPATIBILITY.md +2 -2
- package/docs/CONCEPT.md +185 -91
- package/docs/CROSS_PLATFORM.ru.md +20 -0
- package/docs/CROSS_PLATFORM.zh.md +18 -0
- package/docs/EXECUTION-LOOP.md +211 -0
- package/docs/FEATURES.md +65 -4
- package/docs/FEATURES.zh.md +52 -4
- package/docs/MIGRATION.md +2 -2
- package/docs/PIPELINE.md +293 -0
- package/docs/README.ru.md +44 -5
- package/docs/README.zh.md +46 -6
- package/docs/SAFETY.md +142 -0
- package/docs/SAVINGS.md +29 -0
- package/docs/SAVINGS.ru.md +29 -0
- package/docs/SAVINGS.zh.md +18 -0
- package/gui-experimental.cjs +9 -8
- package/locales/en.json +84 -1
- package/locales/ru.json +84 -1
- package/locales/zh.json +84 -1
- package/opencode.pipeline.yaml +35 -0
- package/package.json +46 -2
- package/scripts/audit-bundle.mjs +215 -0
- package/scripts/test-home.mjs +83 -0
- package/src/cli.ts +412 -0
- package/src/core/dlq-api.ts +98 -0
- package/src/core/dry-run.ts +27 -3
- package/src/core/execution.ts +538 -0
- package/src/core/file-watcher.ts +2 -2
- package/src/core/gate-integrity.ts +160 -0
- package/src/core/git-trigger.ts +18 -5
- package/src/core/guard-run.ts +119 -0
- package/src/core/guard.ts +431 -0
- package/src/core/lock.ts +244 -30
- package/src/core/matrix.ts +95 -50
- package/src/core/mission.ts +485 -0
- package/src/core/native-notify.ts +1 -1
- package/src/core/one-shot.ts +36 -8
- package/src/core/opencode-cli.ts +103 -0
- package/src/core/opencode-models.ts +315 -0
- package/src/core/pipeline-apply.ts +158 -0
- package/src/core/pipeline-io.ts +79 -0
- package/src/core/pipeline-map.ts +264 -0
- package/src/core/pipeline-spec.ts +775 -0
- package/src/core/ps-wrapper.ts +274 -64
- package/src/core/record-history.ts +229 -104
- package/src/core/router/dispatch.ts +94 -0
- package/src/core/router/escalation.ts +129 -0
- package/src/core/router/planner.ts +166 -0
- package/src/core/router/rules.ts +115 -0
- package/src/core/run-panel-api.ts +262 -0
- package/src/core/schedule-gate.ts +7 -3
- package/src/core/schedule-v2.ts +114 -5
- package/src/core/scheduler/at-fallback.ts +190 -5
- package/src/core/scheduler/linux.ts +72 -21
- package/src/core/scheduler/logs.ts +2 -2
- package/src/core/scheduler/mac.ts +174 -14
- package/src/core/scheduler/posix.ts +20 -1
- package/src/core/scheduler/registry.ts +107 -26
- package/src/core/scheduler/run-now.ts +32 -0
- package/src/core/scheduler/windows.ts +19 -15
- package/src/core/secrets.ts +131 -31
- package/src/core/session.ts +23 -10
- package/src/core/structured-log.ts +22 -5
- package/src/core/task-name.ts +67 -0
- package/src/core/telegram.ts +2 -2
- package/src/core/triage-worker.ts +7 -7
- package/src/core/triage.ts +2 -2
- package/src/core/worktree-run.ts +199 -0
- package/src/core/worktree.ts +600 -0
- package/src/core/wrappers/bash.ts +239 -27
- package/src/db/capability-repo.ts +402 -0
- package/src/db/history-repo.ts +151 -22
- package/src/db/index.ts +37 -1
- package/src/db/metrics-repo.ts +2 -2
- package/src/db/migrate.ts +3 -1
- package/src/db/recovery.ts +5 -4
- package/src/db/run-context.ts +370 -0
- package/src/db/run-dlq.ts +310 -0
- package/src/db/run-journal.ts +100 -13
- package/src/db/run-resume.ts +65 -13
- package/src/db/runs-repo.ts +66 -13
- package/src/db/schema.ts +218 -3
- package/src/db/settings-repo.ts +31 -0
- package/src/db/step-idempotency.ts +185 -0
- package/src/gui.tsx +692 -13
- package/src/history.ts +1 -1
- package/src/i18n.ts +12 -4
- package/src/index.ts +648 -8
- package/src/notifications.ts +38 -8
- package/src/profiles.ts +26 -4
- package/src/rpc.ts +379 -0
- package/src/server.ts +206 -54
- package/src/settings.ts +5 -5
- package/src/shared/jobs-view.ts +3 -1
- package/src/task-transfer.ts +31 -0
- package/src/types.ts +9 -0
- package/src/workspaces.ts +58 -9
package/src/core/lock.ts
CHANGED
|
@@ -1,7 +1,137 @@
|
|
|
1
1
|
import fs from "node:fs/promises";
|
|
2
2
|
import path from "node:path";
|
|
3
|
+
import { randomUUID } from "node:crypto";
|
|
3
4
|
import { LOCK_DIR, STALE_LOCK_MAX_AGE_MS } from "../types.js";
|
|
4
5
|
|
|
6
|
+
/**
|
|
7
|
+
* O-2: ошибки очистки больше не проглатываются молча. Прямой импорт
|
|
8
|
+
* `structured-log.ts` создал бы цикл (structured-log → secrets → lock), поэтому
|
|
9
|
+
* здесь используется `console.warn` — предупреждение всё равно видно в логе
|
|
10
|
+
* процесса и не ломает основной сценарий.
|
|
11
|
+
*/
|
|
12
|
+
function warnCleanupFailure(target: string, error: unknown): void {
|
|
13
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
14
|
+
console.warn(`[agent-steward] Не удалось удалить lock-артефакт «${target}»: ${reason}`);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Проверяет, жив ли процесс-владелец lock-файла.
|
|
19
|
+
*
|
|
20
|
+
* `process.kill(pid, 0)` не посылает сигнал, а только проверяет существование.
|
|
21
|
+
* EPERM означает, что процесс есть, но у нас нет прав на сигнал — считаем живым.
|
|
22
|
+
* Остальные ошибки трактуем консервативно как «жив», чтобы не вытеснить
|
|
23
|
+
* чужую критическую секцию.
|
|
24
|
+
*/
|
|
25
|
+
export function isProcessAlive(pid: number): boolean {
|
|
26
|
+
if (!Number.isInteger(pid) || pid <= 0) return false;
|
|
27
|
+
try {
|
|
28
|
+
process.kill(pid, 0);
|
|
29
|
+
return true;
|
|
30
|
+
} catch (err: unknown) {
|
|
31
|
+
if ((err as NodeJS.ErrnoException | undefined)?.code === "ESRCH") return false;
|
|
32
|
+
return true;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Разобранный lock-токен: PID владельца и формат записи. */
|
|
37
|
+
export interface LockToken {
|
|
38
|
+
pid: number | null;
|
|
39
|
+
createdAt: string | null;
|
|
40
|
+
format: "unified" | "node-legacy" | "ps-legacy" | "pid" | "unknown";
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** Достаёт ISO-время из первой части legacy PS-токена (`ISO|PID`). */
|
|
44
|
+
function parseIsoOrNull(value: string | undefined): string | null {
|
|
45
|
+
const text = String(value ?? "").trim();
|
|
46
|
+
return /^\d{4}-\d{2}-\d{2}T/.test(text) ? text : null;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Разбирает содержимое lock-файла в едином для Node/PS/bash виде.
|
|
51
|
+
*
|
|
52
|
+
* Поддерживаемые форматы:
|
|
53
|
+
* `<pid>-<iso>` — единый формат (Node и PS, B6);
|
|
54
|
+
* `<pid>-<epoch>-<uuid>` — старый Node-токен;
|
|
55
|
+
* `<iso>|<pid>` — старый PS-токен (ISO|PID);
|
|
56
|
+
* `<pid>` — голый PID (pid-файл bash-обёртки).
|
|
57
|
+
*
|
|
58
|
+
* Порядок важен: legacy PS проверяется до `<pid>-…`, иначе год из ISO
|
|
59
|
+
* (`2026`) принимался за PID — именно это ломало межпроцессную совместимость.
|
|
60
|
+
*/
|
|
61
|
+
export function parseLockToken(content: string): LockToken {
|
|
62
|
+
const text = String(content ?? "").trim();
|
|
63
|
+
if (!text) return { pid: null, createdAt: null, format: "unknown" };
|
|
64
|
+
|
|
65
|
+
const withPipe = text.indexOf("|");
|
|
66
|
+
if (withPipe !== -1) {
|
|
67
|
+
const pid = Number.parseInt(text.slice(withPipe + 1).trim(), 10);
|
|
68
|
+
return {
|
|
69
|
+
pid: Number.isInteger(pid) && pid > 0 ? pid : null,
|
|
70
|
+
createdAt: parseIsoOrNull(text.slice(0, withPipe)),
|
|
71
|
+
format: "ps-legacy",
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const unified = /^(\d+)-(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)(?:-.*)?$/i.exec(text);
|
|
76
|
+
if (unified) {
|
|
77
|
+
const pid = Number.parseInt(unified[1], 10);
|
|
78
|
+
return {
|
|
79
|
+
pid: Number.isInteger(pid) && pid > 0 ? pid : null,
|
|
80
|
+
createdAt: unified[2],
|
|
81
|
+
format: "unified",
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// Старый Node мог добавить любой суффикс после epoch (`-live-owner`, uuid).
|
|
86
|
+
const legacyNode = /^(\d+)-(\d{10,})(?:-.*)?$/i.exec(text);
|
|
87
|
+
if (legacyNode) {
|
|
88
|
+
const pid = Number.parseInt(legacyNode[1], 10);
|
|
89
|
+
return {
|
|
90
|
+
pid: Number.isInteger(pid) && pid > 0 ? pid : null,
|
|
91
|
+
createdAt: null,
|
|
92
|
+
format: "node-legacy",
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
if (/^\d+$/.test(text)) {
|
|
97
|
+
const pid = Number.parseInt(text, 10);
|
|
98
|
+
// Голое число — PID (pid-файл bash), но не epoch из старого тестового
|
|
99
|
+
// lock: 1.7e12 не может быть PID и раньше не распознавалось владельцем.
|
|
100
|
+
return {
|
|
101
|
+
pid: Number.isInteger(pid) && pid > 0 && pid <= 2_147_483_647 ? pid : null,
|
|
102
|
+
createdAt: null,
|
|
103
|
+
format: "pid",
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
return { pid: null, createdAt: null, format: "unknown" };
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** PID владельца из токена lock-файла (любой поддерживаемый формат). */
|
|
111
|
+
export function ownerPidFromToken(content: string): number | null {
|
|
112
|
+
return parseLockToken(content).pid;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* PID владельца lock-артефакта.
|
|
117
|
+
*
|
|
118
|
+
* B7: файловый lock Node/PS хранит токен в самом файле, а POSIX-обёртка
|
|
119
|
+
* держит каталог `<lock>.d` и пишет `<pid>-<iso>` в `<lock>.d/pid`.
|
|
120
|
+
* Раньше каталог вытеснялся только по возрасту, без чтения этого файла,
|
|
121
|
+
* поэтому легально долгая задача (> 24 ч) после рестарта плагина
|
|
122
|
+
* дублировалась. Нет pid-файла/не разобрался — возвращаем null,
|
|
123
|
+
* и решение принимает возраст (fallback).
|
|
124
|
+
*/
|
|
125
|
+
export async function readLockOwnerPid(
|
|
126
|
+
lockPath: string,
|
|
127
|
+
isDir: boolean,
|
|
128
|
+
): Promise<number | null> {
|
|
129
|
+
const tokenPath = isDir ? path.join(lockPath, "pid") : lockPath;
|
|
130
|
+
const content = await fs.readFile(tokenPath, "utf8").catch(() => "");
|
|
131
|
+
return ownerPidFromToken(content);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
|
|
5
135
|
/**
|
|
6
136
|
* Менеджер файловых блокировок для предотвращения наложения запусков одной задачи.
|
|
7
137
|
*
|
|
@@ -54,12 +184,28 @@ export class LockManager {
|
|
|
54
184
|
// Артефакт предыдущего вытеснения валидным lock быть не может,
|
|
55
185
|
// поэтому убираем его независимо от возраста.
|
|
56
186
|
if (isStaleArtifact) {
|
|
57
|
-
await fs.rm(fullPath, { recursive: true, force: true }).catch(() =>
|
|
187
|
+
await fs.rm(fullPath, { recursive: true, force: true }).catch((err: unknown) =>
|
|
188
|
+
warnCleanupFailure(fullPath, err),
|
|
189
|
+
);
|
|
58
190
|
continue;
|
|
59
191
|
}
|
|
60
192
|
const stat = await fs.stat(fullPath);
|
|
61
193
|
const age = now - stat.mtimeMs;
|
|
62
194
|
if (age > STALE_LOCK_MAX_AGE_MS) {
|
|
195
|
+
// P2-18/B7: возраст — только повод проверить владельца. Задача может
|
|
196
|
+
// легально работать дольше 24 ч (timeoutSeconds: 0), поэтому живой
|
|
197
|
+
// процесс не вытесняем: иначе после рестарта плагина появится дубль.
|
|
198
|
+
// B7: для каталога `.lock.d` владелец лежит в `<dir>/pid`, а не в
|
|
199
|
+
// самом файле; без этого проверка смотрела только на `.lock`.
|
|
200
|
+
const ownerPid = await readLockOwnerPid(fullPath, isLockDir);
|
|
201
|
+
if (ownerPid !== null && isProcessAlive(ownerPid)) {
|
|
202
|
+
console.log(
|
|
203
|
+
`[agent-steward] Lock ${file} старше ${Math.round(
|
|
204
|
+
age / 1000,
|
|
205
|
+
)} с, но процесс ${ownerPid} жив — оставляем.`,
|
|
206
|
+
);
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
63
209
|
// T-03: не удаляем lock напрямую. Два воркера могли одновременно
|
|
64
210
|
// снести один и тот же файл, после чего один из них удалял свежий
|
|
65
211
|
// lock, созданный другим, и задача запускалась повторно. Атомарный
|
|
@@ -71,7 +217,9 @@ export class LockManager {
|
|
|
71
217
|
} catch {
|
|
72
218
|
continue;
|
|
73
219
|
}
|
|
74
|
-
await fs.rm(stalePath, { recursive: true, force: true }).catch(() =>
|
|
220
|
+
await fs.rm(stalePath, { recursive: true, force: true }).catch((err: unknown) =>
|
|
221
|
+
warnCleanupFailure(stalePath, err),
|
|
222
|
+
);
|
|
75
223
|
|
|
76
224
|
console.log(
|
|
77
225
|
`[agent-steward] Вытеснен зависший lock: ${file} (возраст ${Math.round(
|
|
@@ -87,36 +235,89 @@ export class LockManager {
|
|
|
87
235
|
|
|
88
236
|
/**
|
|
89
237
|
* Пытается захватить блокировку для задачи.
|
|
90
|
-
* Возвращает
|
|
91
|
-
*
|
|
238
|
+
* Возвращает уникальный токен владельца (он же содержимое lock-файла) или
|
|
239
|
+
* null, если блокировка уже удерживается другим запуском.
|
|
240
|
+
*
|
|
241
|
+
* Токен нужен release(): процесс, чей lock был вытеснен по stale-таймауту,
|
|
242
|
+
* не должен удалять файл нового владельца.
|
|
92
243
|
*/
|
|
93
|
-
async acquire(taskName: string): Promise<
|
|
244
|
+
async acquire(taskName: string): Promise<string | null> {
|
|
94
245
|
await fs.mkdir(this.lockDir, { recursive: true });
|
|
95
246
|
const lockPath = this.lockFilePath(taskName);
|
|
247
|
+
// B6: единый формат `<pid>-<iso>` (+ uuid для уникальности токена).
|
|
248
|
+
// PS-обёртка пишет тот же префикс, поэтому её stale-проверка видит PID,
|
|
249
|
+
// а не год из даты, и не вытесняет живой Node-лок.
|
|
250
|
+
const token = `${process.pid}-${new Date().toISOString()}-${randomUUID()}`;
|
|
96
251
|
|
|
97
252
|
try {
|
|
98
253
|
// 'wx' — exclusive create: ошибка если файл уже существует.
|
|
99
|
-
await fs.writeFile(lockPath,
|
|
100
|
-
return
|
|
101
|
-
} catch (err:
|
|
102
|
-
|
|
103
|
-
|
|
254
|
+
await fs.writeFile(lockPath, token, { flag: "wx" });
|
|
255
|
+
return token;
|
|
256
|
+
} catch (err: unknown) {
|
|
257
|
+
// P3-39: на Windows конкурентный create/delete того же lock-файла
|
|
258
|
+
// возвращает EPERM/EACCES/EBUSY вместо EEXIST. Это тот же случай
|
|
259
|
+
// «блокировка занята», а не ошибка прав: отдаём null, а withLockWait
|
|
260
|
+
// повторит попытку (withLock корректно пропустит запуск).
|
|
261
|
+
if (
|
|
262
|
+
(err as NodeJS.ErrnoException).code === "EEXIST" ||
|
|
263
|
+
(err as NodeJS.ErrnoException).code === "EPERM" ||
|
|
264
|
+
(err as NodeJS.ErrnoException).code === "EACCES" ||
|
|
265
|
+
(err as NodeJS.ErrnoException).code === "EBUSY"
|
|
266
|
+
) {
|
|
267
|
+
return null;
|
|
104
268
|
}
|
|
105
269
|
throw err;
|
|
106
270
|
}
|
|
107
271
|
}
|
|
108
272
|
|
|
273
|
+
/**
|
|
274
|
+
* Удаляет lock-файл, переживая кратковременные Windows-блокировки.
|
|
275
|
+
*
|
|
276
|
+
* На Windows unlink может вернуть EPERM/EACCES/EBUSY, пока другой процесс
|
|
277
|
+
* читает файл (проверка владельца в withLockWait) или файл ещё «висящий»
|
|
278
|
+
* после создания. Несколько коротких повторов решают гонку; ENOENT — уже
|
|
279
|
+
* удалён, это не ошибка.
|
|
280
|
+
*/
|
|
281
|
+
private async unlinkWithRetry(lockPath: string, attempts = 6): Promise<boolean> {
|
|
282
|
+
for (let i = 0; i < attempts; i += 1) {
|
|
283
|
+
try {
|
|
284
|
+
await fs.unlink(lockPath);
|
|
285
|
+
return true;
|
|
286
|
+
} catch (err: unknown) {
|
|
287
|
+
if ((err as NodeJS.ErrnoException).code === "ENOENT") return false;
|
|
288
|
+
const transient =
|
|
289
|
+
(err as NodeJS.ErrnoException).code === "EPERM" || (err as NodeJS.ErrnoException).code === "EACCES" || (err as NodeJS.ErrnoException).code === "EBUSY";
|
|
290
|
+
if (transient && i < attempts - 1) {
|
|
291
|
+
await new Promise((resolve) => setTimeout(resolve, 10 * (i + 1)));
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
294
|
+
throw err;
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
return false;
|
|
298
|
+
}
|
|
299
|
+
|
|
109
300
|
/**
|
|
110
301
|
* Освобождает блокировку задачи (удаляет lock-файл).
|
|
302
|
+
*
|
|
303
|
+
* Удаляет файл только если в нём лежит токен этого владельца. Если lock
|
|
304
|
+
* успели вытеснить и перехватить, release ничего не трогает и возвращает
|
|
305
|
+
* false — иначе старый процесс снёс бы чужую критическую секцию.
|
|
111
306
|
* Не выбрасывает ошибку, если файл уже удалён.
|
|
112
307
|
*/
|
|
113
|
-
async release(taskName: string): Promise<
|
|
308
|
+
async release(taskName: string, token: string): Promise<boolean> {
|
|
114
309
|
const lockPath = this.lockFilePath(taskName);
|
|
310
|
+
let current: string;
|
|
115
311
|
try {
|
|
116
|
-
await fs.
|
|
117
|
-
} catch (err:
|
|
118
|
-
if (err.code
|
|
312
|
+
current = await fs.readFile(lockPath, "utf8");
|
|
313
|
+
} catch (err: unknown) {
|
|
314
|
+
if ((err as NodeJS.ErrnoException).code === "ENOENT") return false;
|
|
315
|
+
throw err;
|
|
119
316
|
}
|
|
317
|
+
if (current !== token) return false;
|
|
318
|
+
// P3-39: на Windows unlink может кратковременно упасть EPERM, пока
|
|
319
|
+
// другие процессы читают lock для проверки владельца.
|
|
320
|
+
return this.unlinkWithRetry(lockPath);
|
|
120
321
|
}
|
|
121
322
|
|
|
122
323
|
/**
|
|
@@ -127,8 +328,8 @@ export class LockManager {
|
|
|
127
328
|
try {
|
|
128
329
|
await fs.stat(lockPath);
|
|
129
330
|
return true;
|
|
130
|
-
} catch (err:
|
|
131
|
-
if (err.code === "ENOENT") return false;
|
|
331
|
+
} catch (err: unknown) {
|
|
332
|
+
if ((err as NodeJS.ErrnoException).code === "ENOENT") return false;
|
|
132
333
|
throw err;
|
|
133
334
|
}
|
|
134
335
|
}
|
|
@@ -144,8 +345,8 @@ export async function withLock<T>(
|
|
|
144
345
|
taskName: string,
|
|
145
346
|
fn: () => Promise<T>,
|
|
146
347
|
): Promise<T | null> {
|
|
147
|
-
const
|
|
148
|
-
if (!
|
|
348
|
+
const token = await lockManager.acquire(taskName);
|
|
349
|
+
if (!token) {
|
|
149
350
|
|
|
150
351
|
console.log(
|
|
151
352
|
`[agent-steward] Запуск задачи "${taskName}" пропущен: удерживается активная блокировка.`,
|
|
@@ -156,7 +357,7 @@ export async function withLock<T>(
|
|
|
156
357
|
try {
|
|
157
358
|
return await fn();
|
|
158
359
|
} finally {
|
|
159
|
-
await lockManager.release(taskName);
|
|
360
|
+
await lockManager.release(taskName, token);
|
|
160
361
|
}
|
|
161
362
|
}
|
|
162
363
|
|
|
@@ -178,25 +379,38 @@ export async function withLockWait<T>(
|
|
|
178
379
|
const staleMs = options.staleMs ?? 60000;
|
|
179
380
|
const deadline = Date.now() + timeoutMs;
|
|
180
381
|
const lockPath = lockManager.pathFor(taskName);
|
|
382
|
+
let token: string | null = null;
|
|
181
383
|
|
|
182
384
|
for (;;) {
|
|
183
|
-
|
|
385
|
+
token = await lockManager.acquire(taskName);
|
|
386
|
+
if (token) break;
|
|
184
387
|
|
|
185
388
|
// Осиротевший lock: короткая операция не может длиться дольше staleMs.
|
|
186
389
|
if (staleMs > 0) {
|
|
187
390
|
try {
|
|
188
391
|
const stat = await fs.stat(lockPath);
|
|
189
392
|
if (Date.now() - stat.mtimeMs > staleMs) {
|
|
190
|
-
//
|
|
191
|
-
//
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
393
|
+
// P2-18: staleMs — не приговор. Если владелец ещё жив, его
|
|
394
|
+
// критическая секция может быть легально долгой (crontab RMW,
|
|
395
|
+
// длинная задача). Вытесняем только мёртвый/неизвестный lock.
|
|
396
|
+
const content = await fs.readFile(lockPath, "utf8").catch(() => "");
|
|
397
|
+
const ownerPid = ownerPidFromToken(content);
|
|
398
|
+
if (ownerPid !== null && isProcessAlive(ownerPid)) {
|
|
399
|
+
// Ждём дальше; по deadline вернём понятную ошибку.
|
|
400
|
+
} else {
|
|
401
|
+
// T-03-совместимое вытеснение: атомарный rename выигрывает ровно
|
|
402
|
+
// один процесс, остальные получают ENOENT и повторяют попытку.
|
|
403
|
+
const stalePath = `${lockPath}.stale-${process.pid}-${Date.now()}`;
|
|
404
|
+
try {
|
|
405
|
+
await fs.rename(lockPath, stalePath);
|
|
406
|
+
await fs.rm(stalePath, { force: true }).catch((err: unknown) =>
|
|
407
|
+
warnCleanupFailure(stalePath, err),
|
|
408
|
+
);
|
|
409
|
+
} catch {
|
|
410
|
+
// Кто-то успел раньше — просто повторяем acquire.
|
|
411
|
+
}
|
|
412
|
+
continue;
|
|
198
413
|
}
|
|
199
|
-
continue;
|
|
200
414
|
}
|
|
201
415
|
} catch {
|
|
202
416
|
// lock исчез — сразу пробуем acquire снова.
|
|
@@ -214,6 +428,6 @@ export async function withLockWait<T>(
|
|
|
214
428
|
try {
|
|
215
429
|
return await fn();
|
|
216
430
|
} finally {
|
|
217
|
-
await lockManager.release(taskName);
|
|
431
|
+
if (token) await lockManager.release(taskName, token);
|
|
218
432
|
}
|
|
219
433
|
}
|
package/src/core/matrix.ts
CHANGED
|
@@ -21,8 +21,37 @@ import { getHistory, recordRun } from "../db/history-repo.js";
|
|
|
21
21
|
import { notifyMatrixCompletion } from "../notifications.js";
|
|
22
22
|
import { getWorkspace } from "../workspaces.js";
|
|
23
23
|
import { createSchedulerBackend } from "./scheduler/index.js";
|
|
24
|
+
import { appendStructuredLog } from "./structured-log.js";
|
|
24
25
|
import { t } from "../i18n.js";
|
|
25
26
|
|
|
27
|
+
/**
|
|
28
|
+
* Матричные события идут в machine-readable runs.jsonl: человекочитаемый
|
|
29
|
+
* вывод планировщика не позволяет связать клон с родителем и запуском.
|
|
30
|
+
*/
|
|
31
|
+
function logMatrixEvent(
|
|
32
|
+
parentName: string,
|
|
33
|
+
step: string,
|
|
34
|
+
details: {
|
|
35
|
+
cloneName?: string;
|
|
36
|
+
status?: string;
|
|
37
|
+
reason?: string;
|
|
38
|
+
error?: string;
|
|
39
|
+
runId?: number;
|
|
40
|
+
} = {},
|
|
41
|
+
): void {
|
|
42
|
+
appendStructuredLog({
|
|
43
|
+
event: `matrix.${step}`,
|
|
44
|
+
task: parentName,
|
|
45
|
+
matrixId: parentName,
|
|
46
|
+
cloneName: details.cloneName,
|
|
47
|
+
step,
|
|
48
|
+
status: details.status,
|
|
49
|
+
reason: details.reason,
|
|
50
|
+
error: details.error,
|
|
51
|
+
runId: details.runId,
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
|
|
26
55
|
/** Суффикс агрегирующей задачи, запускаемой после всех клонов. */
|
|
27
56
|
export const MATRIX_COMPLETE_SUFFIX = "-matrix-complete";
|
|
28
57
|
|
|
@@ -314,9 +343,10 @@ export async function runMatrix(task: TaskDefinition): Promise<MatrixRunResult>
|
|
|
314
343
|
} else {
|
|
315
344
|
started.push(clone.name);
|
|
316
345
|
}
|
|
317
|
-
} catch (err:
|
|
346
|
+
} catch (err: unknown) {
|
|
318
347
|
const message = err instanceof Error ? err.message : String(err);
|
|
319
348
|
errors.push(`${clone.name}: ${message}`);
|
|
349
|
+
logMatrixEvent(task.name, "start_failed", { cloneName: clone.name, error: message });
|
|
320
350
|
if (failureMode === "stop") {
|
|
321
351
|
const rest = clones
|
|
322
352
|
.slice(clones.indexOf(clone) + 1)
|
|
@@ -332,19 +362,19 @@ export async function runMatrix(task: TaskDefinition): Promise<MatrixRunResult>
|
|
|
332
362
|
failureMode === "stop" && errors.length > 0 ? [] : queued;
|
|
333
363
|
try {
|
|
334
364
|
saveTask({ ...task, matrixQueue: queueToSave.length > 0 ? queueToSave : undefined });
|
|
335
|
-
} catch (err:
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
}`,
|
|
340
|
-
);
|
|
365
|
+
} catch (err: unknown) {
|
|
366
|
+
logMatrixEvent(task.name, "queue_save_failed", {
|
|
367
|
+
error: err instanceof Error ? err.message : String(err),
|
|
368
|
+
});
|
|
341
369
|
}
|
|
342
370
|
|
|
343
|
-
|
|
344
|
-
|
|
371
|
+
logMatrixEvent(task.name, "started", {
|
|
372
|
+
status: started.length > 0 ? "running" : "error",
|
|
373
|
+
reason:
|
|
374
|
+
`strategy=${strategy}, failureMode=${failureMode}, ` +
|
|
345
375
|
`concurrency=${concurrency}, started=${started.length}, skipped=${skipped.length}, ` +
|
|
346
376
|
`queued=${queueToSave.length}, errors=${errors.length}`,
|
|
347
|
-
);
|
|
377
|
+
});
|
|
348
378
|
|
|
349
379
|
return {
|
|
350
380
|
success: errors.length === 0 || failureMode === "continue",
|
|
@@ -387,7 +417,10 @@ export async function advanceSequentialMatrix(
|
|
|
387
417
|
if ((parent.matrix.failureMode ?? "continue") === "stop") {
|
|
388
418
|
const latest = latestCloneStatus(cloneName);
|
|
389
419
|
if (latest.status === "error") {
|
|
390
|
-
|
|
420
|
+
logMatrixEvent(parent.name, "sequential_stopped", {
|
|
421
|
+
cloneName,
|
|
422
|
+
status: "error",
|
|
423
|
+
});
|
|
391
424
|
return undefined;
|
|
392
425
|
}
|
|
393
426
|
}
|
|
@@ -398,11 +431,15 @@ export async function advanceSequentialMatrix(
|
|
|
398
431
|
try {
|
|
399
432
|
const result = await createSchedulerBackend().runTask(next.name);
|
|
400
433
|
if (result.alreadyRunning) return undefined;
|
|
401
|
-
|
|
434
|
+
logMatrixEvent(parent.name, "advanced", {
|
|
435
|
+
cloneName: next.name,
|
|
436
|
+
status: "running",
|
|
437
|
+
reason: "sequential",
|
|
438
|
+
});
|
|
402
439
|
return next.name;
|
|
403
|
-
} catch (err:
|
|
440
|
+
} catch (err: unknown) {
|
|
404
441
|
const message = err instanceof Error ? err.message : String(err);
|
|
405
|
-
|
|
442
|
+
logMatrixEvent(parent.name, "start_failed", { cloneName: next.name, error: message });
|
|
406
443
|
return undefined;
|
|
407
444
|
}
|
|
408
445
|
}
|
|
@@ -449,12 +486,16 @@ export async function advanceMatrixAfterClone(
|
|
|
449
486
|
started.push(nextName);
|
|
450
487
|
slots -= 1;
|
|
451
488
|
if (result.alreadyRunning) slots += 0; // слот уже занят этим клоном
|
|
452
|
-
|
|
453
|
-
|
|
489
|
+
logMatrixEvent(parent.name, "queue_started", {
|
|
490
|
+
cloneName: nextName,
|
|
491
|
+
status: "running",
|
|
492
|
+
});
|
|
493
|
+
} catch (err: unknown) {
|
|
454
494
|
const message = err instanceof Error ? err.message : String(err);
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
495
|
+
logMatrixEvent(parent.name, "queue_start_failed", {
|
|
496
|
+
cloneName: nextName,
|
|
497
|
+
error: message,
|
|
498
|
+
});
|
|
458
499
|
// Попытка запуска «съедает» слот: иначе при недоступном бэкенде
|
|
459
500
|
// цикл уйдёт по всей очереди, проигнорировав лимит.
|
|
460
501
|
slots -= 1;
|
|
@@ -467,12 +508,10 @@ export async function advanceMatrixAfterClone(
|
|
|
467
508
|
|
|
468
509
|
try {
|
|
469
510
|
saveTask({ ...parent, matrixQueue: queue.length > 0 ? queue : undefined });
|
|
470
|
-
} catch (err:
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
}`,
|
|
475
|
-
);
|
|
511
|
+
} catch (err: unknown) {
|
|
512
|
+
logMatrixEvent(parent.name, "queue_save_failed", {
|
|
513
|
+
error: err instanceof Error ? err.message : String(err),
|
|
514
|
+
});
|
|
476
515
|
}
|
|
477
516
|
|
|
478
517
|
return started;
|
|
@@ -789,12 +828,10 @@ export async function cancelPendingMatrixClones(
|
|
|
789
828
|
if (Array.isArray(parent.matrixQueue) && parent.matrixQueue.length > 0) {
|
|
790
829
|
try {
|
|
791
830
|
saveTask({ ...parent, matrixQueue: undefined });
|
|
792
|
-
} catch (err:
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
}`,
|
|
797
|
-
);
|
|
831
|
+
} catch (err: unknown) {
|
|
832
|
+
logMatrixEvent(parent.name, "queue_clear_failed", {
|
|
833
|
+
error: err instanceof Error ? err.message : String(err),
|
|
834
|
+
});
|
|
798
835
|
}
|
|
799
836
|
}
|
|
800
837
|
|
|
@@ -827,12 +864,12 @@ export async function cancelPendingMatrixClones(
|
|
|
827
864
|
"skipped",
|
|
828
865
|
);
|
|
829
866
|
cancelled.push(clone.name);
|
|
830
|
-
} catch (err:
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
);
|
|
867
|
+
} catch (err: unknown) {
|
|
868
|
+
logMatrixEvent(parent.name, "cancelled", {
|
|
869
|
+
cloneName: clone.name,
|
|
870
|
+
status: "error",
|
|
871
|
+
error: err instanceof Error ? err.message : String(err),
|
|
872
|
+
});
|
|
836
873
|
}
|
|
837
874
|
}
|
|
838
875
|
|
|
@@ -919,12 +956,18 @@ export async function finalizeMatrixIfComplete(
|
|
|
919
956
|
error: aggregate.errorCount,
|
|
920
957
|
message: aggregate.message,
|
|
921
958
|
});
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
959
|
+
const aggregateRunId = getHistory(parent.name, 1)[0]?.id;
|
|
960
|
+
logMatrixEvent(parent.name, "aggregated", {
|
|
961
|
+
status: finalStatus,
|
|
962
|
+
runId: aggregateRunId,
|
|
963
|
+
reason:
|
|
964
|
+
`total=${aggregate.totalCount}, success=${aggregate.successCount}, ` +
|
|
965
|
+
`error=${aggregate.errorCount}`,
|
|
966
|
+
});
|
|
967
|
+
} catch (err: unknown) {
|
|
968
|
+
logMatrixEvent(parent.name, "aggregate_failed", {
|
|
969
|
+
error: err instanceof Error ? err.message : String(err),
|
|
970
|
+
});
|
|
928
971
|
}
|
|
929
972
|
|
|
930
973
|
if (parent.matrix.createAggregator) {
|
|
@@ -932,13 +975,15 @@ export async function finalizeMatrixIfComplete(
|
|
|
932
975
|
if (aggregator) {
|
|
933
976
|
try {
|
|
934
977
|
await createSchedulerBackend().runTask(aggregator.name);
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
978
|
+
logMatrixEvent(parent.name, "aggregator_started", {
|
|
979
|
+
cloneName: aggregator.name,
|
|
980
|
+
status: "running",
|
|
981
|
+
});
|
|
982
|
+
} catch (err: unknown) {
|
|
983
|
+
logMatrixEvent(parent.name, "aggregator_start_failed", {
|
|
984
|
+
cloneName: aggregator.name,
|
|
985
|
+
error: err instanceof Error ? err.message : String(err),
|
|
986
|
+
});
|
|
942
987
|
}
|
|
943
988
|
}
|
|
944
989
|
}
|