agent-steward 0.0.4 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +134 -2
- package/CROSS_PLATFORM.md +19 -0
- package/README.md +52 -7
- package/ROADMAP-STRATEGY.md +21 -18
- package/ROADMAP.md +140 -159
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +351 -0
- package/dist/core/execution.d.ts +203 -0
- package/dist/core/execution.d.ts.map +1 -0
- package/dist/core/execution.js +309 -0
- package/dist/core/gate-integrity.d.ts +35 -0
- package/dist/core/gate-integrity.d.ts.map +1 -0
- package/dist/core/gate-integrity.js +132 -0
- package/dist/core/git-trigger.d.ts.map +1 -1
- package/dist/core/git-trigger.js +14 -2
- package/dist/core/guard-run.d.ts +4 -0
- package/dist/core/guard-run.d.ts.map +1 -0
- package/dist/core/guard-run.js +101 -0
- package/dist/core/guard.d.ts +52 -0
- package/dist/core/guard.d.ts.map +1 -0
- package/dist/core/guard.js +362 -0
- package/dist/core/lock.d.ts +41 -0
- package/dist/core/lock.d.ts.map +1 -1
- package/dist/core/lock.js +141 -29
- package/dist/core/matrix.d.ts.map +1 -1
- package/dist/core/matrix.js +75 -15
- package/dist/core/mission.d.ts +83 -0
- package/dist/core/mission.d.ts.map +1 -0
- package/dist/core/mission.js +334 -0
- package/dist/core/pipeline-apply.d.ts +38 -0
- package/dist/core/pipeline-apply.d.ts.map +1 -0
- package/dist/core/pipeline-apply.js +117 -0
- package/dist/core/pipeline-io.d.ts +13 -0
- package/dist/core/pipeline-io.d.ts.map +1 -0
- package/dist/core/pipeline-io.js +64 -0
- package/dist/core/pipeline-map.d.ts +46 -0
- package/dist/core/pipeline-map.d.ts.map +1 -0
- package/dist/core/pipeline-map.js +196 -0
- package/dist/core/pipeline-spec.d.ts +109 -0
- package/dist/core/pipeline-spec.d.ts.map +1 -0
- package/dist/core/pipeline-spec.js +554 -0
- package/dist/core/ps-wrapper.d.ts +19 -2
- package/dist/core/ps-wrapper.d.ts.map +1 -1
- package/dist/core/ps-wrapper.js +250 -62
- package/dist/core/record-history.d.ts.map +1 -1
- package/dist/core/record-history.js +130 -29
- package/dist/core/router/rules.d.ts.map +1 -1
- package/dist/core/router/rules.js +6 -0
- package/dist/core/scheduler/linux.d.ts +8 -1
- package/dist/core/scheduler/linux.d.ts.map +1 -1
- package/dist/core/scheduler/linux.js +34 -9
- package/dist/core/scheduler/mac.d.ts +44 -2
- package/dist/core/scheduler/mac.d.ts.map +1 -1
- package/dist/core/scheduler/mac.js +126 -15
- package/dist/core/scheduler/windows.d.ts.map +1 -1
- package/dist/core/scheduler/windows.js +15 -12
- package/dist/core/structured-log.d.ts +7 -1
- package/dist/core/structured-log.d.ts.map +1 -1
- package/dist/core/worktree-run.d.ts +4 -0
- package/dist/core/worktree-run.d.ts.map +1 -0
- package/dist/core/worktree-run.js +164 -0
- package/dist/core/worktree.d.ts +115 -0
- package/dist/core/worktree.d.ts.map +1 -0
- package/dist/core/worktree.js +462 -0
- package/dist/core/wrappers/bash.d.ts +17 -0
- package/dist/core/wrappers/bash.d.ts.map +1 -1
- package/dist/core/wrappers/bash.js +175 -24
- package/dist/db/capability-repo.d.ts +43 -2
- package/dist/db/capability-repo.d.ts.map +1 -1
- package/dist/db/capability-repo.js +96 -11
- package/dist/db/history-repo.d.ts +38 -5
- package/dist/db/history-repo.d.ts.map +1 -1
- package/dist/db/history-repo.js +88 -25
- package/dist/db/metrics-repo.js +2 -2
- package/dist/db/recovery.d.ts.map +1 -1
- package/dist/db/recovery.js +5 -4
- package/dist/db/run-journal.d.ts +33 -5
- package/dist/db/run-journal.d.ts.map +1 -1
- package/dist/db/run-journal.js +78 -14
- package/dist/db/run-resume.d.ts +5 -0
- package/dist/db/run-resume.d.ts.map +1 -1
- package/dist/db/run-resume.js +9 -2
- package/dist/db/runs-repo.d.ts +19 -2
- package/dist/db/runs-repo.d.ts.map +1 -1
- package/dist/db/runs-repo.js +22 -6
- package/dist/db/schema.d.ts +2 -2
- package/dist/db/schema.d.ts.map +1 -1
- package/dist/db/schema.js +63 -2
- package/dist/history.d.ts +1 -1
- package/dist/history.d.ts.map +1 -1
- package/dist/i18n.d.ts +5 -0
- package/dist/i18n.d.ts.map +1 -1
- package/dist/i18n.js +11 -4
- package/dist/index.d.ts +136 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +378 -14
- package/dist/notifications.d.ts +22 -0
- package/dist/notifications.d.ts.map +1 -1
- package/dist/notifications.js +36 -8
- package/dist/profiles.d.ts.map +1 -1
- package/dist/profiles.js +25 -4
- package/dist/rpc.d.ts +246 -0
- package/dist/rpc.d.ts.map +1 -1
- package/dist/rpc.js +117 -0
- package/dist/server.d.ts +15 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +77 -18
- package/dist/settings.d.ts +2 -2
- package/dist/settings.d.ts.map +1 -1
- package/dist/settings.js +3 -3
- package/dist/shared/jobs-view.d.ts +1 -1
- package/dist/shared/jobs-view.d.ts.map +1 -1
- package/dist/shared/jobs-view.js +2 -0
- package/dist/task-transfer.d.ts +16 -0
- package/dist/task-transfer.d.ts.map +1 -1
- package/dist/task-transfer.js +16 -0
- package/docs/ARCHITECTURE.md +281 -65
- package/docs/COMPATIBILITY.md +2 -2
- package/docs/CONCEPT.md +18 -4
- package/docs/CROSS_PLATFORM.ru.md +20 -0
- package/docs/CROSS_PLATFORM.zh.md +18 -0
- package/docs/EXECUTION-LOOP.md +22 -7
- package/docs/FEATURES.md +65 -4
- package/docs/FEATURES.zh.md +52 -4
- package/docs/MIGRATION.md +2 -2
- package/docs/PIPELINE.md +293 -0
- package/docs/README.ru.md +42 -3
- package/docs/README.zh.md +43 -3
- package/docs/SAFETY.md +23 -3
- package/gui-experimental.cjs +9 -8
- package/locales/en.json +17 -1
- package/locales/ru.json +17 -1
- package/locales/zh.json +17 -1
- package/opencode.pipeline.yaml +35 -0
- package/package.json +23 -4
- package/scripts/test-home.mjs +83 -0
- package/src/cli.ts +412 -0
- package/src/core/dry-run.ts +2 -2
- package/src/core/execution.ts +538 -0
- package/src/core/file-watcher.ts +2 -2
- package/src/core/gate-integrity.ts +160 -0
- package/src/core/git-trigger.ts +18 -5
- package/src/core/guard-run.ts +119 -0
- package/src/core/guard.ts +431 -0
- package/src/core/lock.ts +178 -37
- package/src/core/matrix.ts +95 -50
- package/src/core/mission.ts +485 -0
- package/src/core/native-notify.ts +1 -1
- package/src/core/pipeline-apply.ts +158 -0
- package/src/core/pipeline-io.ts +79 -0
- package/src/core/pipeline-map.ts +264 -0
- package/src/core/pipeline-spec.ts +775 -0
- package/src/core/ps-wrapper.ts +263 -63
- package/src/core/record-history.ts +164 -77
- package/src/core/router/rules.ts +6 -0
- package/src/core/schedule-gate.ts +1 -1
- package/src/core/scheduler/at-fallback.ts +1 -1
- package/src/core/scheduler/linux.ts +40 -11
- package/src/core/scheduler/logs.ts +2 -2
- package/src/core/scheduler/mac.ts +174 -14
- package/src/core/scheduler/registry.ts +2 -2
- package/src/core/scheduler/windows.ts +19 -15
- package/src/core/secrets.ts +8 -8
- package/src/core/structured-log.ts +22 -5
- package/src/core/telegram.ts +2 -2
- package/src/core/triage-worker.ts +7 -7
- package/src/core/triage.ts +2 -2
- package/src/core/worktree-run.ts +199 -0
- package/src/core/worktree.ts +600 -0
- package/src/core/wrappers/bash.ts +190 -23
- package/src/db/capability-repo.ts +127 -10
- package/src/db/history-repo.ts +129 -22
- package/src/db/metrics-repo.ts +2 -2
- package/src/db/recovery.ts +5 -4
- package/src/db/run-journal.ts +100 -13
- package/src/db/run-resume.ts +17 -1
- package/src/db/runs-repo.ts +36 -6
- package/src/db/schema.ts +65 -2
- package/src/gui.tsx +177 -13
- package/src/history.ts +1 -1
- package/src/i18n.ts +12 -4
- package/src/index.ts +524 -4
- package/src/notifications.ts +38 -8
- package/src/profiles.ts +26 -4
- package/src/rpc.ts +128 -0
- package/src/server.ts +94 -18
- package/src/settings.ts +5 -5
- package/src/shared/jobs-view.ts +3 -1
- package/src/task-transfer.ts +31 -0
package/src/core/lock.ts
CHANGED
|
@@ -3,6 +3,17 @@ import path from "node:path";
|
|
|
3
3
|
import { randomUUID } from "node:crypto";
|
|
4
4
|
import { LOCK_DIR, STALE_LOCK_MAX_AGE_MS } from "../types.js";
|
|
5
5
|
|
|
6
|
+
/**
|
|
7
|
+
* O-2: ошибки очистки больше не проглатываются молча. Прямой импорт
|
|
8
|
+
* `structured-log.ts` создал бы цикл (structured-log → secrets → lock), поэтому
|
|
9
|
+
* здесь используется `console.warn` — предупреждение всё равно видно в логе
|
|
10
|
+
* процесса и не ломает основной сценарий.
|
|
11
|
+
*/
|
|
12
|
+
function warnCleanupFailure(target: string, error: unknown): void {
|
|
13
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
14
|
+
console.warn(`[agent-steward] Не удалось удалить lock-артефакт «${target}»: ${reason}`);
|
|
15
|
+
}
|
|
16
|
+
|
|
6
17
|
/**
|
|
7
18
|
* Проверяет, жив ли процесс-владелец lock-файла.
|
|
8
19
|
*
|
|
@@ -16,18 +27,108 @@ export function isProcessAlive(pid: number): boolean {
|
|
|
16
27
|
try {
|
|
17
28
|
process.kill(pid, 0);
|
|
18
29
|
return true;
|
|
19
|
-
} catch (err:
|
|
20
|
-
if (err?.code === "ESRCH") return false;
|
|
30
|
+
} catch (err: unknown) {
|
|
31
|
+
if ((err as NodeJS.ErrnoException | undefined)?.code === "ESRCH") return false;
|
|
21
32
|
return true;
|
|
22
33
|
}
|
|
23
34
|
}
|
|
24
35
|
|
|
25
|
-
/** PID владельца
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
36
|
+
/** Разобранный lock-токен: PID владельца и формат записи. */
|
|
37
|
+
export interface LockToken {
|
|
38
|
+
pid: number | null;
|
|
39
|
+
createdAt: string | null;
|
|
40
|
+
format: "unified" | "node-legacy" | "ps-legacy" | "pid" | "unknown";
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** Достаёт ISO-время из первой части legacy PS-токена (`ISO|PID`). */
|
|
44
|
+
function parseIsoOrNull(value: string | undefined): string | null {
|
|
45
|
+
const text = String(value ?? "").trim();
|
|
46
|
+
return /^\d{4}-\d{2}-\d{2}T/.test(text) ? text : null;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Разбирает содержимое lock-файла в едином для Node/PS/bash виде.
|
|
51
|
+
*
|
|
52
|
+
* Поддерживаемые форматы:
|
|
53
|
+
* `<pid>-<iso>` — единый формат (Node и PS, B6);
|
|
54
|
+
* `<pid>-<epoch>-<uuid>` — старый Node-токен;
|
|
55
|
+
* `<iso>|<pid>` — старый PS-токен (ISO|PID);
|
|
56
|
+
* `<pid>` — голый PID (pid-файл bash-обёртки).
|
|
57
|
+
*
|
|
58
|
+
* Порядок важен: legacy PS проверяется до `<pid>-…`, иначе год из ISO
|
|
59
|
+
* (`2026`) принимался за PID — именно это ломало межпроцессную совместимость.
|
|
60
|
+
*/
|
|
61
|
+
export function parseLockToken(content: string): LockToken {
|
|
62
|
+
const text = String(content ?? "").trim();
|
|
63
|
+
if (!text) return { pid: null, createdAt: null, format: "unknown" };
|
|
64
|
+
|
|
65
|
+
const withPipe = text.indexOf("|");
|
|
66
|
+
if (withPipe !== -1) {
|
|
67
|
+
const pid = Number.parseInt(text.slice(withPipe + 1).trim(), 10);
|
|
68
|
+
return {
|
|
69
|
+
pid: Number.isInteger(pid) && pid > 0 ? pid : null,
|
|
70
|
+
createdAt: parseIsoOrNull(text.slice(0, withPipe)),
|
|
71
|
+
format: "ps-legacy",
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const unified = /^(\d+)-(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)(?:-.*)?$/i.exec(text);
|
|
76
|
+
if (unified) {
|
|
77
|
+
const pid = Number.parseInt(unified[1], 10);
|
|
78
|
+
return {
|
|
79
|
+
pid: Number.isInteger(pid) && pid > 0 ? pid : null,
|
|
80
|
+
createdAt: unified[2],
|
|
81
|
+
format: "unified",
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// Старый Node мог добавить любой суффикс после epoch (`-live-owner`, uuid).
|
|
86
|
+
const legacyNode = /^(\d+)-(\d{10,})(?:-.*)?$/i.exec(text);
|
|
87
|
+
if (legacyNode) {
|
|
88
|
+
const pid = Number.parseInt(legacyNode[1], 10);
|
|
89
|
+
return {
|
|
90
|
+
pid: Number.isInteger(pid) && pid > 0 ? pid : null,
|
|
91
|
+
createdAt: null,
|
|
92
|
+
format: "node-legacy",
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
if (/^\d+$/.test(text)) {
|
|
97
|
+
const pid = Number.parseInt(text, 10);
|
|
98
|
+
// Голое число — PID (pid-файл bash), но не epoch из старого тестового
|
|
99
|
+
// lock: 1.7e12 не может быть PID и раньше не распознавалось владельцем.
|
|
100
|
+
return {
|
|
101
|
+
pid: Number.isInteger(pid) && pid > 0 && pid <= 2_147_483_647 ? pid : null,
|
|
102
|
+
createdAt: null,
|
|
103
|
+
format: "pid",
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
return { pid: null, createdAt: null, format: "unknown" };
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** PID владельца из токена lock-файла (любой поддерживаемый формат). */
|
|
111
|
+
export function ownerPidFromToken(content: string): number | null {
|
|
112
|
+
return parseLockToken(content).pid;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* PID владельца lock-артефакта.
|
|
117
|
+
*
|
|
118
|
+
* B7: файловый lock Node/PS хранит токен в самом файле, а POSIX-обёртка
|
|
119
|
+
* держит каталог `<lock>.d` и пишет `<pid>-<iso>` в `<lock>.d/pid`.
|
|
120
|
+
* Раньше каталог вытеснялся только по возрасту, без чтения этого файла,
|
|
121
|
+
* поэтому легально долгая задача (> 24 ч) после рестарта плагина
|
|
122
|
+
* дублировалась. Нет pid-файла/не разобрался — возвращаем null,
|
|
123
|
+
* и решение принимает возраст (fallback).
|
|
124
|
+
*/
|
|
125
|
+
export async function readLockOwnerPid(
|
|
126
|
+
lockPath: string,
|
|
127
|
+
isDir: boolean,
|
|
128
|
+
): Promise<number | null> {
|
|
129
|
+
const tokenPath = isDir ? path.join(lockPath, "pid") : lockPath;
|
|
130
|
+
const content = await fs.readFile(tokenPath, "utf8").catch(() => "");
|
|
131
|
+
return ownerPidFromToken(content);
|
|
31
132
|
}
|
|
32
133
|
|
|
33
134
|
|
|
@@ -83,26 +184,27 @@ export class LockManager {
|
|
|
83
184
|
// Артефакт предыдущего вытеснения валидным lock быть не может,
|
|
84
185
|
// поэтому убираем его независимо от возраста.
|
|
85
186
|
if (isStaleArtifact) {
|
|
86
|
-
await fs.rm(fullPath, { recursive: true, force: true }).catch(() =>
|
|
187
|
+
await fs.rm(fullPath, { recursive: true, force: true }).catch((err: unknown) =>
|
|
188
|
+
warnCleanupFailure(fullPath, err),
|
|
189
|
+
);
|
|
87
190
|
continue;
|
|
88
191
|
}
|
|
89
192
|
const stat = await fs.stat(fullPath);
|
|
90
193
|
const age = now - stat.mtimeMs;
|
|
91
194
|
if (age > STALE_LOCK_MAX_AGE_MS) {
|
|
92
|
-
// P2-18: возраст — только повод проверить владельца. Задача может
|
|
195
|
+
// P2-18/B7: возраст — только повод проверить владельца. Задача может
|
|
93
196
|
// легально работать дольше 24 ч (timeoutSeconds: 0), поэтому живой
|
|
94
197
|
// процесс не вытесняем: иначе после рестарта плагина появится дубль.
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
}
|
|
198
|
+
// B7: для каталога `.lock.d` владелец лежит в `<dir>/pid`, а не в
|
|
199
|
+
// самом файле; без этого проверка смотрела только на `.lock`.
|
|
200
|
+
const ownerPid = await readLockOwnerPid(fullPath, isLockDir);
|
|
201
|
+
if (ownerPid !== null && isProcessAlive(ownerPid)) {
|
|
202
|
+
console.log(
|
|
203
|
+
`[agent-steward] Lock ${file} старше ${Math.round(
|
|
204
|
+
age / 1000,
|
|
205
|
+
)} с, но процесс ${ownerPid} жив — оставляем.`,
|
|
206
|
+
);
|
|
207
|
+
continue;
|
|
106
208
|
}
|
|
107
209
|
// T-03: не удаляем lock напрямую. Два воркера могли одновременно
|
|
108
210
|
// снести один и тот же файл, после чего один из них удалял свежий
|
|
@@ -115,7 +217,9 @@ export class LockManager {
|
|
|
115
217
|
} catch {
|
|
116
218
|
continue;
|
|
117
219
|
}
|
|
118
|
-
await fs.rm(stalePath, { recursive: true, force: true }).catch(() =>
|
|
220
|
+
await fs.rm(stalePath, { recursive: true, force: true }).catch((err: unknown) =>
|
|
221
|
+
warnCleanupFailure(stalePath, err),
|
|
222
|
+
);
|
|
119
223
|
|
|
120
224
|
console.log(
|
|
121
225
|
`[agent-steward] Вытеснен зависший lock: ${file} (возраст ${Math.round(
|
|
@@ -140,20 +244,59 @@ export class LockManager {
|
|
|
140
244
|
async acquire(taskName: string): Promise<string | null> {
|
|
141
245
|
await fs.mkdir(this.lockDir, { recursive: true });
|
|
142
246
|
const lockPath = this.lockFilePath(taskName);
|
|
143
|
-
|
|
247
|
+
// B6: единый формат `<pid>-<iso>` (+ uuid для уникальности токена).
|
|
248
|
+
// PS-обёртка пишет тот же префикс, поэтому её stale-проверка видит PID,
|
|
249
|
+
// а не год из даты, и не вытесняет живой Node-лок.
|
|
250
|
+
const token = `${process.pid}-${new Date().toISOString()}-${randomUUID()}`;
|
|
144
251
|
|
|
145
252
|
try {
|
|
146
253
|
// 'wx' — exclusive create: ошибка если файл уже существует.
|
|
147
254
|
await fs.writeFile(lockPath, token, { flag: "wx" });
|
|
148
255
|
return token;
|
|
149
|
-
} catch (err:
|
|
150
|
-
|
|
256
|
+
} catch (err: unknown) {
|
|
257
|
+
// P3-39: на Windows конкурентный create/delete того же lock-файла
|
|
258
|
+
// возвращает EPERM/EACCES/EBUSY вместо EEXIST. Это тот же случай
|
|
259
|
+
// «блокировка занята», а не ошибка прав: отдаём null, а withLockWait
|
|
260
|
+
// повторит попытку (withLock корректно пропустит запуск).
|
|
261
|
+
if (
|
|
262
|
+
(err as NodeJS.ErrnoException).code === "EEXIST" ||
|
|
263
|
+
(err as NodeJS.ErrnoException).code === "EPERM" ||
|
|
264
|
+
(err as NodeJS.ErrnoException).code === "EACCES" ||
|
|
265
|
+
(err as NodeJS.ErrnoException).code === "EBUSY"
|
|
266
|
+
) {
|
|
151
267
|
return null;
|
|
152
268
|
}
|
|
153
269
|
throw err;
|
|
154
270
|
}
|
|
155
271
|
}
|
|
156
272
|
|
|
273
|
+
/**
|
|
274
|
+
* Удаляет lock-файл, переживая кратковременные Windows-блокировки.
|
|
275
|
+
*
|
|
276
|
+
* На Windows unlink может вернуть EPERM/EACCES/EBUSY, пока другой процесс
|
|
277
|
+
* читает файл (проверка владельца в withLockWait) или файл ещё «висящий»
|
|
278
|
+
* после создания. Несколько коротких повторов решают гонку; ENOENT — уже
|
|
279
|
+
* удалён, это не ошибка.
|
|
280
|
+
*/
|
|
281
|
+
private async unlinkWithRetry(lockPath: string, attempts = 6): Promise<boolean> {
|
|
282
|
+
for (let i = 0; i < attempts; i += 1) {
|
|
283
|
+
try {
|
|
284
|
+
await fs.unlink(lockPath);
|
|
285
|
+
return true;
|
|
286
|
+
} catch (err: unknown) {
|
|
287
|
+
if ((err as NodeJS.ErrnoException).code === "ENOENT") return false;
|
|
288
|
+
const transient =
|
|
289
|
+
(err as NodeJS.ErrnoException).code === "EPERM" || (err as NodeJS.ErrnoException).code === "EACCES" || (err as NodeJS.ErrnoException).code === "EBUSY";
|
|
290
|
+
if (transient && i < attempts - 1) {
|
|
291
|
+
await new Promise((resolve) => setTimeout(resolve, 10 * (i + 1)));
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
294
|
+
throw err;
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
return false;
|
|
298
|
+
}
|
|
299
|
+
|
|
157
300
|
/**
|
|
158
301
|
* Освобождает блокировку задачи (удаляет lock-файл).
|
|
159
302
|
*
|
|
@@ -167,18 +310,14 @@ export class LockManager {
|
|
|
167
310
|
let current: string;
|
|
168
311
|
try {
|
|
169
312
|
current = await fs.readFile(lockPath, "utf8");
|
|
170
|
-
} catch (err:
|
|
171
|
-
if (err.code === "ENOENT") return false;
|
|
313
|
+
} catch (err: unknown) {
|
|
314
|
+
if ((err as NodeJS.ErrnoException).code === "ENOENT") return false;
|
|
172
315
|
throw err;
|
|
173
316
|
}
|
|
174
317
|
if (current !== token) return false;
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
} catch (err: any) {
|
|
179
|
-
if (err.code === "ENOENT") return false;
|
|
180
|
-
throw err;
|
|
181
|
-
}
|
|
318
|
+
// P3-39: на Windows unlink может кратковременно упасть EPERM, пока
|
|
319
|
+
// другие процессы читают lock для проверки владельца.
|
|
320
|
+
return this.unlinkWithRetry(lockPath);
|
|
182
321
|
}
|
|
183
322
|
|
|
184
323
|
/**
|
|
@@ -189,8 +328,8 @@ export class LockManager {
|
|
|
189
328
|
try {
|
|
190
329
|
await fs.stat(lockPath);
|
|
191
330
|
return true;
|
|
192
|
-
} catch (err:
|
|
193
|
-
if (err.code === "ENOENT") return false;
|
|
331
|
+
} catch (err: unknown) {
|
|
332
|
+
if ((err as NodeJS.ErrnoException).code === "ENOENT") return false;
|
|
194
333
|
throw err;
|
|
195
334
|
}
|
|
196
335
|
}
|
|
@@ -264,7 +403,9 @@ export async function withLockWait<T>(
|
|
|
264
403
|
const stalePath = `${lockPath}.stale-${process.pid}-${Date.now()}`;
|
|
265
404
|
try {
|
|
266
405
|
await fs.rename(lockPath, stalePath);
|
|
267
|
-
await fs.rm(stalePath, { force: true }).catch(() =>
|
|
406
|
+
await fs.rm(stalePath, { force: true }).catch((err: unknown) =>
|
|
407
|
+
warnCleanupFailure(stalePath, err),
|
|
408
|
+
);
|
|
268
409
|
} catch {
|
|
269
410
|
// Кто-то успел раньше — просто повторяем acquire.
|
|
270
411
|
}
|
package/src/core/matrix.ts
CHANGED
|
@@ -21,8 +21,37 @@ import { getHistory, recordRun } from "../db/history-repo.js";
|
|
|
21
21
|
import { notifyMatrixCompletion } from "../notifications.js";
|
|
22
22
|
import { getWorkspace } from "../workspaces.js";
|
|
23
23
|
import { createSchedulerBackend } from "./scheduler/index.js";
|
|
24
|
+
import { appendStructuredLog } from "./structured-log.js";
|
|
24
25
|
import { t } from "../i18n.js";
|
|
25
26
|
|
|
27
|
+
/**
|
|
28
|
+
* Матричные события идут в machine-readable runs.jsonl: человекочитаемый
|
|
29
|
+
* вывод планировщика не позволяет связать клон с родителем и запуском.
|
|
30
|
+
*/
|
|
31
|
+
function logMatrixEvent(
|
|
32
|
+
parentName: string,
|
|
33
|
+
step: string,
|
|
34
|
+
details: {
|
|
35
|
+
cloneName?: string;
|
|
36
|
+
status?: string;
|
|
37
|
+
reason?: string;
|
|
38
|
+
error?: string;
|
|
39
|
+
runId?: number;
|
|
40
|
+
} = {},
|
|
41
|
+
): void {
|
|
42
|
+
appendStructuredLog({
|
|
43
|
+
event: `matrix.${step}`,
|
|
44
|
+
task: parentName,
|
|
45
|
+
matrixId: parentName,
|
|
46
|
+
cloneName: details.cloneName,
|
|
47
|
+
step,
|
|
48
|
+
status: details.status,
|
|
49
|
+
reason: details.reason,
|
|
50
|
+
error: details.error,
|
|
51
|
+
runId: details.runId,
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
|
|
26
55
|
/** Суффикс агрегирующей задачи, запускаемой после всех клонов. */
|
|
27
56
|
export const MATRIX_COMPLETE_SUFFIX = "-matrix-complete";
|
|
28
57
|
|
|
@@ -314,9 +343,10 @@ export async function runMatrix(task: TaskDefinition): Promise<MatrixRunResult>
|
|
|
314
343
|
} else {
|
|
315
344
|
started.push(clone.name);
|
|
316
345
|
}
|
|
317
|
-
} catch (err:
|
|
346
|
+
} catch (err: unknown) {
|
|
318
347
|
const message = err instanceof Error ? err.message : String(err);
|
|
319
348
|
errors.push(`${clone.name}: ${message}`);
|
|
349
|
+
logMatrixEvent(task.name, "start_failed", { cloneName: clone.name, error: message });
|
|
320
350
|
if (failureMode === "stop") {
|
|
321
351
|
const rest = clones
|
|
322
352
|
.slice(clones.indexOf(clone) + 1)
|
|
@@ -332,19 +362,19 @@ export async function runMatrix(task: TaskDefinition): Promise<MatrixRunResult>
|
|
|
332
362
|
failureMode === "stop" && errors.length > 0 ? [] : queued;
|
|
333
363
|
try {
|
|
334
364
|
saveTask({ ...task, matrixQueue: queueToSave.length > 0 ? queueToSave : undefined });
|
|
335
|
-
} catch (err:
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
}`,
|
|
340
|
-
);
|
|
365
|
+
} catch (err: unknown) {
|
|
366
|
+
logMatrixEvent(task.name, "queue_save_failed", {
|
|
367
|
+
error: err instanceof Error ? err.message : String(err),
|
|
368
|
+
});
|
|
341
369
|
}
|
|
342
370
|
|
|
343
|
-
|
|
344
|
-
|
|
371
|
+
logMatrixEvent(task.name, "started", {
|
|
372
|
+
status: started.length > 0 ? "running" : "error",
|
|
373
|
+
reason:
|
|
374
|
+
`strategy=${strategy}, failureMode=${failureMode}, ` +
|
|
345
375
|
`concurrency=${concurrency}, started=${started.length}, skipped=${skipped.length}, ` +
|
|
346
376
|
`queued=${queueToSave.length}, errors=${errors.length}`,
|
|
347
|
-
);
|
|
377
|
+
});
|
|
348
378
|
|
|
349
379
|
return {
|
|
350
380
|
success: errors.length === 0 || failureMode === "continue",
|
|
@@ -387,7 +417,10 @@ export async function advanceSequentialMatrix(
|
|
|
387
417
|
if ((parent.matrix.failureMode ?? "continue") === "stop") {
|
|
388
418
|
const latest = latestCloneStatus(cloneName);
|
|
389
419
|
if (latest.status === "error") {
|
|
390
|
-
|
|
420
|
+
logMatrixEvent(parent.name, "sequential_stopped", {
|
|
421
|
+
cloneName,
|
|
422
|
+
status: "error",
|
|
423
|
+
});
|
|
391
424
|
return undefined;
|
|
392
425
|
}
|
|
393
426
|
}
|
|
@@ -398,11 +431,15 @@ export async function advanceSequentialMatrix(
|
|
|
398
431
|
try {
|
|
399
432
|
const result = await createSchedulerBackend().runTask(next.name);
|
|
400
433
|
if (result.alreadyRunning) return undefined;
|
|
401
|
-
|
|
434
|
+
logMatrixEvent(parent.name, "advanced", {
|
|
435
|
+
cloneName: next.name,
|
|
436
|
+
status: "running",
|
|
437
|
+
reason: "sequential",
|
|
438
|
+
});
|
|
402
439
|
return next.name;
|
|
403
|
-
} catch (err:
|
|
440
|
+
} catch (err: unknown) {
|
|
404
441
|
const message = err instanceof Error ? err.message : String(err);
|
|
405
|
-
|
|
442
|
+
logMatrixEvent(parent.name, "start_failed", { cloneName: next.name, error: message });
|
|
406
443
|
return undefined;
|
|
407
444
|
}
|
|
408
445
|
}
|
|
@@ -449,12 +486,16 @@ export async function advanceMatrixAfterClone(
|
|
|
449
486
|
started.push(nextName);
|
|
450
487
|
slots -= 1;
|
|
451
488
|
if (result.alreadyRunning) slots += 0; // слот уже занят этим клоном
|
|
452
|
-
|
|
453
|
-
|
|
489
|
+
logMatrixEvent(parent.name, "queue_started", {
|
|
490
|
+
cloneName: nextName,
|
|
491
|
+
status: "running",
|
|
492
|
+
});
|
|
493
|
+
} catch (err: unknown) {
|
|
454
494
|
const message = err instanceof Error ? err.message : String(err);
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
495
|
+
logMatrixEvent(parent.name, "queue_start_failed", {
|
|
496
|
+
cloneName: nextName,
|
|
497
|
+
error: message,
|
|
498
|
+
});
|
|
458
499
|
// Попытка запуска «съедает» слот: иначе при недоступном бэкенде
|
|
459
500
|
// цикл уйдёт по всей очереди, проигнорировав лимит.
|
|
460
501
|
slots -= 1;
|
|
@@ -467,12 +508,10 @@ export async function advanceMatrixAfterClone(
|
|
|
467
508
|
|
|
468
509
|
try {
|
|
469
510
|
saveTask({ ...parent, matrixQueue: queue.length > 0 ? queue : undefined });
|
|
470
|
-
} catch (err:
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
}`,
|
|
475
|
-
);
|
|
511
|
+
} catch (err: unknown) {
|
|
512
|
+
logMatrixEvent(parent.name, "queue_save_failed", {
|
|
513
|
+
error: err instanceof Error ? err.message : String(err),
|
|
514
|
+
});
|
|
476
515
|
}
|
|
477
516
|
|
|
478
517
|
return started;
|
|
@@ -789,12 +828,10 @@ export async function cancelPendingMatrixClones(
|
|
|
789
828
|
if (Array.isArray(parent.matrixQueue) && parent.matrixQueue.length > 0) {
|
|
790
829
|
try {
|
|
791
830
|
saveTask({ ...parent, matrixQueue: undefined });
|
|
792
|
-
} catch (err:
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
}`,
|
|
797
|
-
);
|
|
831
|
+
} catch (err: unknown) {
|
|
832
|
+
logMatrixEvent(parent.name, "queue_clear_failed", {
|
|
833
|
+
error: err instanceof Error ? err.message : String(err),
|
|
834
|
+
});
|
|
798
835
|
}
|
|
799
836
|
}
|
|
800
837
|
|
|
@@ -827,12 +864,12 @@ export async function cancelPendingMatrixClones(
|
|
|
827
864
|
"skipped",
|
|
828
865
|
);
|
|
829
866
|
cancelled.push(clone.name);
|
|
830
|
-
} catch (err:
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
);
|
|
867
|
+
} catch (err: unknown) {
|
|
868
|
+
logMatrixEvent(parent.name, "cancelled", {
|
|
869
|
+
cloneName: clone.name,
|
|
870
|
+
status: "error",
|
|
871
|
+
error: err instanceof Error ? err.message : String(err),
|
|
872
|
+
});
|
|
836
873
|
}
|
|
837
874
|
}
|
|
838
875
|
|
|
@@ -919,12 +956,18 @@ export async function finalizeMatrixIfComplete(
|
|
|
919
956
|
error: aggregate.errorCount,
|
|
920
957
|
message: aggregate.message,
|
|
921
958
|
});
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
959
|
+
const aggregateRunId = getHistory(parent.name, 1)[0]?.id;
|
|
960
|
+
logMatrixEvent(parent.name, "aggregated", {
|
|
961
|
+
status: finalStatus,
|
|
962
|
+
runId: aggregateRunId,
|
|
963
|
+
reason:
|
|
964
|
+
`total=${aggregate.totalCount}, success=${aggregate.successCount}, ` +
|
|
965
|
+
`error=${aggregate.errorCount}`,
|
|
966
|
+
});
|
|
967
|
+
} catch (err: unknown) {
|
|
968
|
+
logMatrixEvent(parent.name, "aggregate_failed", {
|
|
969
|
+
error: err instanceof Error ? err.message : String(err),
|
|
970
|
+
});
|
|
928
971
|
}
|
|
929
972
|
|
|
930
973
|
if (parent.matrix.createAggregator) {
|
|
@@ -932,13 +975,15 @@ export async function finalizeMatrixIfComplete(
|
|
|
932
975
|
if (aggregator) {
|
|
933
976
|
try {
|
|
934
977
|
await createSchedulerBackend().runTask(aggregator.name);
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
978
|
+
logMatrixEvent(parent.name, "aggregator_started", {
|
|
979
|
+
cloneName: aggregator.name,
|
|
980
|
+
status: "running",
|
|
981
|
+
});
|
|
982
|
+
} catch (err: unknown) {
|
|
983
|
+
logMatrixEvent(parent.name, "aggregator_start_failed", {
|
|
984
|
+
cloneName: aggregator.name,
|
|
985
|
+
error: err instanceof Error ? err.message : String(err),
|
|
986
|
+
});
|
|
942
987
|
}
|
|
943
988
|
}
|
|
944
989
|
}
|