agent-steward 0.0.4 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +309 -2
- package/CROSS_PLATFORM.md +19 -0
- package/README.md +55 -8
- package/ROADMAP-STRATEGY.md +22 -19
- package/ROADMAP.md +230 -159
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +356 -0
- package/dist/core/analytics.d.ts +6 -0
- package/dist/core/analytics.d.ts.map +1 -1
- package/dist/core/analytics.js +20 -3
- package/dist/core/audit.d.ts +6 -0
- package/dist/core/audit.d.ts.map +1 -1
- package/dist/core/audit.js +24 -0
- package/dist/core/budget.d.ts +118 -0
- package/dist/core/budget.d.ts.map +1 -0
- package/dist/core/budget.js +206 -0
- package/dist/core/cron-to-schtasks.d.ts.map +1 -1
- package/dist/core/cron-to-schtasks.js +38 -0
- package/dist/core/dry-run.d.ts +26 -1
- package/dist/core/dry-run.d.ts.map +1 -1
- package/dist/core/dry-run.js +199 -8
- package/dist/core/execution.d.ts +203 -0
- package/dist/core/execution.d.ts.map +1 -0
- package/dist/core/execution.js +309 -0
- package/dist/core/feedback.d.ts +192 -0
- package/dist/core/feedback.d.ts.map +1 -0
- package/dist/core/feedback.js +426 -0
- package/dist/core/gate-integrity.d.ts +71 -0
- package/dist/core/gate-integrity.d.ts.map +1 -0
- package/dist/core/gate-integrity.js +275 -0
- package/dist/core/git-trigger.d.ts.map +1 -1
- package/dist/core/git-trigger.js +14 -2
- package/dist/core/guard-run.d.ts +4 -0
- package/dist/core/guard-run.d.ts.map +1 -0
- package/dist/core/guard-run.js +101 -0
- package/dist/core/guard.d.ts +52 -0
- package/dist/core/guard.d.ts.map +1 -0
- package/dist/core/guard.js +362 -0
- package/dist/core/local-ai.d.ts.map +1 -1
- package/dist/core/local-ai.js +7 -3
- package/dist/core/lock.d.ts +41 -0
- package/dist/core/lock.d.ts.map +1 -1
- package/dist/core/lock.js +141 -29
- package/dist/core/matrix.d.ts +8 -0
- package/dist/core/matrix.d.ts.map +1 -1
- package/dist/core/matrix.js +132 -17
- package/dist/core/mission.d.ts +103 -0
- package/dist/core/mission.d.ts.map +1 -0
- package/dist/core/mission.js +374 -0
- package/dist/core/opencode-models.d.ts +2 -2
- package/dist/core/opencode-models.d.ts.map +1 -1
- package/dist/core/opencode-models.js +28 -21
- package/dist/core/opencode-usage.d.ts +35 -0
- package/dist/core/opencode-usage.d.ts.map +1 -0
- package/dist/core/opencode-usage.js +132 -0
- package/dist/core/pipeline-apply.d.ts +38 -0
- package/dist/core/pipeline-apply.d.ts.map +1 -0
- package/dist/core/pipeline-apply.js +117 -0
- package/dist/core/pipeline-io.d.ts +13 -0
- package/dist/core/pipeline-io.d.ts.map +1 -0
- package/dist/core/pipeline-io.js +78 -0
- package/dist/core/pipeline-map.d.ts +52 -0
- package/dist/core/pipeline-map.d.ts.map +1 -0
- package/dist/core/pipeline-map.js +208 -0
- package/dist/core/pipeline-spec.d.ts +109 -0
- package/dist/core/pipeline-spec.d.ts.map +1 -0
- package/dist/core/pipeline-spec.js +554 -0
- package/dist/core/ps-wrapper.d.ts +27 -2
- package/dist/core/ps-wrapper.d.ts.map +1 -1
- package/dist/core/ps-wrapper.js +424 -65
- package/dist/core/record-history.d.ts.map +1 -1
- package/dist/core/record-history.js +206 -33
- package/dist/core/router/rules.d.ts.map +1 -1
- package/dist/core/router/rules.js +6 -0
- package/dist/core/run-mode.d.ts +12 -0
- package/dist/core/run-mode.d.ts.map +1 -0
- package/dist/core/run-mode.js +17 -0
- package/dist/core/scheduler/linux.d.ts +8 -1
- package/dist/core/scheduler/linux.d.ts.map +1 -1
- package/dist/core/scheduler/linux.js +34 -9
- package/dist/core/scheduler/logs.d.ts +4 -0
- package/dist/core/scheduler/logs.d.ts.map +1 -1
- package/dist/core/scheduler/logs.js +25 -1
- package/dist/core/scheduler/mac.d.ts +44 -2
- package/dist/core/scheduler/mac.d.ts.map +1 -1
- package/dist/core/scheduler/mac.js +126 -15
- package/dist/core/scheduler/windows.d.ts.map +1 -1
- package/dist/core/scheduler/windows.js +15 -12
- package/dist/core/secrets.d.ts.map +1 -1
- package/dist/core/secrets.js +18 -8
- package/dist/core/self-analysis.d.ts +32 -2
- package/dist/core/self-analysis.d.ts.map +1 -1
- package/dist/core/self-analysis.js +38 -27
- package/dist/core/shutdown.d.ts +33 -2
- package/dist/core/shutdown.d.ts.map +1 -1
- package/dist/core/shutdown.js +58 -2
- package/dist/core/structured-log.d.ts +37 -1
- package/dist/core/structured-log.d.ts.map +1 -1
- package/dist/core/structured-log.js +37 -0
- package/dist/core/usage-surface.d.ts +25 -0
- package/dist/core/usage-surface.d.ts.map +1 -0
- package/dist/core/usage-surface.js +41 -0
- package/dist/core/worktree-run.d.ts +4 -0
- package/dist/core/worktree-run.d.ts.map +1 -0
- package/dist/core/worktree-run.js +164 -0
- package/dist/core/worktree.d.ts +115 -0
- package/dist/core/worktree.d.ts.map +1 -0
- package/dist/core/worktree.js +462 -0
- package/dist/core/wrappers/bash.d.ts +19 -0
- package/dist/core/wrappers/bash.d.ts.map +1 -1
- package/dist/core/wrappers/bash.js +230 -27
- package/dist/db/capability-repo.d.ts +43 -2
- package/dist/db/capability-repo.d.ts.map +1 -1
- package/dist/db/capability-repo.js +96 -11
- package/dist/db/history-repo.d.ts +70 -5
- package/dist/db/history-repo.d.ts.map +1 -1
- package/dist/db/history-repo.js +170 -29
- package/dist/db/metrics-repo.d.ts +20 -2
- package/dist/db/metrics-repo.d.ts.map +1 -1
- package/dist/db/metrics-repo.js +35 -13
- package/dist/db/migrate.d.ts.map +1 -1
- package/dist/db/migrate.js +37 -17
- package/dist/db/recovery.d.ts +3 -1
- package/dist/db/recovery.d.ts.map +1 -1
- package/dist/db/recovery.js +11 -5
- package/dist/db/run-journal.d.ts +33 -5
- package/dist/db/run-journal.d.ts.map +1 -1
- package/dist/db/run-journal.js +78 -14
- package/dist/db/run-resume.d.ts +5 -0
- package/dist/db/run-resume.d.ts.map +1 -1
- package/dist/db/run-resume.js +9 -2
- package/dist/db/runs-repo.d.ts +19 -2
- package/dist/db/runs-repo.d.ts.map +1 -1
- package/dist/db/runs-repo.js +91 -60
- package/dist/db/schema.d.ts +2 -2
- package/dist/db/schema.d.ts.map +1 -1
- package/dist/db/schema.js +132 -2
- package/dist/db/templates-repo.d.ts +2 -2
- package/dist/db/templates-repo.d.ts.map +1 -1
- package/dist/db/templates-repo.js +5 -3
- package/dist/history.d.ts +7 -2
- package/dist/history.d.ts.map +1 -1
- package/dist/history.js +4 -2
- package/dist/hook-output.d.ts +2 -0
- package/dist/hook-output.d.ts.map +1 -0
- package/dist/hook-output.js +31 -0
- package/dist/i18n.d.ts +5 -0
- package/dist/i18n.d.ts.map +1 -1
- package/dist/i18n.js +11 -4
- package/dist/index.d.ts +375 -47
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1344 -192
- package/dist/notifications.d.ts +53 -2
- package/dist/notifications.d.ts.map +1 -1
- package/dist/notifications.js +168 -21
- package/dist/profiles.d.ts.map +1 -1
- package/dist/profiles.js +82 -10
- package/dist/rpc.d.ts +354 -0
- package/dist/rpc.d.ts.map +1 -1
- package/dist/rpc.js +145 -0
- package/dist/server.d.ts +32 -2
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +154 -30
- package/dist/settings.d.ts +44 -2
- package/dist/settings.d.ts.map +1 -1
- package/dist/settings.js +85 -3
- package/dist/shared/jobs-view.d.ts +1 -1
- package/dist/shared/jobs-view.d.ts.map +1 -1
- package/dist/shared/jobs-view.js +2 -0
- package/dist/task-transfer.d.ts +21 -0
- package/dist/task-transfer.d.ts.map +1 -1
- package/dist/task-transfer.js +64 -3
- package/dist/templates.d.ts +2 -2
- package/dist/templates.d.ts.map +1 -1
- package/dist/templates.js +3 -3
- package/dist/types.d.ts +22 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/workspaces.d.ts.map +1 -1
- package/dist/workspaces.js +34 -11
- package/docs/ARCHITECTURE.md +319 -66
- package/docs/COMPATIBILITY.md +2 -2
- package/docs/CONCEPT.md +18 -4
- package/docs/CROSS_PLATFORM.ru.md +20 -0
- package/docs/CROSS_PLATFORM.zh.md +18 -0
- package/docs/EXECUTION-LOOP.md +28 -10
- package/docs/FEATURES.md +129 -7
- package/docs/FEATURES.zh.md +97 -6
- package/docs/MIGRATION.md +2 -2
- package/docs/PIPELINE.md +293 -0
- package/docs/QA-A2-GUI-TUI.md +205 -0
- package/docs/README.ru.md +43 -4
- package/docs/README.zh.md +44 -4
- package/docs/SAFETY.md +23 -3
- package/gui-experimental.cjs +9 -8
- package/locales/en.json +58 -3
- package/locales/ru.json +58 -3
- package/locales/zh.json +58 -3
- package/opencode.pipeline.yaml +35 -0
- package/package.json +36 -4
- package/scripts/check-eslint-baseline.mjs +139 -0
- package/scripts/gen-test-plan-counters.mjs +117 -0
- package/scripts/github-ci-fetch.ps1 +37 -0
- package/scripts/pl-commit.ps1 +112 -0
- package/scripts/qa-a2-commit.ps1 +104 -0
- package/scripts/test-home.mjs +107 -0
- package/skills/scheduled-job-best-practices/SKILL.md +95 -1
- package/skills/scheduled-job-best-practices/references/recipes.md +139 -0
- package/src/cli.ts +417 -0
- package/src/core/analytics.ts +27 -4
- package/src/core/audit.ts +25 -0
- package/src/core/budget.ts +340 -0
- package/src/core/cron-to-schtasks.ts +42 -0
- package/src/core/dry-run.ts +235 -12
- package/src/core/execution.ts +538 -0
- package/src/core/feedback.ts +582 -0
- package/src/core/file-watcher.ts +2 -2
- package/src/core/gate-integrity.ts +330 -0
- package/src/core/git-trigger.ts +18 -5
- package/src/core/guard-run.ts +119 -0
- package/src/core/guard.ts +431 -0
- package/src/core/local-ai.ts +7 -3
- package/src/core/lock.ts +178 -37
- package/src/core/matrix.ts +158 -52
- package/src/core/mission.ts +531 -0
- package/src/core/native-notify.ts +1 -1
- package/src/core/opencode-models.ts +30 -21
- package/src/core/opencode-usage.ts +182 -0
- package/src/core/pipeline-apply.ts +158 -0
- package/src/core/pipeline-io.ts +96 -0
- package/src/core/pipeline-map.ts +283 -0
- package/src/core/pipeline-spec.ts +775 -0
- package/src/core/ps-wrapper.ts +435 -66
- package/src/core/record-history.ts +258 -81
- package/src/core/router/rules.ts +6 -0
- package/src/core/run-mode.ts +19 -0
- package/src/core/schedule-gate.ts +1 -1
- package/src/core/scheduler/at-fallback.ts +1 -1
- package/src/core/scheduler/linux.ts +40 -11
- package/src/core/scheduler/logs.ts +34 -3
- package/src/core/scheduler/mac.ts +174 -14
- package/src/core/scheduler/registry.ts +2 -2
- package/src/core/scheduler/windows.ts +19 -15
- package/src/core/secrets.ts +26 -19
- package/src/core/self-analysis.ts +75 -29
- package/src/core/shutdown.ts +90 -4
- package/src/core/structured-log.ts +89 -5
- package/src/core/telegram.ts +2 -2
- package/src/core/triage-worker.ts +7 -7
- package/src/core/triage.ts +2 -2
- package/src/core/usage-surface.ts +70 -0
- package/src/core/worktree-run.ts +199 -0
- package/src/core/worktree.ts +600 -0
- package/src/core/wrappers/bash.ts +248 -26
- package/src/db/capability-repo.ts +127 -10
- package/src/db/history-repo.ts +281 -40
- package/src/db/metrics-repo.ts +40 -13
- package/src/db/migrate.ts +57 -32
- package/src/db/recovery.ts +12 -5
- package/src/db/run-journal.ts +100 -13
- package/src/db/run-resume.ts +17 -1
- package/src/db/runs-repo.ts +147 -65
- package/src/db/schema.ts +134 -2
- package/src/db/templates-repo.ts +5 -3
- package/src/gui.tsx +438 -18
- package/src/history.ts +11 -4
- package/src/hook-output.ts +39 -0
- package/src/i18n.ts +12 -4
- package/src/index.ts +1902 -320
- package/src/notifications.ts +227 -47
- package/src/profiles.ts +88 -10
- package/src/rpc.ts +158 -0
- package/src/server.ts +201 -41
- package/src/settings.ts +125 -5
- package/src/shared/jobs-view.ts +3 -1
- package/src/task-transfer.ts +90 -6
- package/src/templates.ts +3 -3
- package/src/types.ts +23 -2
- package/src/workspaces.ts +37 -14
package/src/core/lock.ts
CHANGED
|
@@ -3,6 +3,17 @@ import path from "node:path";
|
|
|
3
3
|
import { randomUUID } from "node:crypto";
|
|
4
4
|
import { LOCK_DIR, STALE_LOCK_MAX_AGE_MS } from "../types.js";
|
|
5
5
|
|
|
6
|
+
/**
|
|
7
|
+
* O-2: ошибки очистки больше не проглатываются молча. Прямой импорт
|
|
8
|
+
* `structured-log.ts` создал бы цикл (structured-log → secrets → lock), поэтому
|
|
9
|
+
* здесь используется `console.warn` — предупреждение всё равно видно в логе
|
|
10
|
+
* процесса и не ломает основной сценарий.
|
|
11
|
+
*/
|
|
12
|
+
function warnCleanupFailure(target: string, error: unknown): void {
|
|
13
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
14
|
+
console.warn(`[agent-steward] Не удалось удалить lock-артефакт «${target}»: ${reason}`);
|
|
15
|
+
}
|
|
16
|
+
|
|
6
17
|
/**
|
|
7
18
|
* Проверяет, жив ли процесс-владелец lock-файла.
|
|
8
19
|
*
|
|
@@ -16,18 +27,108 @@ export function isProcessAlive(pid: number): boolean {
|
|
|
16
27
|
try {
|
|
17
28
|
process.kill(pid, 0);
|
|
18
29
|
return true;
|
|
19
|
-
} catch (err:
|
|
20
|
-
if (err?.code === "ESRCH") return false;
|
|
30
|
+
} catch (err: unknown) {
|
|
31
|
+
if ((err as NodeJS.ErrnoException | undefined)?.code === "ESRCH") return false;
|
|
21
32
|
return true;
|
|
22
33
|
}
|
|
23
34
|
}
|
|
24
35
|
|
|
25
|
-
/** PID владельца
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
36
|
+
/** Разобранный lock-токен: PID владельца и формат записи. */
|
|
37
|
+
export interface LockToken {
|
|
38
|
+
pid: number | null;
|
|
39
|
+
createdAt: string | null;
|
|
40
|
+
format: "unified" | "node-legacy" | "ps-legacy" | "pid" | "unknown";
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** Достаёт ISO-время из первой части legacy PS-токена (`ISO|PID`). */
|
|
44
|
+
function parseIsoOrNull(value: string | undefined): string | null {
|
|
45
|
+
const text = String(value ?? "").trim();
|
|
46
|
+
return /^\d{4}-\d{2}-\d{2}T/.test(text) ? text : null;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Разбирает содержимое lock-файла в едином для Node/PS/bash виде.
|
|
51
|
+
*
|
|
52
|
+
* Поддерживаемые форматы:
|
|
53
|
+
* `<pid>-<iso>` — единый формат (Node и PS, B6);
|
|
54
|
+
* `<pid>-<epoch>-<uuid>` — старый Node-токен;
|
|
55
|
+
* `<iso>|<pid>` — старый PS-токен (ISO|PID);
|
|
56
|
+
* `<pid>` — голый PID (pid-файл bash-обёртки).
|
|
57
|
+
*
|
|
58
|
+
* Порядок важен: legacy PS проверяется до `<pid>-…`, иначе год из ISO
|
|
59
|
+
* (`2026`) принимался за PID — именно это ломало межпроцессную совместимость.
|
|
60
|
+
*/
|
|
61
|
+
export function parseLockToken(content: string): LockToken {
|
|
62
|
+
const text = String(content ?? "").trim();
|
|
63
|
+
if (!text) return { pid: null, createdAt: null, format: "unknown" };
|
|
64
|
+
|
|
65
|
+
const withPipe = text.indexOf("|");
|
|
66
|
+
if (withPipe !== -1) {
|
|
67
|
+
const pid = Number.parseInt(text.slice(withPipe + 1).trim(), 10);
|
|
68
|
+
return {
|
|
69
|
+
pid: Number.isInteger(pid) && pid > 0 ? pid : null,
|
|
70
|
+
createdAt: parseIsoOrNull(text.slice(0, withPipe)),
|
|
71
|
+
format: "ps-legacy",
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const unified = /^(\d+)-(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)(?:-.*)?$/i.exec(text);
|
|
76
|
+
if (unified) {
|
|
77
|
+
const pid = Number.parseInt(unified[1], 10);
|
|
78
|
+
return {
|
|
79
|
+
pid: Number.isInteger(pid) && pid > 0 ? pid : null,
|
|
80
|
+
createdAt: unified[2],
|
|
81
|
+
format: "unified",
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// Старый Node мог добавить любой суффикс после epoch (`-live-owner`, uuid).
|
|
86
|
+
const legacyNode = /^(\d+)-(\d{10,})(?:-.*)?$/i.exec(text);
|
|
87
|
+
if (legacyNode) {
|
|
88
|
+
const pid = Number.parseInt(legacyNode[1], 10);
|
|
89
|
+
return {
|
|
90
|
+
pid: Number.isInteger(pid) && pid > 0 ? pid : null,
|
|
91
|
+
createdAt: null,
|
|
92
|
+
format: "node-legacy",
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
if (/^\d+$/.test(text)) {
|
|
97
|
+
const pid = Number.parseInt(text, 10);
|
|
98
|
+
// Голое число — PID (pid-файл bash), но не epoch из старого тестового
|
|
99
|
+
// lock: 1.7e12 не может быть PID и раньше не распознавалось владельцем.
|
|
100
|
+
return {
|
|
101
|
+
pid: Number.isInteger(pid) && pid > 0 && pid <= 2_147_483_647 ? pid : null,
|
|
102
|
+
createdAt: null,
|
|
103
|
+
format: "pid",
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
return { pid: null, createdAt: null, format: "unknown" };
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** PID владельца из токена lock-файла (любой поддерживаемый формат). */
|
|
111
|
+
export function ownerPidFromToken(content: string): number | null {
|
|
112
|
+
return parseLockToken(content).pid;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* PID владельца lock-артефакта.
|
|
117
|
+
*
|
|
118
|
+
* B7: файловый lock Node/PS хранит токен в самом файле, а POSIX-обёртка
|
|
119
|
+
* держит каталог `<lock>.d` и пишет `<pid>-<iso>` в `<lock>.d/pid`.
|
|
120
|
+
* Раньше каталог вытеснялся только по возрасту, без чтения этого файла,
|
|
121
|
+
* поэтому легально долгая задача (> 24 ч) после рестарта плагина
|
|
122
|
+
* дублировалась. Нет pid-файла/не разобрался — возвращаем null,
|
|
123
|
+
* и решение принимает возраст (fallback).
|
|
124
|
+
*/
|
|
125
|
+
export async function readLockOwnerPid(
|
|
126
|
+
lockPath: string,
|
|
127
|
+
isDir: boolean,
|
|
128
|
+
): Promise<number | null> {
|
|
129
|
+
const tokenPath = isDir ? path.join(lockPath, "pid") : lockPath;
|
|
130
|
+
const content = await fs.readFile(tokenPath, "utf8").catch(() => "");
|
|
131
|
+
return ownerPidFromToken(content);
|
|
31
132
|
}
|
|
32
133
|
|
|
33
134
|
|
|
@@ -83,26 +184,27 @@ export class LockManager {
|
|
|
83
184
|
// Артефакт предыдущего вытеснения валидным lock быть не может,
|
|
84
185
|
// поэтому убираем его независимо от возраста.
|
|
85
186
|
if (isStaleArtifact) {
|
|
86
|
-
await fs.rm(fullPath, { recursive: true, force: true }).catch(() =>
|
|
187
|
+
await fs.rm(fullPath, { recursive: true, force: true }).catch((err: unknown) =>
|
|
188
|
+
warnCleanupFailure(fullPath, err),
|
|
189
|
+
);
|
|
87
190
|
continue;
|
|
88
191
|
}
|
|
89
192
|
const stat = await fs.stat(fullPath);
|
|
90
193
|
const age = now - stat.mtimeMs;
|
|
91
194
|
if (age > STALE_LOCK_MAX_AGE_MS) {
|
|
92
|
-
// P2-18: возраст — только повод проверить владельца. Задача может
|
|
195
|
+
// P2-18/B7: возраст — только повод проверить владельца. Задача может
|
|
93
196
|
// легально работать дольше 24 ч (timeoutSeconds: 0), поэтому живой
|
|
94
197
|
// процесс не вытесняем: иначе после рестарта плагина появится дубль.
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
}
|
|
198
|
+
// B7: для каталога `.lock.d` владелец лежит в `<dir>/pid`, а не в
|
|
199
|
+
// самом файле; без этого проверка смотрела только на `.lock`.
|
|
200
|
+
const ownerPid = await readLockOwnerPid(fullPath, isLockDir);
|
|
201
|
+
if (ownerPid !== null && isProcessAlive(ownerPid)) {
|
|
202
|
+
console.log(
|
|
203
|
+
`[agent-steward] Lock ${file} старше ${Math.round(
|
|
204
|
+
age / 1000,
|
|
205
|
+
)} с, но процесс ${ownerPid} жив — оставляем.`,
|
|
206
|
+
);
|
|
207
|
+
continue;
|
|
106
208
|
}
|
|
107
209
|
// T-03: не удаляем lock напрямую. Два воркера могли одновременно
|
|
108
210
|
// снести один и тот же файл, после чего один из них удалял свежий
|
|
@@ -115,7 +217,9 @@ export class LockManager {
|
|
|
115
217
|
} catch {
|
|
116
218
|
continue;
|
|
117
219
|
}
|
|
118
|
-
await fs.rm(stalePath, { recursive: true, force: true }).catch(() =>
|
|
220
|
+
await fs.rm(stalePath, { recursive: true, force: true }).catch((err: unknown) =>
|
|
221
|
+
warnCleanupFailure(stalePath, err),
|
|
222
|
+
);
|
|
119
223
|
|
|
120
224
|
console.log(
|
|
121
225
|
`[agent-steward] Вытеснен зависший lock: ${file} (возраст ${Math.round(
|
|
@@ -140,20 +244,59 @@ export class LockManager {
|
|
|
140
244
|
async acquire(taskName: string): Promise<string | null> {
|
|
141
245
|
await fs.mkdir(this.lockDir, { recursive: true });
|
|
142
246
|
const lockPath = this.lockFilePath(taskName);
|
|
143
|
-
|
|
247
|
+
// B6: единый формат `<pid>-<iso>` (+ uuid для уникальности токена).
|
|
248
|
+
// PS-обёртка пишет тот же префикс, поэтому её stale-проверка видит PID,
|
|
249
|
+
// а не год из даты, и не вытесняет живой Node-лок.
|
|
250
|
+
const token = `${process.pid}-${new Date().toISOString()}-${randomUUID()}`;
|
|
144
251
|
|
|
145
252
|
try {
|
|
146
253
|
// 'wx' — exclusive create: ошибка если файл уже существует.
|
|
147
254
|
await fs.writeFile(lockPath, token, { flag: "wx" });
|
|
148
255
|
return token;
|
|
149
|
-
} catch (err:
|
|
150
|
-
|
|
256
|
+
} catch (err: unknown) {
|
|
257
|
+
// P3-39: на Windows конкурентный create/delete того же lock-файла
|
|
258
|
+
// возвращает EPERM/EACCES/EBUSY вместо EEXIST. Это тот же случай
|
|
259
|
+
// «блокировка занята», а не ошибка прав: отдаём null, а withLockWait
|
|
260
|
+
// повторит попытку (withLock корректно пропустит запуск).
|
|
261
|
+
if (
|
|
262
|
+
(err as NodeJS.ErrnoException).code === "EEXIST" ||
|
|
263
|
+
(err as NodeJS.ErrnoException).code === "EPERM" ||
|
|
264
|
+
(err as NodeJS.ErrnoException).code === "EACCES" ||
|
|
265
|
+
(err as NodeJS.ErrnoException).code === "EBUSY"
|
|
266
|
+
) {
|
|
151
267
|
return null;
|
|
152
268
|
}
|
|
153
269
|
throw err;
|
|
154
270
|
}
|
|
155
271
|
}
|
|
156
272
|
|
|
273
|
+
/**
|
|
274
|
+
* Удаляет lock-файл, переживая кратковременные Windows-блокировки.
|
|
275
|
+
*
|
|
276
|
+
* На Windows unlink может вернуть EPERM/EACCES/EBUSY, пока другой процесс
|
|
277
|
+
* читает файл (проверка владельца в withLockWait) или файл ещё «висящий»
|
|
278
|
+
* после создания. Несколько коротких повторов решают гонку; ENOENT — уже
|
|
279
|
+
* удалён, это не ошибка.
|
|
280
|
+
*/
|
|
281
|
+
private async unlinkWithRetry(lockPath: string, attempts = 6): Promise<boolean> {
|
|
282
|
+
for (let i = 0; i < attempts; i += 1) {
|
|
283
|
+
try {
|
|
284
|
+
await fs.unlink(lockPath);
|
|
285
|
+
return true;
|
|
286
|
+
} catch (err: unknown) {
|
|
287
|
+
if ((err as NodeJS.ErrnoException).code === "ENOENT") return false;
|
|
288
|
+
const transient =
|
|
289
|
+
(err as NodeJS.ErrnoException).code === "EPERM" || (err as NodeJS.ErrnoException).code === "EACCES" || (err as NodeJS.ErrnoException).code === "EBUSY";
|
|
290
|
+
if (transient && i < attempts - 1) {
|
|
291
|
+
await new Promise((resolve) => setTimeout(resolve, 10 * (i + 1)));
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
294
|
+
throw err;
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
return false;
|
|
298
|
+
}
|
|
299
|
+
|
|
157
300
|
/**
|
|
158
301
|
* Освобождает блокировку задачи (удаляет lock-файл).
|
|
159
302
|
*
|
|
@@ -167,18 +310,14 @@ export class LockManager {
|
|
|
167
310
|
let current: string;
|
|
168
311
|
try {
|
|
169
312
|
current = await fs.readFile(lockPath, "utf8");
|
|
170
|
-
} catch (err:
|
|
171
|
-
if (err.code === "ENOENT") return false;
|
|
313
|
+
} catch (err: unknown) {
|
|
314
|
+
if ((err as NodeJS.ErrnoException).code === "ENOENT") return false;
|
|
172
315
|
throw err;
|
|
173
316
|
}
|
|
174
317
|
if (current !== token) return false;
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
} catch (err: any) {
|
|
179
|
-
if (err.code === "ENOENT") return false;
|
|
180
|
-
throw err;
|
|
181
|
-
}
|
|
318
|
+
// P3-39: на Windows unlink может кратковременно упасть EPERM, пока
|
|
319
|
+
// другие процессы читают lock для проверки владельца.
|
|
320
|
+
return this.unlinkWithRetry(lockPath);
|
|
182
321
|
}
|
|
183
322
|
|
|
184
323
|
/**
|
|
@@ -189,8 +328,8 @@ export class LockManager {
|
|
|
189
328
|
try {
|
|
190
329
|
await fs.stat(lockPath);
|
|
191
330
|
return true;
|
|
192
|
-
} catch (err:
|
|
193
|
-
if (err.code === "ENOENT") return false;
|
|
331
|
+
} catch (err: unknown) {
|
|
332
|
+
if ((err as NodeJS.ErrnoException).code === "ENOENT") return false;
|
|
194
333
|
throw err;
|
|
195
334
|
}
|
|
196
335
|
}
|
|
@@ -264,7 +403,9 @@ export async function withLockWait<T>(
|
|
|
264
403
|
const stalePath = `${lockPath}.stale-${process.pid}-${Date.now()}`;
|
|
265
404
|
try {
|
|
266
405
|
await fs.rename(lockPath, stalePath);
|
|
267
|
-
await fs.rm(stalePath, { force: true }).catch(() =>
|
|
406
|
+
await fs.rm(stalePath, { force: true }).catch((err: unknown) =>
|
|
407
|
+
warnCleanupFailure(stalePath, err),
|
|
408
|
+
);
|
|
268
409
|
} catch {
|
|
269
410
|
// Кто-то успел раньше — просто повторяем acquire.
|
|
270
411
|
}
|
package/src/core/matrix.ts
CHANGED
|
@@ -17,12 +17,41 @@ import {
|
|
|
17
17
|
listTasks,
|
|
18
18
|
saveTask,
|
|
19
19
|
} from "../db/tasks-repo.js";
|
|
20
|
-
import { getHistory, recordRun } from "../db/history-repo.js";
|
|
20
|
+
import { getHistory, getLatestHistoryByTask, recordRun } from "../db/history-repo.js";
|
|
21
21
|
import { notifyMatrixCompletion } from "../notifications.js";
|
|
22
22
|
import { getWorkspace } from "../workspaces.js";
|
|
23
23
|
import { createSchedulerBackend } from "./scheduler/index.js";
|
|
24
|
+
import { appendStructuredLog } from "./structured-log.js";
|
|
24
25
|
import { t } from "../i18n.js";
|
|
25
26
|
|
|
27
|
+
/**
|
|
28
|
+
* Матричные события идут в machine-readable runs.jsonl: человекочитаемый
|
|
29
|
+
* вывод планировщика не позволяет связать клон с родителем и запуском.
|
|
30
|
+
*/
|
|
31
|
+
function logMatrixEvent(
|
|
32
|
+
parentName: string,
|
|
33
|
+
step: string,
|
|
34
|
+
details: {
|
|
35
|
+
cloneName?: string;
|
|
36
|
+
status?: string;
|
|
37
|
+
reason?: string;
|
|
38
|
+
error?: string;
|
|
39
|
+
runId?: number;
|
|
40
|
+
} = {},
|
|
41
|
+
): void {
|
|
42
|
+
appendStructuredLog({
|
|
43
|
+
event: `matrix.${step}`,
|
|
44
|
+
task: parentName,
|
|
45
|
+
matrixId: parentName,
|
|
46
|
+
cloneName: details.cloneName,
|
|
47
|
+
step,
|
|
48
|
+
status: details.status,
|
|
49
|
+
reason: details.reason,
|
|
50
|
+
error: details.error,
|
|
51
|
+
runId: details.runId,
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
|
|
26
55
|
/** Суффикс агрегирующей задачи, запускаемой после всех клонов. */
|
|
27
56
|
export const MATRIX_COMPLETE_SUFFIX = "-matrix-complete";
|
|
28
57
|
|
|
@@ -145,6 +174,52 @@ function latestCloneStatus(cloneName: string): {
|
|
|
145
174
|
};
|
|
146
175
|
}
|
|
147
176
|
|
|
177
|
+
/**
|
|
178
|
+
* QA-F.2: клон матрицы может остаться только в истории — определение удаляется
|
|
179
|
+
* при реконфигурации матрицы (`syncMatrixFamily`) или `cleanup`, но аналитика
|
|
180
|
+
* продолжает показывать задачу по имени. История не хранит `matrixParentId`,
|
|
181
|
+
* поэтому родитель восстанавливается из канонического префикса имени клона
|
|
182
|
+
* `<parent>-matrix-<workspace>` (тот же, что порождает `matrixCloneName`).
|
|
183
|
+
*/
|
|
184
|
+
function listMatrixClonesFromHistory(parentId: string): { name: string; workspaceId: string | null }[] {
|
|
185
|
+
const prefix = `${parentId.trim()}-matrix-`.toLowerCase();
|
|
186
|
+
const clones: { name: string; workspaceId: string | null }[] = [];
|
|
187
|
+
const seen = new Set<string>();
|
|
188
|
+
for (const record of getLatestHistoryByTask(5000)) {
|
|
189
|
+
const name = record.taskName?.trim();
|
|
190
|
+
if (!name) continue;
|
|
191
|
+
const key = name.toLowerCase();
|
|
192
|
+
// Агрегатор `-matrix-complete` тоже содержит префикс, но это не клон.
|
|
193
|
+
if (!key.startsWith(prefix) || isMatrixCompleteName(name) || seen.has(key)) continue;
|
|
194
|
+
seen.add(key);
|
|
195
|
+
clones.push({ name, workspaceId: record.workspaceId ?? null });
|
|
196
|
+
}
|
|
197
|
+
return clones;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/**
|
|
201
|
+
* QA-P3.7: задача считается матрицей, если у неё есть конфиг `matrix` либо
|
|
202
|
+
* в истории есть записи с каноническим префиксом клона `<name>-matrix-`
|
|
203
|
+
* (включая агрегатор `-matrix-complete`). Проверка истории важна для QA-P1.2:
|
|
204
|
+
* определение родителя могло быть удалено, но запуски клонов остались, и
|
|
205
|
+
* matrix_control должен продолжать их показывать.
|
|
206
|
+
*/
|
|
207
|
+
export function isKnownMatrix(name: string): boolean {
|
|
208
|
+
const trimmed = name.trim();
|
|
209
|
+
if (!trimmed) return false;
|
|
210
|
+
if (getTask(trimmed)?.matrix) return true;
|
|
211
|
+
const prefix = `${trimmed}-matrix-`.toLowerCase();
|
|
212
|
+
return getLatestHistoryByTask(5000).some((record) =>
|
|
213
|
+
(record.taskName ?? "").trim().toLowerCase().startsWith(prefix),
|
|
214
|
+
);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/** Источник клона для агрегации: определение задачи или запись истории. */
|
|
218
|
+
interface MatrixCloneSource {
|
|
219
|
+
name: string;
|
|
220
|
+
workspaceId?: string | null;
|
|
221
|
+
}
|
|
222
|
+
|
|
148
223
|
export interface MatrixCloneResult {
|
|
149
224
|
name: string;
|
|
150
225
|
workspaceId: string | null;
|
|
@@ -169,7 +244,22 @@ export interface MatrixAggregate {
|
|
|
169
244
|
/** Собирает результаты клонов и вычисляет агрегированный статус. */
|
|
170
245
|
export function aggregateMatrixResults(parentId: string): MatrixAggregate {
|
|
171
246
|
const parent = getTask(parentId);
|
|
172
|
-
const
|
|
247
|
+
const definedClones = listMatrixClones(parentId);
|
|
248
|
+
// QA-F.2 / QA-P1.2: объединяем определения с клонами, оставшимися только
|
|
249
|
+
// в истории, чтобы matrix_control показывал тот же набор, что и аналитика.
|
|
250
|
+
// История — независимый источник:definition родителя может быть удалён
|
|
251
|
+
// (например, при чистке сирот), но запуски клонов остаются в истории, и
|
|
252
|
+
// аналитика по-прежнему их видит. Раньше здесь стоял guard `parent ? … : []`,
|
|
253
|
+
// из-за которого live-матрица __matrixprobe__ показывала «нет клонов», хотя
|
|
254
|
+
// история содержала __matrixprobe__-matrix-__mw__. Пустой агрегат остаётся
|
|
255
|
+
// только когда нет ни определений, ни записей истории с каноническим
|
|
256
|
+
// префиксом <parent>-matrix-.
|
|
257
|
+
const historyClones = listMatrixClonesFromHistory(parentId);
|
|
258
|
+
const known = new Set(definedClones.map((clone) => clone.name.toLowerCase()));
|
|
259
|
+
const clones: MatrixCloneSource[] = [
|
|
260
|
+
...definedClones,
|
|
261
|
+
...historyClones.filter((clone) => !known.has(clone.name.toLowerCase())),
|
|
262
|
+
];
|
|
173
263
|
const failureMode = parent?.matrix?.failureMode ?? "continue";
|
|
174
264
|
|
|
175
265
|
const results: MatrixCloneResult[] = clones.map((clone) => {
|
|
@@ -314,9 +404,10 @@ export async function runMatrix(task: TaskDefinition): Promise<MatrixRunResult>
|
|
|
314
404
|
} else {
|
|
315
405
|
started.push(clone.name);
|
|
316
406
|
}
|
|
317
|
-
} catch (err:
|
|
407
|
+
} catch (err: unknown) {
|
|
318
408
|
const message = err instanceof Error ? err.message : String(err);
|
|
319
409
|
errors.push(`${clone.name}: ${message}`);
|
|
410
|
+
logMatrixEvent(task.name, "start_failed", { cloneName: clone.name, error: message });
|
|
320
411
|
if (failureMode === "stop") {
|
|
321
412
|
const rest = clones
|
|
322
413
|
.slice(clones.indexOf(clone) + 1)
|
|
@@ -332,19 +423,19 @@ export async function runMatrix(task: TaskDefinition): Promise<MatrixRunResult>
|
|
|
332
423
|
failureMode === "stop" && errors.length > 0 ? [] : queued;
|
|
333
424
|
try {
|
|
334
425
|
saveTask({ ...task, matrixQueue: queueToSave.length > 0 ? queueToSave : undefined });
|
|
335
|
-
} catch (err:
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
}`,
|
|
340
|
-
);
|
|
426
|
+
} catch (err: unknown) {
|
|
427
|
+
logMatrixEvent(task.name, "queue_save_failed", {
|
|
428
|
+
error: err instanceof Error ? err.message : String(err),
|
|
429
|
+
});
|
|
341
430
|
}
|
|
342
431
|
|
|
343
|
-
|
|
344
|
-
|
|
432
|
+
logMatrixEvent(task.name, "started", {
|
|
433
|
+
status: started.length > 0 ? "running" : "error",
|
|
434
|
+
reason:
|
|
435
|
+
`strategy=${strategy}, failureMode=${failureMode}, ` +
|
|
345
436
|
`concurrency=${concurrency}, started=${started.length}, skipped=${skipped.length}, ` +
|
|
346
437
|
`queued=${queueToSave.length}, errors=${errors.length}`,
|
|
347
|
-
);
|
|
438
|
+
});
|
|
348
439
|
|
|
349
440
|
return {
|
|
350
441
|
success: errors.length === 0 || failureMode === "continue",
|
|
@@ -387,7 +478,10 @@ export async function advanceSequentialMatrix(
|
|
|
387
478
|
if ((parent.matrix.failureMode ?? "continue") === "stop") {
|
|
388
479
|
const latest = latestCloneStatus(cloneName);
|
|
389
480
|
if (latest.status === "error") {
|
|
390
|
-
|
|
481
|
+
logMatrixEvent(parent.name, "sequential_stopped", {
|
|
482
|
+
cloneName,
|
|
483
|
+
status: "error",
|
|
484
|
+
});
|
|
391
485
|
return undefined;
|
|
392
486
|
}
|
|
393
487
|
}
|
|
@@ -398,11 +492,15 @@ export async function advanceSequentialMatrix(
|
|
|
398
492
|
try {
|
|
399
493
|
const result = await createSchedulerBackend().runTask(next.name);
|
|
400
494
|
if (result.alreadyRunning) return undefined;
|
|
401
|
-
|
|
495
|
+
logMatrixEvent(parent.name, "advanced", {
|
|
496
|
+
cloneName: next.name,
|
|
497
|
+
status: "running",
|
|
498
|
+
reason: "sequential",
|
|
499
|
+
});
|
|
402
500
|
return next.name;
|
|
403
|
-
} catch (err:
|
|
501
|
+
} catch (err: unknown) {
|
|
404
502
|
const message = err instanceof Error ? err.message : String(err);
|
|
405
|
-
|
|
503
|
+
logMatrixEvent(parent.name, "start_failed", { cloneName: next.name, error: message });
|
|
406
504
|
return undefined;
|
|
407
505
|
}
|
|
408
506
|
}
|
|
@@ -449,12 +547,16 @@ export async function advanceMatrixAfterClone(
|
|
|
449
547
|
started.push(nextName);
|
|
450
548
|
slots -= 1;
|
|
451
549
|
if (result.alreadyRunning) slots += 0; // слот уже занят этим клоном
|
|
452
|
-
|
|
453
|
-
|
|
550
|
+
logMatrixEvent(parent.name, "queue_started", {
|
|
551
|
+
cloneName: nextName,
|
|
552
|
+
status: "running",
|
|
553
|
+
});
|
|
554
|
+
} catch (err: unknown) {
|
|
454
555
|
const message = err instanceof Error ? err.message : String(err);
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
556
|
+
logMatrixEvent(parent.name, "queue_start_failed", {
|
|
557
|
+
cloneName: nextName,
|
|
558
|
+
error: message,
|
|
559
|
+
});
|
|
458
560
|
// Попытка запуска «съедает» слот: иначе при недоступном бэкенде
|
|
459
561
|
// цикл уйдёт по всей очереди, проигнорировав лимит.
|
|
460
562
|
slots -= 1;
|
|
@@ -467,12 +569,10 @@ export async function advanceMatrixAfterClone(
|
|
|
467
569
|
|
|
468
570
|
try {
|
|
469
571
|
saveTask({ ...parent, matrixQueue: queue.length > 0 ? queue : undefined });
|
|
470
|
-
} catch (err:
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
}`,
|
|
475
|
-
);
|
|
572
|
+
} catch (err: unknown) {
|
|
573
|
+
logMatrixEvent(parent.name, "queue_save_failed", {
|
|
574
|
+
error: err instanceof Error ? err.message : String(err),
|
|
575
|
+
});
|
|
476
576
|
}
|
|
477
577
|
|
|
478
578
|
return started;
|
|
@@ -789,12 +889,10 @@ export async function cancelPendingMatrixClones(
|
|
|
789
889
|
if (Array.isArray(parent.matrixQueue) && parent.matrixQueue.length > 0) {
|
|
790
890
|
try {
|
|
791
891
|
saveTask({ ...parent, matrixQueue: undefined });
|
|
792
|
-
} catch (err:
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
}`,
|
|
797
|
-
);
|
|
892
|
+
} catch (err: unknown) {
|
|
893
|
+
logMatrixEvent(parent.name, "queue_clear_failed", {
|
|
894
|
+
error: err instanceof Error ? err.message : String(err),
|
|
895
|
+
});
|
|
798
896
|
}
|
|
799
897
|
}
|
|
800
898
|
|
|
@@ -827,12 +925,12 @@ export async function cancelPendingMatrixClones(
|
|
|
827
925
|
"skipped",
|
|
828
926
|
);
|
|
829
927
|
cancelled.push(clone.name);
|
|
830
|
-
} catch (err:
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
);
|
|
928
|
+
} catch (err: unknown) {
|
|
929
|
+
logMatrixEvent(parent.name, "cancelled", {
|
|
930
|
+
cloneName: clone.name,
|
|
931
|
+
status: "error",
|
|
932
|
+
error: err instanceof Error ? err.message : String(err),
|
|
933
|
+
});
|
|
836
934
|
}
|
|
837
935
|
}
|
|
838
936
|
|
|
@@ -919,12 +1017,18 @@ export async function finalizeMatrixIfComplete(
|
|
|
919
1017
|
error: aggregate.errorCount,
|
|
920
1018
|
message: aggregate.message,
|
|
921
1019
|
});
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
1020
|
+
const aggregateRunId = getHistory(parent.name, 1)[0]?.id;
|
|
1021
|
+
logMatrixEvent(parent.name, "aggregated", {
|
|
1022
|
+
status: finalStatus,
|
|
1023
|
+
runId: aggregateRunId,
|
|
1024
|
+
reason:
|
|
1025
|
+
`total=${aggregate.totalCount}, success=${aggregate.successCount}, ` +
|
|
1026
|
+
`error=${aggregate.errorCount}`,
|
|
1027
|
+
});
|
|
1028
|
+
} catch (err: unknown) {
|
|
1029
|
+
logMatrixEvent(parent.name, "aggregate_failed", {
|
|
1030
|
+
error: err instanceof Error ? err.message : String(err),
|
|
1031
|
+
});
|
|
928
1032
|
}
|
|
929
1033
|
|
|
930
1034
|
if (parent.matrix.createAggregator) {
|
|
@@ -932,13 +1036,15 @@ export async function finalizeMatrixIfComplete(
|
|
|
932
1036
|
if (aggregator) {
|
|
933
1037
|
try {
|
|
934
1038
|
await createSchedulerBackend().runTask(aggregator.name);
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
1039
|
+
logMatrixEvent(parent.name, "aggregator_started", {
|
|
1040
|
+
cloneName: aggregator.name,
|
|
1041
|
+
status: "running",
|
|
1042
|
+
});
|
|
1043
|
+
} catch (err: unknown) {
|
|
1044
|
+
logMatrixEvent(parent.name, "aggregator_start_failed", {
|
|
1045
|
+
cloneName: aggregator.name,
|
|
1046
|
+
error: err instanceof Error ? err.message : String(err),
|
|
1047
|
+
});
|
|
942
1048
|
}
|
|
943
1049
|
}
|
|
944
1050
|
}
|