agent-steward 0.0.4 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +134 -2
- package/CROSS_PLATFORM.md +19 -0
- package/README.md +52 -7
- package/ROADMAP-STRATEGY.md +21 -18
- package/ROADMAP.md +140 -159
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +351 -0
- package/dist/core/execution.d.ts +203 -0
- package/dist/core/execution.d.ts.map +1 -0
- package/dist/core/execution.js +309 -0
- package/dist/core/gate-integrity.d.ts +35 -0
- package/dist/core/gate-integrity.d.ts.map +1 -0
- package/dist/core/gate-integrity.js +132 -0
- package/dist/core/git-trigger.d.ts.map +1 -1
- package/dist/core/git-trigger.js +14 -2
- package/dist/core/guard-run.d.ts +4 -0
- package/dist/core/guard-run.d.ts.map +1 -0
- package/dist/core/guard-run.js +101 -0
- package/dist/core/guard.d.ts +52 -0
- package/dist/core/guard.d.ts.map +1 -0
- package/dist/core/guard.js +362 -0
- package/dist/core/lock.d.ts +41 -0
- package/dist/core/lock.d.ts.map +1 -1
- package/dist/core/lock.js +141 -29
- package/dist/core/matrix.d.ts.map +1 -1
- package/dist/core/matrix.js +75 -15
- package/dist/core/mission.d.ts +83 -0
- package/dist/core/mission.d.ts.map +1 -0
- package/dist/core/mission.js +334 -0
- package/dist/core/pipeline-apply.d.ts +38 -0
- package/dist/core/pipeline-apply.d.ts.map +1 -0
- package/dist/core/pipeline-apply.js +117 -0
- package/dist/core/pipeline-io.d.ts +13 -0
- package/dist/core/pipeline-io.d.ts.map +1 -0
- package/dist/core/pipeline-io.js +64 -0
- package/dist/core/pipeline-map.d.ts +46 -0
- package/dist/core/pipeline-map.d.ts.map +1 -0
- package/dist/core/pipeline-map.js +196 -0
- package/dist/core/pipeline-spec.d.ts +109 -0
- package/dist/core/pipeline-spec.d.ts.map +1 -0
- package/dist/core/pipeline-spec.js +554 -0
- package/dist/core/ps-wrapper.d.ts +19 -2
- package/dist/core/ps-wrapper.d.ts.map +1 -1
- package/dist/core/ps-wrapper.js +250 -62
- package/dist/core/record-history.d.ts.map +1 -1
- package/dist/core/record-history.js +130 -29
- package/dist/core/router/rules.d.ts.map +1 -1
- package/dist/core/router/rules.js +6 -0
- package/dist/core/scheduler/linux.d.ts +8 -1
- package/dist/core/scheduler/linux.d.ts.map +1 -1
- package/dist/core/scheduler/linux.js +34 -9
- package/dist/core/scheduler/mac.d.ts +44 -2
- package/dist/core/scheduler/mac.d.ts.map +1 -1
- package/dist/core/scheduler/mac.js +126 -15
- package/dist/core/scheduler/windows.d.ts.map +1 -1
- package/dist/core/scheduler/windows.js +15 -12
- package/dist/core/structured-log.d.ts +7 -1
- package/dist/core/structured-log.d.ts.map +1 -1
- package/dist/core/worktree-run.d.ts +4 -0
- package/dist/core/worktree-run.d.ts.map +1 -0
- package/dist/core/worktree-run.js +164 -0
- package/dist/core/worktree.d.ts +115 -0
- package/dist/core/worktree.d.ts.map +1 -0
- package/dist/core/worktree.js +462 -0
- package/dist/core/wrappers/bash.d.ts +17 -0
- package/dist/core/wrappers/bash.d.ts.map +1 -1
- package/dist/core/wrappers/bash.js +175 -24
- package/dist/db/capability-repo.d.ts +43 -2
- package/dist/db/capability-repo.d.ts.map +1 -1
- package/dist/db/capability-repo.js +96 -11
- package/dist/db/history-repo.d.ts +38 -5
- package/dist/db/history-repo.d.ts.map +1 -1
- package/dist/db/history-repo.js +88 -25
- package/dist/db/metrics-repo.js +2 -2
- package/dist/db/recovery.d.ts.map +1 -1
- package/dist/db/recovery.js +5 -4
- package/dist/db/run-journal.d.ts +33 -5
- package/dist/db/run-journal.d.ts.map +1 -1
- package/dist/db/run-journal.js +78 -14
- package/dist/db/run-resume.d.ts +5 -0
- package/dist/db/run-resume.d.ts.map +1 -1
- package/dist/db/run-resume.js +9 -2
- package/dist/db/runs-repo.d.ts +19 -2
- package/dist/db/runs-repo.d.ts.map +1 -1
- package/dist/db/runs-repo.js +22 -6
- package/dist/db/schema.d.ts +2 -2
- package/dist/db/schema.d.ts.map +1 -1
- package/dist/db/schema.js +63 -2
- package/dist/history.d.ts +1 -1
- package/dist/history.d.ts.map +1 -1
- package/dist/i18n.d.ts +5 -0
- package/dist/i18n.d.ts.map +1 -1
- package/dist/i18n.js +11 -4
- package/dist/index.d.ts +136 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +378 -14
- package/dist/notifications.d.ts +22 -0
- package/dist/notifications.d.ts.map +1 -1
- package/dist/notifications.js +36 -8
- package/dist/profiles.d.ts.map +1 -1
- package/dist/profiles.js +25 -4
- package/dist/rpc.d.ts +246 -0
- package/dist/rpc.d.ts.map +1 -1
- package/dist/rpc.js +117 -0
- package/dist/server.d.ts +15 -0
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +77 -18
- package/dist/settings.d.ts +2 -2
- package/dist/settings.d.ts.map +1 -1
- package/dist/settings.js +3 -3
- package/dist/shared/jobs-view.d.ts +1 -1
- package/dist/shared/jobs-view.d.ts.map +1 -1
- package/dist/shared/jobs-view.js +2 -0
- package/dist/task-transfer.d.ts +16 -0
- package/dist/task-transfer.d.ts.map +1 -1
- package/dist/task-transfer.js +16 -0
- package/docs/ARCHITECTURE.md +281 -65
- package/docs/COMPATIBILITY.md +2 -2
- package/docs/CONCEPT.md +18 -4
- package/docs/CROSS_PLATFORM.ru.md +20 -0
- package/docs/CROSS_PLATFORM.zh.md +18 -0
- package/docs/EXECUTION-LOOP.md +22 -7
- package/docs/FEATURES.md +65 -4
- package/docs/FEATURES.zh.md +52 -4
- package/docs/MIGRATION.md +2 -2
- package/docs/PIPELINE.md +293 -0
- package/docs/README.ru.md +42 -3
- package/docs/README.zh.md +43 -3
- package/docs/SAFETY.md +23 -3
- package/gui-experimental.cjs +9 -8
- package/locales/en.json +17 -1
- package/locales/ru.json +17 -1
- package/locales/zh.json +17 -1
- package/opencode.pipeline.yaml +35 -0
- package/package.json +23 -4
- package/scripts/test-home.mjs +83 -0
- package/src/cli.ts +412 -0
- package/src/core/dry-run.ts +2 -2
- package/src/core/execution.ts +538 -0
- package/src/core/file-watcher.ts +2 -2
- package/src/core/gate-integrity.ts +160 -0
- package/src/core/git-trigger.ts +18 -5
- package/src/core/guard-run.ts +119 -0
- package/src/core/guard.ts +431 -0
- package/src/core/lock.ts +178 -37
- package/src/core/matrix.ts +95 -50
- package/src/core/mission.ts +485 -0
- package/src/core/native-notify.ts +1 -1
- package/src/core/pipeline-apply.ts +158 -0
- package/src/core/pipeline-io.ts +79 -0
- package/src/core/pipeline-map.ts +264 -0
- package/src/core/pipeline-spec.ts +775 -0
- package/src/core/ps-wrapper.ts +263 -63
- package/src/core/record-history.ts +164 -77
- package/src/core/router/rules.ts +6 -0
- package/src/core/schedule-gate.ts +1 -1
- package/src/core/scheduler/at-fallback.ts +1 -1
- package/src/core/scheduler/linux.ts +40 -11
- package/src/core/scheduler/logs.ts +2 -2
- package/src/core/scheduler/mac.ts +174 -14
- package/src/core/scheduler/registry.ts +2 -2
- package/src/core/scheduler/windows.ts +19 -15
- package/src/core/secrets.ts +8 -8
- package/src/core/structured-log.ts +22 -5
- package/src/core/telegram.ts +2 -2
- package/src/core/triage-worker.ts +7 -7
- package/src/core/triage.ts +2 -2
- package/src/core/worktree-run.ts +199 -0
- package/src/core/worktree.ts +600 -0
- package/src/core/wrappers/bash.ts +190 -23
- package/src/db/capability-repo.ts +127 -10
- package/src/db/history-repo.ts +129 -22
- package/src/db/metrics-repo.ts +2 -2
- package/src/db/recovery.ts +5 -4
- package/src/db/run-journal.ts +100 -13
- package/src/db/run-resume.ts +17 -1
- package/src/db/runs-repo.ts +36 -6
- package/src/db/schema.ts +65 -2
- package/src/gui.tsx +177 -13
- package/src/history.ts +1 -1
- package/src/i18n.ts +12 -4
- package/src/index.ts +524 -4
- package/src/notifications.ts +38 -8
- package/src/profiles.ts +26 -4
- package/src/rpc.ts +128 -0
- package/src/server.ts +94 -18
- package/src/settings.ts +5 -5
- package/src/shared/jobs-view.ts +3 -1
- package/src/task-transfer.ts +31 -0
|
@@ -24,6 +24,10 @@ export interface StoredHistoryRecord {
|
|
|
24
24
|
workspaceId?: string | null;
|
|
25
25
|
allowedTools?: string[] | null;
|
|
26
26
|
deniedTools?: string[] | null;
|
|
27
|
+
/** Владелец незавершённого запуска (схема v8). */
|
|
28
|
+
ownerPid?: number | null;
|
|
29
|
+
ownerHost?: string | null;
|
|
30
|
+
ownerStartedAt?: string | null;
|
|
27
31
|
}
|
|
28
32
|
/** Поля, которые пишутся в history при завершении запуска. */
|
|
29
33
|
export interface RunResult {
|
|
@@ -43,7 +47,11 @@ export interface RunResult {
|
|
|
43
47
|
/** Вставляет запись истории без инкремента метрик (совместимость с appendHistory). */
|
|
44
48
|
export declare function insertHistory(record: StoredHistoryRecord): number;
|
|
45
49
|
/** Начало запуска: строка с finished_at = NULL (переживёт сбой процесса). */
|
|
46
|
-
export declare function startRun(taskName: string, startedAt?: string, workspaceId?: string | null, allowedTools?: string[] | null, deniedTools?: string[] | null
|
|
50
|
+
export declare function startRun(taskName: string, startedAt?: string, workspaceId?: string | null, allowedTools?: string[] | null, deniedTools?: string[] | null, owner?: {
|
|
51
|
+
pid?: number | null;
|
|
52
|
+
host?: string | null;
|
|
53
|
+
startedAt?: string | null;
|
|
54
|
+
}): number;
|
|
47
55
|
/**
|
|
48
56
|
* Завершает ранее начатый запуск и инкрементирует метрики.
|
|
49
57
|
* Возвращает false, если запись уже завершена/не найдена (защита от двойного учёта).
|
|
@@ -67,11 +75,36 @@ export declare function getLatestHistoryByTask(limit?: number): StoredHistoryRec
|
|
|
67
75
|
/** История запусков, начатых не раньше timestamp (ISO-строка). */
|
|
68
76
|
export declare function getHistorySince(timestamp: string, limit?: number): StoredHistoryRecord[];
|
|
69
77
|
/**
|
|
70
|
-
* Crash recovery
|
|
71
|
-
*
|
|
72
|
-
*
|
|
78
|
+
* Crash recovery для истории.
|
|
79
|
+
*
|
|
80
|
+
* Незавершённая запись (finished_at IS NULL) финализируется как error, только
|
|
81
|
+
* если её владелец больше не может её завершить:
|
|
82
|
+
* - владелец известен, жив и на этом же хосте → запись оставляем (B1);
|
|
83
|
+
* - владелец известен и мёртв → финализируем сразу;
|
|
84
|
+
* - владелец неизвестен (legacy-строка или другой хост) → финализируем,
|
|
85
|
+
* только когда запись старше `maxUnverifiableAgeMs`.
|
|
73
86
|
*/
|
|
74
|
-
|
|
87
|
+
/** Возраст, после которого запись с непроверяемым владельцем считается осиротевшей. */
|
|
88
|
+
export declare const UNVERIFIABLE_OWNER_MAX_AGE_MS: number;
|
|
89
|
+
export interface RecoverInterruptedResult {
|
|
90
|
+
/** Сколько записей финализировано как error. */
|
|
91
|
+
recovered: number;
|
|
92
|
+
/** Сколько записей оставлено живым владельцам. */
|
|
93
|
+
keptAlive: number;
|
|
94
|
+
}
|
|
95
|
+
export interface RecoverInterruptedOptions {
|
|
96
|
+
/** Текущий момент (для тестов). */
|
|
97
|
+
now?: Date;
|
|
98
|
+
/** Порог возраста для записей с непроверяемым владельцем. */
|
|
99
|
+
maxUnverifiableAgeMs?: number;
|
|
100
|
+
/** Имя текущего хоста (для тестов). */
|
|
101
|
+
hostname?: string;
|
|
102
|
+
/** Проверка живости PID (для тестов). */
|
|
103
|
+
isAlive?: (pid: number) => boolean;
|
|
104
|
+
}
|
|
105
|
+
/** Проверяет существование процесса: signal 0 не убивает, только пробует. */
|
|
106
|
+
export declare function isProcessAlive(pid: number): boolean;
|
|
107
|
+
export declare function recoverInterrupted(options?: RecoverInterruptedOptions): RecoverInterruptedResult;
|
|
75
108
|
/** Количество записей истории (для миграции и тестов). */
|
|
76
109
|
export declare function countHistory(): number;
|
|
77
110
|
/** Число активных (незавершённых) запусков: глубина очереди для /api/health. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"history-repo.d.ts","sourceRoot":"","sources":["../../src/db/history-repo.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"history-repo.d.ts","sourceRoot":"","sources":["../../src/db/history-repo.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAEhD;;;;;;GAMG;AAEH,MAAM,MAAM,aAAa,GAAG,UAAU,GAAG,SAAS,CAAC;AAEnD,MAAM,WAAW,mBAAmB;IAClC,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,aAAa,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,eAAe,CAAC,EAAE,SAAS,GAAG,OAAO,GAAG,SAAS,CAAC;IAClD,gBAAgB,CAAC,EAAE,SAAS,GAAG,OAAO,GAAG,SAAS,CAAC;IACnD,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAC/B,WAAW,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IAC9B,kDAAkD;IAClD,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAChC;AAED,8DAA8D;AAC9D,MAAM,WAAW,SAAS;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,UAAU,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,eAAe,CAAC,EAAE,SAAS,GAAG,OAAO,GAAG,SAAS,CAAC;IAClD,gBAAgB,CAAC,EAAE,SAAS,GAAG,OAAO,GAAG,SAAS,CAAC;IACnD,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AA2CD,sFAAsF;AACtF,wBAAgB,aAAa,CAAC,MAAM,EAAE,mBAAmB,GAAG,MAAM,CAwBjE;AAED,6EAA6E;AAC7E,wBAAgB,QAAQ,CACtB,QAAQ,EAAE,MAAM,EAChB,SAAS,SAA2B,EACpC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,EAC3B,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI,EAC9B,WAAW,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI,EAC7B,KAAK,CAAC,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAC/E,MAAM,CAuBR;AAED;;;GAGG;AACH,wBAAgB,SAAS,CACvB,EAAE,EAAE,MAAM,EACV,MAAM,EAAE,UAAU,EAClB,UAAU,EAAE,MAAM,EAClB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,EACrB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,EACxB,MAAM,CAAC,EAAE,MAAM,EACf,OAAO,CAAC,EAAE,MAAM,EAChB,KAAK,CAAC,EAAE,MAAM,EACd,eAAe,CAAC,EAAE,SAAS,GAAG,OAAO,GAAG,SAAS,EACjD,gBAAgB,CAAC,EAAE,SAAS,GAAG,OAAO,GAAG,SAAS,GACjD,OAAO,CAiET;AAED;;;GAGG;AACH,wBAAgB,SAAS,CACvB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,UAAU,EAClB,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,EACrB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,EACxB,MAAM,CAAC,EAAE,MAAM,EACf,OAAO,CAAC,EAAE,MAAM,EAChB,KAAK,CAAC,EAAE,MAAM,EACd,eAAe,CAAC,EAAE,SAAS,GAAG,OAAO,GAAG,SAAS,EACjD,gBAAgB,CAAC,EAAE,SAAS,GAAG,OAAO,GAAG,SAAS,EAClD,UAAU,SAA2B,EACrC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,EAC3B,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI,EAC9B,WAAW,CAAC,EAAE,MAAM,EAAE,GAAG,IAAI,GAC5B,MAAM,CAgDR;AAED,uEAAuE;AACvE,wBAAgB,UAAU,CACxB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,EACxB,KAAK,SAAM,EACX,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,GACrB,mBAAmB,EAAE,CAgCvB;AAED;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CAAC,KAAK,SAAO,GAAG,mBAAmB,EAAE,CAa1E;AAED,kEAAkE;AAClE,wBAAgB,eAAe,CAC7B,SAAS,EAAE,MAAM,EACjB,KAAK,SAAO,GACX,mBAAmB,EAAE,CAWvB;AAED;;;;;;;;;GASG;AAEH,uFAAuF;AACvF,eAAO,MAAM,6BAA6B,QAAsB,CAAC;AAEjE,MAAM,WAAW,wBAAwB;IACvC,gDAAgD;IAChD,SAAS,EAAE,MAAM,CAAC;IAClB,kDAAkD;IAClD,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,yBAAyB;IACxC,mCAAmC;IACnC,GAAG,CAAC,EAAE,IAAI,CAAC;IACX,6DAA6D;IAC7D,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,uCAAuC;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,yCAAyC;IACzC,OAAO,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;CACpC;AAED,6EAA6E;AAC7E,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CASnD;AAED,wBAAgB,kBAAkB,CAChC,OAAO,GAAE,yBAA8B,GACtC,wBAAwB,CAkE1B;AAED,0DAA0D;AAC1D,wBAAgB,YAAY,IAAI,MAAM,CAGrC;AAED,gFAAgF;AAChF,wBAAgB,mBAAmB,IAAI,MAAM,CAK5C;AAED,kFAAkF;AAClF,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,2EAA2E;AAC3E,wBAAgB,kBAAkB,IAAI,oBAAoB,EAAE,CAa3D;AAED,wEAAwE;AACxE,MAAM,WAAW,mBAAmB;IAClC,gDAAgD;IAChD,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,6CAA6C;IAC7C,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,2CAA2C;IAC3C,GAAG,CAAC,EAAE,IAAI,CAAC;CACZ;AAED,MAAM,WAAW,kBAAkB;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,wEAAwE;IACxE,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,OAAO,GAAE,mBAAwB,GAAG,kBAAkB,CA2ClF;AAED;;;;GAIG;AACH,wBAAgB,aAAa,IAAI,OAAO,CAOvC"}
|
package/dist/db/history-repo.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { getDb, withTransaction } from "./index.js";
|
|
2
2
|
import { incrementMetricsUnsafe } from "./metrics-repo.js";
|
|
3
3
|
import { appendStructuredLog } from "../core/structured-log.js";
|
|
4
|
+
import os from "node:os";
|
|
4
5
|
function rowToRecord(row) {
|
|
5
6
|
const status = String(row.status);
|
|
6
7
|
return {
|
|
@@ -24,13 +25,16 @@ function rowToRecord(row) {
|
|
|
24
25
|
workspaceId: row.workspace_id == null ? null : String(row.workspace_id),
|
|
25
26
|
allowedTools: row.allowed_tools == null ? null : JSON.parse(String(row.allowed_tools)),
|
|
26
27
|
deniedTools: row.denied_tools == null ? null : JSON.parse(String(row.denied_tools)),
|
|
28
|
+
ownerPid: row.owner_pid == null ? null : Number(row.owner_pid),
|
|
29
|
+
ownerHost: row.owner_host == null ? null : String(row.owner_host),
|
|
30
|
+
ownerStartedAt: row.owner_started_at == null ? null : String(row.owner_started_at),
|
|
27
31
|
};
|
|
28
32
|
}
|
|
29
33
|
const INSERT_SQL = `INSERT INTO history (
|
|
30
34
|
task_name, status, started_at, finished_at, duration_ms, error,
|
|
31
35
|
tokens, cost_usd, model, pre_flight_status, post_flight_status, next_task,
|
|
32
|
-
workspace_id, allowed_tools, denied_tools
|
|
33
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
|
|
36
|
+
workspace_id, allowed_tools, denied_tools, owner_pid, owner_host, owner_started_at
|
|
37
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
|
|
34
38
|
function lastId(result) {
|
|
35
39
|
return Number(result.lastInsertRowid) || 0;
|
|
36
40
|
}
|
|
@@ -38,11 +42,11 @@ function lastId(result) {
|
|
|
38
42
|
export function insertHistory(record) {
|
|
39
43
|
const result = getDb()
|
|
40
44
|
.prepare(INSERT_SQL)
|
|
41
|
-
.run(record.taskName, record.status, record.startedAt, record.finishedAt ?? null, record.durationMs ?? null, record.error ?? null, record.tokens ?? null, record.costUsd ?? null, record.model ?? null, record.preFlightStatus ?? null, record.postFlightStatus ?? null, record.nextTask ?? null, record.workspaceId ?? null, record.allowedTools == null ? null : JSON.stringify(record.allowedTools), record.deniedTools == null ? null : JSON.stringify(record.deniedTools));
|
|
45
|
+
.run(record.taskName, record.status, record.startedAt, record.finishedAt ?? null, record.durationMs ?? null, record.error ?? null, record.tokens ?? null, record.costUsd ?? null, record.model ?? null, record.preFlightStatus ?? null, record.postFlightStatus ?? null, record.nextTask ?? null, record.workspaceId ?? null, record.allowedTools == null ? null : JSON.stringify(record.allowedTools), record.deniedTools == null ? null : JSON.stringify(record.deniedTools), record.ownerPid ?? null, record.ownerHost ?? null, record.ownerStartedAt ?? null);
|
|
42
46
|
return lastId(result);
|
|
43
47
|
}
|
|
44
48
|
/** Начало запуска: строка с finished_at = NULL (переживёт сбой процесса). */
|
|
45
|
-
export function startRun(taskName, startedAt = new Date().toISOString(), workspaceId, allowedTools, deniedTools) {
|
|
49
|
+
export function startRun(taskName, startedAt = new Date().toISOString(), workspaceId, allowedTools, deniedTools, owner) {
|
|
46
50
|
const id = insertHistory({
|
|
47
51
|
taskName,
|
|
48
52
|
status: "running",
|
|
@@ -51,6 +55,9 @@ export function startRun(taskName, startedAt = new Date().toISOString(), workspa
|
|
|
51
55
|
workspaceId,
|
|
52
56
|
allowedTools,
|
|
53
57
|
deniedTools,
|
|
58
|
+
ownerPid: owner?.pid ?? null,
|
|
59
|
+
ownerHost: owner?.host ?? null,
|
|
60
|
+
ownerStartedAt: owner?.startedAt ?? null,
|
|
54
61
|
});
|
|
55
62
|
appendStructuredLog({
|
|
56
63
|
event: "run.started",
|
|
@@ -125,7 +132,9 @@ export function recordRun(taskName, status, startedAt, durationMs, error, nextTa
|
|
|
125
132
|
const id = withTransaction(() => {
|
|
126
133
|
const result = getDb()
|
|
127
134
|
.prepare(INSERT_SQL)
|
|
128
|
-
.run(taskName, status, startedAt, finishedAt, safeDuration, error ?? null, tokens ?? null, costUsd ?? null, model ?? null, preFlightStatus ?? null, postFlightStatus ?? null, nextTask ?? null, workspaceId ?? null, allowedTools == null ? null : JSON.stringify(allowedTools), deniedTools == null ? null : JSON.stringify(deniedTools)
|
|
135
|
+
.run(taskName, status, startedAt, finishedAt, safeDuration, error ?? null, tokens ?? null, costUsd ?? null, model ?? null, preFlightStatus ?? null, postFlightStatus ?? null, nextTask ?? null, workspaceId ?? null, allowedTools == null ? null : JSON.stringify(allowedTools), deniedTools == null ? null : JSON.stringify(deniedTools),
|
|
136
|
+
// owner-колонки относятся только к незавершённым запускам (startRun).
|
|
137
|
+
null, null, null);
|
|
129
138
|
incrementMetricsUnsafe(taskName, status, tokens, costUsd);
|
|
130
139
|
return lastId(result);
|
|
131
140
|
});
|
|
@@ -212,27 +221,81 @@ export function getHistorySince(timestamp, limit = 5000) {
|
|
|
212
221
|
return rows.map(rowToRecord);
|
|
213
222
|
}
|
|
214
223
|
/**
|
|
215
|
-
* Crash recovery
|
|
216
|
-
*
|
|
217
|
-
*
|
|
224
|
+
* Crash recovery для истории.
|
|
225
|
+
*
|
|
226
|
+
* Незавершённая запись (finished_at IS NULL) финализируется как error, только
|
|
227
|
+
* если её владелец больше не может её завершить:
|
|
228
|
+
* - владелец известен, жив и на этом же хосте → запись оставляем (B1);
|
|
229
|
+
* - владелец известен и мёртв → финализируем сразу;
|
|
230
|
+
* - владелец неизвестен (legacy-строка или другой хост) → финализируем,
|
|
231
|
+
* только когда запись старше `maxUnverifiableAgeMs`.
|
|
218
232
|
*/
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
233
|
+
/** Возраст, после которого запись с непроверяемым владельцем считается осиротевшей. */
|
|
234
|
+
export const UNVERIFIABLE_OWNER_MAX_AGE_MS = 24 * 60 * 60 * 1000;
|
|
235
|
+
/** Проверяет существование процесса: signal 0 не убивает, только пробует. */
|
|
236
|
+
export function isProcessAlive(pid) {
|
|
237
|
+
if (!Number.isInteger(pid) || pid <= 0)
|
|
238
|
+
return false;
|
|
239
|
+
try {
|
|
240
|
+
process.kill(pid, 0);
|
|
241
|
+
return true;
|
|
242
|
+
}
|
|
243
|
+
catch (error) {
|
|
244
|
+
// EPERM — процесс существует, но сигнал запрещён; ESRCH — процесса нет.
|
|
245
|
+
return error?.code === "EPERM";
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
export function recoverInterrupted(options = {}) {
|
|
249
|
+
const now = options.now ?? new Date();
|
|
250
|
+
const nowIso = now.toISOString();
|
|
251
|
+
const maxAge = typeof options.maxUnverifiableAgeMs === "number" &&
|
|
252
|
+
Number.isFinite(options.maxUnverifiableAgeMs)
|
|
253
|
+
? Math.max(0, options.maxUnverifiableAgeMs)
|
|
254
|
+
: UNVERIFIABLE_OWNER_MAX_AGE_MS;
|
|
255
|
+
const hostname = options.hostname ?? os.hostname();
|
|
256
|
+
const isAlive = options.isAlive ?? isProcessAlive;
|
|
257
|
+
const db = getDb();
|
|
258
|
+
const rows = db
|
|
259
|
+
.prepare(`SELECT id, started_at, owner_pid, owner_host, owner_started_at
|
|
260
|
+
FROM history WHERE finished_at IS NULL`)
|
|
261
|
+
.all();
|
|
262
|
+
const update = db.prepare(`UPDATE history SET
|
|
263
|
+
status = 'error',
|
|
264
|
+
error = CASE
|
|
265
|
+
WHEN error IS NULL OR error = '' THEN 'Прервано (сбой/перезагрузка)'
|
|
266
|
+
ELSE error
|
|
267
|
+
END,
|
|
268
|
+
finished_at = COALESCE(finished_at, ?),
|
|
269
|
+
duration_ms = COALESCE(
|
|
270
|
+
duration_ms,
|
|
271
|
+
CAST((julianday(?) - julianday(started_at)) * 86400000 AS INTEGER)
|
|
272
|
+
)
|
|
273
|
+
WHERE id = ? AND finished_at IS NULL`);
|
|
274
|
+
let recovered = 0;
|
|
275
|
+
let keptAlive = 0;
|
|
276
|
+
withTransaction(() => {
|
|
277
|
+
for (const row of rows) {
|
|
278
|
+
const ownerPid = row.owner_pid == null ? null : Number(row.owner_pid);
|
|
279
|
+
const ownerHost = row.owner_host == null ? null : String(row.owner_host);
|
|
280
|
+
const verifiable = ownerPid != null && ownerPid > 0 && ownerHost != null && ownerHost === hostname;
|
|
281
|
+
if (verifiable && ownerPid != null && isAlive(ownerPid)) {
|
|
282
|
+
keptAlive += 1;
|
|
283
|
+
continue;
|
|
284
|
+
}
|
|
285
|
+
if (!verifiable) {
|
|
286
|
+
const since = Date.parse(String(row.owner_started_at ?? row.started_at ?? ""));
|
|
287
|
+
const age = Number.isFinite(since) ? now.getTime() - since : Infinity;
|
|
288
|
+
if (age < maxAge) {
|
|
289
|
+
keptAlive += 1;
|
|
290
|
+
continue;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
const result = update.run(nowIso, nowIso, row.id);
|
|
294
|
+
if (Number(result.changes) > 0)
|
|
295
|
+
recovered += 1;
|
|
296
|
+
}
|
|
297
|
+
});
|
|
298
|
+
return { recovered, keptAlive };
|
|
236
299
|
}
|
|
237
300
|
/** Количество записей истории (для миграции и тестов). */
|
|
238
301
|
export function countHistory() {
|
package/dist/db/metrics-repo.js
CHANGED
|
@@ -17,8 +17,8 @@ function rowToMetrics(row) {
|
|
|
17
17
|
}
|
|
18
18
|
/** Инкремент метрик без собственной транзакции (для использования внутри recordRun). */
|
|
19
19
|
export function incrementMetricsUnsafe(taskName, status, tokens = 0, costUsd = 0) {
|
|
20
|
-
// error — только ошибки и таймауты; partial
|
|
21
|
-
//
|
|
20
|
+
// error — только ошибки и таймауты; partial и skipped хранятся отдельно
|
|
21
|
+
// (skipped увеличивает только total: запуск не расходовал токены и не завершился).
|
|
22
22
|
const success = status === "success" ? 1 : 0;
|
|
23
23
|
const partial = status === "partial" ? 1 : 0;
|
|
24
24
|
const error = status === "error" || status === "timeout" ? 1 : 0;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"recovery.d.ts","sourceRoot":"","sources":["../../src/db/recovery.ts"],"names":[],"mappings":"AASA;;;GAGG;AACH,wBAAgB,sBAAsB,IAAI,MAAM,
|
|
1
|
+
{"version":3,"file":"recovery.d.ts","sourceRoot":"","sources":["../../src/db/recovery.ts"],"names":[],"mappings":"AASA;;;GAGG;AACH,wBAAgB,sBAAsB,IAAI,MAAM,CAS/C;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,IAAI,OAAO,CAkB1C"}
|
package/dist/db/recovery.js
CHANGED
|
@@ -10,11 +10,12 @@ const JSON_FILES = ["tasks.json", "history.json", "metrics.json", "templates.jso
|
|
|
10
10
|
* Вызывается при setup() до запуска catch-up и файловых триггеров.
|
|
11
11
|
*/
|
|
12
12
|
export function recoverInterruptedRuns() {
|
|
13
|
-
const
|
|
14
|
-
if (
|
|
15
|
-
console.log(`[recovery] Восстановлено ${
|
|
13
|
+
const { recovered, keptAlive } = recoverInterrupted();
|
|
14
|
+
if (recovered > 0 || keptAlive > 0) {
|
|
15
|
+
console.log(`[recovery] Восстановлено ${recovered} незавершённых записей истории — помечены как error.` +
|
|
16
|
+
(keptAlive > 0 ? ` Оставлено живым владельцам: ${keptAlive}.` : ""));
|
|
16
17
|
}
|
|
17
|
-
return
|
|
18
|
+
return recovered;
|
|
18
19
|
}
|
|
19
20
|
/**
|
|
20
21
|
* Если БД пуста, а .json.bak остались (миграция не удалась или БД удалили),
|
package/dist/db/run-journal.d.ts
CHANGED
|
@@ -9,10 +9,20 @@ import { type RunRecord, type RunStepRecord } from "./runs-repo.js";
|
|
|
9
9
|
*/
|
|
10
10
|
/** Текст ошибки, которым помечается шаг, прерванный падением процесса. */
|
|
11
11
|
export declare const STEP_INTERRUPTED_ERROR = "\u041F\u0440\u0435\u0440\u0432\u0430\u043D\u043E \u043F\u0430\u0434\u0435\u043D\u0438\u0435\u043C \u043F\u0440\u043E\u0446\u0435\u0441\u0441\u0430";
|
|
12
|
+
/**
|
|
13
|
+
* B12: срок lease шага по умолчанию. Живой процесс продлевает lease
|
|
14
|
+
* (heartbeat в `runStepStatefullyAsync` или явный `renewStepLease`), поэтому
|
|
15
|
+
* recovery финализирует шаг только после истечения этого срока.
|
|
16
|
+
*/
|
|
17
|
+
export declare const STEP_LEASE_MS = 60000;
|
|
18
|
+
/** B12: идентификатор текущего процесса — `host:pid:startedAt`. */
|
|
19
|
+
export declare function getStepOwnerId(): string;
|
|
12
20
|
/** Опции начала попытки. */
|
|
13
21
|
export interface BeginStepOptions {
|
|
14
22
|
/** Начать запуск, если он ещё в статусе pending (по умолчанию true). */
|
|
15
23
|
startRun?: boolean;
|
|
24
|
+
/** B12: срок lease в миллисекундах (по умолчанию {@link STEP_LEASE_MS}). */
|
|
25
|
+
leaseMs?: number;
|
|
16
26
|
}
|
|
17
27
|
/** Результат выполнения шага через `runStepStatefully`. */
|
|
18
28
|
export interface StepExecution<T> {
|
|
@@ -33,6 +43,14 @@ export interface StepExecution<T> {
|
|
|
33
43
|
* попытки увеличивается, а не начинается заново.
|
|
34
44
|
*/
|
|
35
45
|
export declare function beginStepAttempt(runId: string, stepName: string, options?: BeginStepOptions): RunStepRecord;
|
|
46
|
+
/**
|
|
47
|
+
* B12: продлевает lease шага текущего процесса.
|
|
48
|
+
*
|
|
49
|
+
* Нужен долгим синхронным шагам: без продления recovery сочтёт шаг
|
|
50
|
+
* осиротевшим после {@link STEP_LEASE_MS}. Возвращает false, если шаг уже
|
|
51
|
+
* не `running`.
|
|
52
|
+
*/
|
|
53
|
+
export declare function renewStepLease(runId: string, stepName: string, leaseMs?: number): boolean;
|
|
36
54
|
/** Закрывает попытку успехом: running → passed, сохраняет output. */
|
|
37
55
|
export declare function recordStepSuccess(runId: string, stepName: string, output?: unknown): RunStepRecord;
|
|
38
56
|
/** Закрывает попытку ошибкой: running → failed, сохраняет текст ошибки. */
|
|
@@ -45,14 +63,24 @@ export declare function recordStepFailure(runId: string, stepName: string, error
|
|
|
45
63
|
export declare function runStepStatefully<T>(runId: string, stepName: string, body: () => T, options?: BeginStepOptions): StepExecution<T>;
|
|
46
64
|
/** Асинхронный вариант `runStepStatefully` для шагов с I/O. */
|
|
47
65
|
export declare function runStepStatefullyAsync<T>(runId: string, stepName: string, body: () => Promise<T> | T, options?: BeginStepOptions): Promise<StepExecution<T>>;
|
|
66
|
+
export interface RecoverStepsOptions {
|
|
67
|
+
/**
|
|
68
|
+
* B12: текущий момент для проверки lease. По умолчанию now; тесты и
|
|
69
|
+
* отложенный recovery передают «будущее», чтобы lease считался истёкшим.
|
|
70
|
+
*/
|
|
71
|
+
now?: string | Date;
|
|
72
|
+
}
|
|
48
73
|
/**
|
|
49
|
-
* Помечает
|
|
74
|
+
* Помечает осиротевшие `running`-шаги как failed, сохраняя `attempt`.
|
|
75
|
+
*
|
|
76
|
+
* B12: шаг финализируется только при отсутствии lease (legacy-строки) или
|
|
77
|
+
* истёкшем `lease_until`. Шаг с живым lease принадлежит работающему процессу
|
|
78
|
+
* и не трогается. Возвращает обновлённые записи.
|
|
50
79
|
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
* падение процесса и восстановление.
|
|
80
|
+
* `attempt` не сбрасывается — следующая попытка получит `attempt + 1` через
|
|
81
|
+
* `retryStep`, поэтому номер попытки переживает падение процесса и восстановление.
|
|
54
82
|
*/
|
|
55
|
-
export declare function recoverInterruptedSteps(runId?: string): RunStepRecord[];
|
|
83
|
+
export declare function recoverInterruptedSteps(runId?: string, options?: RecoverStepsOptions): RunStepRecord[];
|
|
56
84
|
/** Список шагов в статусе running (для диагностики и восстановления). */
|
|
57
85
|
export declare function listRunningSteps(runId?: string): RunStepRecord[];
|
|
58
86
|
/** Запуск по id либо ошибка (удобно для журнала и тестов). */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-journal.d.ts","sourceRoot":"","sources":["../../src/db/run-journal.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"run-journal.d.ts","sourceRoot":"","sources":["../../src/db/run-journal.ts"],"names":[],"mappings":"AAEA,OAAO,EASL,KAAK,SAAS,EACd,KAAK,aAAa,EAEnB,MAAM,gBAAgB,CAAC;AAExB;;;;;;;GAOG;AAEH,0EAA0E;AAC1E,eAAO,MAAM,sBAAsB,uJAA+B,CAAC;AAEnE;;;;GAIG;AACH,eAAO,MAAM,aAAa,QAAS,CAAC;AAIpC,mEAAmE;AACnE,wBAAgB,cAAc,IAAI,MAAM,CAKvC;AAED,4BAA4B;AAC5B,MAAM,WAAW,gBAAgB;IAC/B,wEAAwE;IACxE,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,4EAA4E;IAC5E,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,2DAA2D;AAC3D,MAAM,WAAW,aAAa,CAAC,CAAC;IAC9B,uCAAuC;IACvC,IAAI,EAAE,aAAa,CAAC;IACpB,2CAA2C;IAC3C,MAAM,EAAE,CAAC,CAAC;IACV,+CAA+C;IAC/C,OAAO,EAAE,MAAM,CAAC;CACjB;AAWD;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,gBAAqB,GAC7B,aAAa,CAef;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAC5B,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,MAAsB,GAC9B,OAAO,CAIT;AAUD,qEAAqE;AACrE,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,OAAO,GACf,aAAa,CAGf;AAED,2EAA2E;AAC3E,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,OAAO,GACb,aAAa,CAGf;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,CAAC,EACjC,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,CAAC,EACb,OAAO,GAAE,gBAAqB,GAC7B,aAAa,CAAC,CAAC,CAAC,CAWlB;AAED,+DAA+D;AAC/D,wBAAsB,sBAAsB,CAAC,CAAC,EAC5C,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,EAC1B,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CA4B3B;AAED,MAAM,WAAW,mBAAmB;IAClC;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACrB;AAED;;;;;;;;;GASG;AACH,wBAAgB,uBAAuB,CACrC,KAAK,CAAC,EAAE,MAAM,EACd,OAAO,GAAE,mBAAwB,GAChC,aAAa,EAAE,CAuBjB;AAED,yEAAyE;AACzE,wBAAgB,gBAAgB,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,aAAa,EAAE,CAgBhE;AAED,8DAA8D;AAC9D,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CAInD"}
|
package/dist/db/run-journal.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
import os from "node:os";
|
|
1
2
|
import { getDb, withTransaction } from "./index.js";
|
|
2
|
-
import { finishStep, getRun, getStep, retryStep, startRun, startStep, } from "./runs-repo.js";
|
|
3
|
+
import { finishStep, getRun, getStep, renewStepLease as renewStepLeaseRow, retryStep, startRun, startStep, } from "./runs-repo.js";
|
|
3
4
|
/**
|
|
4
5
|
* Журнал выполнения шагов (Sprint 36.1.b).
|
|
5
6
|
*
|
|
@@ -10,6 +11,20 @@ import { finishStep, getRun, getStep, retryStep, startRun, startStep, } from "./
|
|
|
10
11
|
*/
|
|
11
12
|
/** Текст ошибки, которым помечается шаг, прерванный падением процесса. */
|
|
12
13
|
export const STEP_INTERRUPTED_ERROR = "Прервано падением процесса";
|
|
14
|
+
/**
|
|
15
|
+
* B12: срок lease шага по умолчанию. Живой процесс продлевает lease
|
|
16
|
+
* (heartbeat в `runStepStatefullyAsync` или явный `renewStepLease`), поэтому
|
|
17
|
+
* recovery финализирует шаг только после истечения этого срока.
|
|
18
|
+
*/
|
|
19
|
+
export const STEP_LEASE_MS = 60000;
|
|
20
|
+
let cachedOwnerId = null;
|
|
21
|
+
/** B12: идентификатор текущего процесса — `host:pid:startedAt`. */
|
|
22
|
+
export function getStepOwnerId() {
|
|
23
|
+
if (!cachedOwnerId) {
|
|
24
|
+
cachedOwnerId = `${os.hostname()}:${process.pid}:${nowIso()}`;
|
|
25
|
+
}
|
|
26
|
+
return cachedOwnerId;
|
|
27
|
+
}
|
|
13
28
|
function nowIso() {
|
|
14
29
|
return new Date().toISOString();
|
|
15
30
|
}
|
|
@@ -34,14 +49,35 @@ export function beginStepAttempt(runId, stepName, options = {}) {
|
|
|
34
49
|
startRun(runId);
|
|
35
50
|
}
|
|
36
51
|
const step = requireStep(runId, stepName);
|
|
52
|
+
const lease = buildLease(options.leaseMs);
|
|
37
53
|
if (step.status === "failed") {
|
|
38
|
-
retryStep(step.id);
|
|
54
|
+
retryStep(step.id, undefined, lease);
|
|
39
55
|
}
|
|
40
56
|
else {
|
|
41
|
-
startStep(step.id);
|
|
57
|
+
startStep(step.id, undefined, lease);
|
|
42
58
|
}
|
|
43
59
|
return requireStep(runId, stepName);
|
|
44
60
|
}
|
|
61
|
+
/**
|
|
62
|
+
* B12: продлевает lease шага текущего процесса.
|
|
63
|
+
*
|
|
64
|
+
* Нужен долгим синхронным шагам: без продления recovery сочтёт шаг
|
|
65
|
+
* осиротевшим после {@link STEP_LEASE_MS}. Возвращает false, если шаг уже
|
|
66
|
+
* не `running`.
|
|
67
|
+
*/
|
|
68
|
+
export function renewStepLease(runId, stepName, leaseMs = STEP_LEASE_MS) {
|
|
69
|
+
const step = requireStep(runId, stepName);
|
|
70
|
+
if (step.status !== "running")
|
|
71
|
+
return false;
|
|
72
|
+
return renewStepLeaseRow(step.id, buildLease(leaseMs));
|
|
73
|
+
}
|
|
74
|
+
function buildLease(leaseMs = STEP_LEASE_MS) {
|
|
75
|
+
const duration = Number.isFinite(leaseMs) && leaseMs > 0 ? leaseMs : STEP_LEASE_MS;
|
|
76
|
+
return {
|
|
77
|
+
ownerId: getStepOwnerId(),
|
|
78
|
+
leaseUntil: new Date(Date.now() + duration).toISOString(),
|
|
79
|
+
};
|
|
80
|
+
}
|
|
45
81
|
/** Закрывает попытку успехом: running → passed, сохраняет output. */
|
|
46
82
|
export function recordStepSuccess(runId, stepName, output) {
|
|
47
83
|
completeStep(runId, stepName, "passed", { output });
|
|
@@ -72,37 +108,65 @@ export function runStepStatefully(runId, stepName, body, options = {}) {
|
|
|
72
108
|
}
|
|
73
109
|
/** Асинхронный вариант `runStepStatefully` для шагов с I/O. */
|
|
74
110
|
export async function runStepStatefullyAsync(runId, stepName, body, options = {}) {
|
|
111
|
+
const leaseMs = options.leaseMs ?? STEP_LEASE_MS;
|
|
75
112
|
const attempt = beginStepAttempt(runId, stepName, options).attempt;
|
|
113
|
+
// B12: heartbeat продлевает lease, пока тело шага реально выполняется.
|
|
114
|
+
const timer = setInterval(() => {
|
|
115
|
+
try {
|
|
116
|
+
renewStepLease(runId, stepName, leaseMs);
|
|
117
|
+
}
|
|
118
|
+
catch {
|
|
119
|
+
// Шаг уже финализирован — продлевать нечего.
|
|
120
|
+
}
|
|
121
|
+
}, Math.max(10, Math.floor(leaseMs / 3)));
|
|
122
|
+
timer.unref?.();
|
|
76
123
|
let result;
|
|
77
124
|
try {
|
|
78
|
-
|
|
125
|
+
try {
|
|
126
|
+
result = await body();
|
|
127
|
+
}
|
|
128
|
+
catch (err) {
|
|
129
|
+
recordStepFailure(runId, stepName, err);
|
|
130
|
+
throw err;
|
|
131
|
+
}
|
|
79
132
|
}
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
throw err;
|
|
133
|
+
finally {
|
|
134
|
+
clearInterval(timer);
|
|
83
135
|
}
|
|
84
136
|
const step = recordStepSuccess(runId, stepName, result);
|
|
85
137
|
return { step, result, attempt };
|
|
86
138
|
}
|
|
87
139
|
/**
|
|
88
|
-
* Помечает
|
|
140
|
+
* Помечает осиротевшие `running`-шаги как failed, сохраняя `attempt`.
|
|
141
|
+
*
|
|
142
|
+
* B12: шаг финализируется только при отсутствии lease (legacy-строки) или
|
|
143
|
+
* истёкшем `lease_until`. Шаг с живым lease принадлежит работающему процессу
|
|
144
|
+
* и не трогается. Возвращает обновлённые записи.
|
|
89
145
|
*
|
|
90
|
-
*
|
|
91
|
-
*
|
|
92
|
-
* падение процесса и восстановление.
|
|
146
|
+
* `attempt` не сбрасывается — следующая попытка получит `attempt + 1` через
|
|
147
|
+
* `retryStep`, поэтому номер попытки переживает падение процесса и восстановление.
|
|
93
148
|
*/
|
|
94
|
-
export function recoverInterruptedSteps(runId) {
|
|
149
|
+
export function recoverInterruptedSteps(runId, options = {}) {
|
|
95
150
|
const running = listRunningSteps(runId);
|
|
96
151
|
if (running.length === 0)
|
|
97
152
|
return [];
|
|
153
|
+
const nowMs = options.now ? new Date(options.now).getTime() : Date.now();
|
|
154
|
+
const expired = running.filter((step) => {
|
|
155
|
+
if (!step.leaseUntil)
|
|
156
|
+
return true;
|
|
157
|
+
const until = Date.parse(step.leaseUntil);
|
|
158
|
+
return Number.isNaN(until) || until <= nowMs;
|
|
159
|
+
});
|
|
160
|
+
if (expired.length === 0)
|
|
161
|
+
return [];
|
|
98
162
|
return withTransaction(() => {
|
|
99
163
|
const timestamp = nowIso();
|
|
100
164
|
const update = getDb().prepare(`UPDATE run_steps SET status = 'failed', error = ?, finished_at = ?,
|
|
101
165
|
updated_at = ? WHERE id = ? AND status = 'running'`);
|
|
102
|
-
for (const step of
|
|
166
|
+
for (const step of expired) {
|
|
103
167
|
update.run(STEP_INTERRUPTED_ERROR, timestamp, timestamp, step.id);
|
|
104
168
|
}
|
|
105
|
-
return
|
|
169
|
+
return expired.map((step) => requireStep(step.runId, step.stepName));
|
|
106
170
|
});
|
|
107
171
|
}
|
|
108
172
|
/** Список шагов в статусе running (для диагностики и восстановления). */
|
package/dist/db/run-resume.d.ts
CHANGED
|
@@ -16,6 +16,11 @@ export interface ResumeOptions {
|
|
|
16
16
|
handlers: Record<string, StepHandler>;
|
|
17
17
|
/** Восстанавливать зависшие `running`-шаги перед продолжением (по умолчанию true). */
|
|
18
18
|
recover?: boolean;
|
|
19
|
+
/**
|
|
20
|
+
* B12: момент для проверки lease при восстановлении. По умолчанию now.
|
|
21
|
+
* Передавайте будущее время, чтобы считать lease зависшего шага истёкшим.
|
|
22
|
+
*/
|
|
23
|
+
recoverNow?: string | Date;
|
|
19
24
|
/** Останавливаться на первой ошибке (по умолчанию true). */
|
|
20
25
|
stopOnError?: boolean;
|
|
21
26
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-resume.d.ts","sourceRoot":"","sources":["../../src/db/run-resume.ts"],"names":[],"mappings":"AAAA,OAAO,EAML,KAAK,SAAS,EACd,KAAK,aAAa,EACnB,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"run-resume.d.ts","sourceRoot":"","sources":["../../src/db/run-resume.ts"],"names":[],"mappings":"AAAA,OAAO,EAML,KAAK,SAAS,EACd,KAAK,aAAa,EACnB,MAAM,gBAAgB,CAAC;AAQxB,OAAO,EAIL,KAAK,WAAW,EACjB,MAAM,kBAAkB,CAAC;AAE1B;;;;;;;GAOG;AAEH,8EAA8E;AAC9E,MAAM,MAAM,WAAW,GAAG,CAAC,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,WAAW,KAAK,OAAO,CAAC;AAEjF,2BAA2B;AAC3B,MAAM,WAAW,aAAa;IAC5B,+EAA+E;IAC/E,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACtC,sFAAsF;IACtF,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,4DAA4D;IAC5D,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB;AAED,uCAAuC;AACvC,MAAM,WAAW,YAAY;IAC3B,6CAA6C;IAC7C,GAAG,EAAE,SAAS,CAAC;IACf,iEAAiE;IACjE,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,qEAAqE;IACrE,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,oCAAoC;IACpC,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,6CAA6C;IAC7C,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,qEAAqE;IACrE,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,qDAAqD;IACrD,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,kCAAkC;IAClC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAOD,gFAAgF;AAChF,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CAGlE;AA4CD;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,GAAG,YAAY,CA4CjF;AAED,2DAA2D;AAC3D,wBAAsB,SAAS,CAC7B,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,YAAY,CAAC,CA4CvB"}
|
package/dist/db/run-resume.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { finishRun, listRunSteps, reopenRun, startRun, } from "./runs-repo.js";
|
|
2
|
-
import { recoverInterruptedSteps, requireRun, runStepStatefully, runStepStatefullyAsync, } from "./run-journal.js";
|
|
2
|
+
import { listRunningSteps, recoverInterruptedSteps, requireRun, runStepStatefully, runStepStatefullyAsync, } from "./run-journal.js";
|
|
3
3
|
import { consumeInjections, consumeOrphanInjections, pendingInjectionsForStep, } from "./run-context.js";
|
|
4
4
|
/** Завершён ли шаг успешно (повторять не нужно). */
|
|
5
5
|
function isStepDone(status) {
|
|
@@ -16,7 +16,14 @@ function prepareResume(runId, options) {
|
|
|
16
16
|
throw new Error(`Запуск отменён, возобновление невозможно: ${runId}`);
|
|
17
17
|
}
|
|
18
18
|
if (options.recover !== false)
|
|
19
|
-
recoverInterruptedSteps(runId);
|
|
19
|
+
recoverInterruptedSteps(runId, { now: options.recoverNow });
|
|
20
|
+
// B12: шаг с живым lease принадлежит работающему процессу. Возобновлять его
|
|
21
|
+
// здесь нельзя — иначе два исполнителя возьмут один шаг.
|
|
22
|
+
const active = listRunningSteps(runId);
|
|
23
|
+
if (active.length > 0) {
|
|
24
|
+
const step = active[0];
|
|
25
|
+
throw new Error(`Шаг ${step.stepName} выполняется другим процессом (lease до ${step.leaseUntil ?? "неизвестно"}); возобновление отложено.`);
|
|
26
|
+
}
|
|
20
27
|
const steps = listRunSteps(runId);
|
|
21
28
|
const startIndex = steps.findIndex((step) => !isStepDone(step.status));
|
|
22
29
|
if (startIndex >= 0) {
|
package/dist/db/runs-repo.d.ts
CHANGED
|
@@ -39,7 +39,12 @@ export interface RunStepRecord {
|
|
|
39
39
|
position: number;
|
|
40
40
|
status: StepStatus;
|
|
41
41
|
attempt: number;
|
|
42
|
+
/** Ключ идемпотентности (36.3); уникален в пределах запуска. */
|
|
42
43
|
idempotencyKey: string | null;
|
|
44
|
+
/** B12: идентификатор процесса-владельца шага (`host:pid:startedAt`). */
|
|
45
|
+
ownerId: string | null;
|
|
46
|
+
/** B12: момент, до которого владелец обязан продлить lease (ISO). */
|
|
47
|
+
leaseUntil: string | null;
|
|
43
48
|
startedAt: string | null;
|
|
44
49
|
finishedAt: string | null;
|
|
45
50
|
error: string | null;
|
|
@@ -71,6 +76,11 @@ export interface FinishOptions {
|
|
|
71
76
|
error?: string | null;
|
|
72
77
|
output?: unknown;
|
|
73
78
|
}
|
|
79
|
+
/** B12: владелец шага и срок его lease. */
|
|
80
|
+
export interface StepLease {
|
|
81
|
+
ownerId: string;
|
|
82
|
+
leaseUntil: string;
|
|
83
|
+
}
|
|
74
84
|
/** Создаёт запуск и (опционально) его шаги одной транзакцией. */
|
|
75
85
|
export declare function createRun(options: CreateRunOptions): RunRecord;
|
|
76
86
|
/** Добавляет шаг в существующий запуск. Имя шага уникально внутри запуска. */
|
|
@@ -107,7 +117,7 @@ export declare function listRunSteps(runId: string): RunStepRecord[];
|
|
|
107
117
|
/** Поиск шага по ключу идемпотентности (36.3) в пределах запуска. */
|
|
108
118
|
export declare function findStepByIdempotencyKey(runId: string, idempotencyKey: string): RunStepRecord | undefined;
|
|
109
119
|
/** pending → running. Возвращает false, если шаг не найден. */
|
|
110
|
-
export declare function startStep(id: string, startedAt?: string): boolean;
|
|
120
|
+
export declare function startStep(id: string, startedAt?: string, lease?: StepLease): boolean;
|
|
111
121
|
/** running → passed/failed; pending → skipped допустим. */
|
|
112
122
|
export declare function finishStep(id: string, status: Extract<StepStatus, "passed" | "failed" | "skipped">, options?: FinishOptions): boolean;
|
|
113
123
|
/**
|
|
@@ -116,7 +126,14 @@ export declare function finishStep(id: string, status: Extract<StepStatus, "pass
|
|
|
116
126
|
* Это единственный переход из финального статуса шага. Предыдущая попытка
|
|
117
127
|
* остаётся видимой в счётчике attempt; для 36.4 (DLQ) здесь появится лимит.
|
|
118
128
|
*/
|
|
119
|
-
export declare function retryStep(id: string, startedAt?: string): boolean;
|
|
129
|
+
export declare function retryStep(id: string, startedAt?: string, lease?: StepLease): boolean;
|
|
130
|
+
/**
|
|
131
|
+
* B12: продлевает lease выполняющегося шага.
|
|
132
|
+
*
|
|
133
|
+
* Возвращает false, если шаг уже не `running` (например, recovery успел
|
|
134
|
+
* финализировать его после истечения lease): продлевать нечего.
|
|
135
|
+
*/
|
|
136
|
+
export declare function renewStepLease(id: string, lease: StepLease): boolean;
|
|
120
137
|
/** Количество шагов запуска (для health/отчётов). */
|
|
121
138
|
export declare function countRunSteps(runId: string): number;
|
|
122
139
|
/** Удаляет запуск; шаги удаляются каскадом (ON DELETE CASCADE). */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runs-repo.d.ts","sourceRoot":"","sources":["../../src/db/runs-repo.ts"],"names":[],"mappings":"AAGA;;;;;;;;;GASG;AAEH,8FAA8F;AAC9F,MAAM,MAAM,SAAS,GAAG,SAAS,GAAG,SAAS,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,WAAW,CAAC;AAE7F,qFAAqF;AACrF,MAAM,MAAM,UAAU,GAAG,SAAS,GAAG,SAAS,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;AA8BjF,qBAAa,sBAAuB,SAAQ,KAAK;IAE7C,QAAQ,CAAC,MAAM,EAAE,KAAK,GAAG,MAAM;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM;IACrB,QAAQ,CAAC,EAAE,EAAE,MAAM;gBAFV,MAAM,EAAE,KAAK,GAAG,MAAM,EACtB,IAAI,EAAE,MAAM,EACZ,EAAE,EAAE,MAAM;CAKtB;AAED,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,SAAS,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,oDAAoD;IACpD,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,gBAAgB;IAC/B,oEAAoE;IACpE,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,8CAA8C;IAC9C,MAAM,CAAC,EAAE,OAAO,CAAC,SAAS,EAAE,SAAS,GAAG,SAAS,CAAC,CAAC;IACnD,4DAA4D;IAC5D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wDAAwD;IACxD,KAAK,CAAC,EAAE,iBAAiB,EAAE,CAAC;CAC7B;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,8CAA8C;IAC9C,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;
|
|
1
|
+
{"version":3,"file":"runs-repo.d.ts","sourceRoot":"","sources":["../../src/db/runs-repo.ts"],"names":[],"mappings":"AAGA;;;;;;;;;GASG;AAEH,8FAA8F;AAC9F,MAAM,MAAM,SAAS,GAAG,SAAS,GAAG,SAAS,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,GAAG,WAAW,CAAC;AAE7F,qFAAqF;AACrF,MAAM,MAAM,UAAU,GAAG,SAAS,GAAG,SAAS,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;AA8BjF,qBAAa,sBAAuB,SAAQ,KAAK;IAE7C,QAAQ,CAAC,MAAM,EAAE,KAAK,GAAG,MAAM;IAC/B,QAAQ,CAAC,IAAI,EAAE,MAAM;IACrB,QAAQ,CAAC,EAAE,EAAE,MAAM;gBAFV,MAAM,EAAE,KAAK,GAAG,MAAM,EACtB,IAAI,EAAE,MAAM,EACZ,EAAE,EAAE,MAAM;CAKtB;AAED,MAAM,WAAW,SAAS;IACxB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,SAAS,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,oDAAoD;IACpD,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,gEAAgE;IAChE,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,yEAAyE;IACzE,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,qEAAqE;IACrE,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,gBAAgB;IAC/B,oEAAoE;IACpE,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,8CAA8C;IAC9C,MAAM,CAAC,EAAE,OAAO,CAAC,SAAS,EAAE,SAAS,GAAG,SAAS,CAAC,CAAC;IACnD,4DAA4D;IAC5D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wDAAwD;IACxD,KAAK,CAAC,EAAE,iBAAiB,EAAE,CAAC;CAC7B;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,8CAA8C;IAC9C,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,2CAA2C;AAC3C,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;CACpB;AA+ED,iEAAiE;AACjE,wBAAgB,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,SAAS,CAgC9D;AAqCD,8EAA8E;AAC9E,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,aAAa,CAKnF;AAED,sEAAsE;AACtE,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,iBAAiB,EAAE,GAAG,aAAa,EAAE,CAUtF;AAED,wBAAgB,MAAM,CAAC,EAAE,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS,CAGxD;AAED,sEAAsE;AACtE,wBAAgB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,EAAE,KAAK,SAAM,GAAG,SAAS,EAAE,CAe3E;AAED,kEAAkE;AAClE,wBAAgB,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,SAAS,SAAW,GAAG,OAAO,CAYlE;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,EAAE,EAAE,MAAM,EAAE,QAAQ,SAAW,GAAG,OAAO,CAUvE;AAED,0EAA0E;AAC1E,wBAAgB,SAAS,CACvB,EAAE,EAAE,MAAM,EACV,MAAM,EAAE,OAAO,CAAC,SAAS,EAAE,QAAQ,GAAG,QAAQ,GAAG,WAAW,CAAC,EAC7D,OAAO,GAAE,aAAkB,GAC1B,OAAO,CAWT;AAED;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,SAAS,SAAW,GAAG,OAAO,CAanE;AAED,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,CAGjE;AAED,mDAAmD;AACnD,wBAAgB,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,CAKlF;AAED,2CAA2C;AAC3C,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,aAAa,EAAE,CAK3D;AAED,qEAAqE;AACrE,wBAAgB,wBAAwB,CACtC,KAAK,EAAE,MAAM,EACb,cAAc,EAAE,MAAM,GACrB,aAAa,GAAG,SAAS,CAK3B;AAED,+DAA+D;AAC/D,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,SAAS,SAAW,EAAE,KAAK,CAAC,EAAE,SAAS,GAAG,OAAO,CAatF;AAED,2DAA2D;AAC3D,wBAAgB,UAAU,CACxB,EAAE,EAAE,MAAM,EACV,MAAM,EAAE,OAAO,CAAC,UAAU,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC,EAC5D,OAAO,GAAE,aAAkB,GAC1B,OAAO,CAqBT;AAED;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,SAAS,SAAW,EAAE,KAAK,CAAC,EAAE,SAAS,GAAG,OAAO,CAgBtF;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,GAAG,OAAO,CAQpE;AAED,qDAAqD;AACrD,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAKnD;AAED,mEAAmE;AACnE,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAG7C;AAED,iEAAiE;AACjE,wBAAgB,eAAe,IAAI,OAAO,CAKzC;AAED,+EAA+E;AAC/E,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAEhE"}
|