agent-hitch 0.2.7 → 0.2.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (287) hide show
  1. package/README.md +16 -2
  2. package/README.zh-CN.md +14 -2
  3. package/dist/bin/hitch.js +25 -3
  4. package/dist/bin/hitch.js.map +1 -1
  5. package/dist/scripts/canary-benchmark.js +54 -0
  6. package/dist/scripts/canary-benchmark.js.map +1 -0
  7. package/dist/scripts/canary-eval-scheduler-throughput.js +47 -0
  8. package/dist/scripts/canary-eval-scheduler-throughput.js.map +1 -0
  9. package/dist/scripts/check-architecture.js +3 -2
  10. package/dist/scripts/check-architecture.js.map +1 -1
  11. package/dist/src/adapters/catalog.js +2 -1
  12. package/dist/src/adapters/catalog.js.map +1 -1
  13. package/dist/src/adapters/providers/codex-auth.js +29 -0
  14. package/dist/src/adapters/providers/codex-auth.js.map +1 -0
  15. package/dist/src/adapters/providers/codex.js +23 -6
  16. package/dist/src/adapters/providers/codex.js.map +1 -1
  17. package/dist/src/adapters/providers/model-call.js +26 -0
  18. package/dist/src/adapters/providers/model-call.js.map +1 -0
  19. package/dist/src/artifacts/index.js +1 -1
  20. package/dist/src/artifacts/index.js.map +1 -1
  21. package/dist/src/artifacts/preparer.js +7 -2
  22. package/dist/src/artifacts/preparer.js.map +1 -1
  23. package/dist/src/backends/harbor/agent-budget.js +31 -0
  24. package/dist/src/backends/harbor/agent-budget.js.map +1 -0
  25. package/dist/src/backends/harbor/backend.js +6 -4
  26. package/dist/src/backends/harbor/backend.js.map +1 -1
  27. package/dist/src/backends/harbor/index.js +1 -0
  28. package/dist/src/backends/harbor/index.js.map +1 -1
  29. package/dist/src/backends/harbor/regrade.js +77 -0
  30. package/dist/src/backends/harbor/regrade.js.map +1 -0
  31. package/dist/src/backends/index.js +1 -0
  32. package/dist/src/backends/index.js.map +1 -1
  33. package/dist/src/benchmarks/index.js +3 -0
  34. package/dist/src/benchmarks/index.js.map +1 -0
  35. package/dist/src/benchmarks/loader.js +271 -0
  36. package/dist/src/benchmarks/loader.js.map +1 -0
  37. package/dist/src/benchmarks/metrics.js +16 -0
  38. package/dist/src/benchmarks/metrics.js.map +1 -0
  39. package/dist/src/benchmarks/toml.js +13 -0
  40. package/dist/src/benchmarks/toml.js.map +1 -0
  41. package/dist/src/benchmarks/validation.js +242 -0
  42. package/dist/src/benchmarks/validation.js.map +1 -0
  43. package/dist/src/cli/arguments.js +6 -4
  44. package/dist/src/cli/arguments.js.map +1 -1
  45. package/dist/src/cli/commands/benchmark.js +22 -0
  46. package/dist/src/cli/commands/benchmark.js.map +1 -0
  47. package/dist/src/cli/commands/capabilities.js +18 -0
  48. package/dist/src/cli/commands/capabilities.js.map +1 -0
  49. package/dist/src/cli/commands/eval.js +15 -3
  50. package/dist/src/cli/commands/eval.js.map +1 -1
  51. package/dist/src/cli/commands/run.js +37 -15
  52. package/dist/src/cli/commands/run.js.map +1 -1
  53. package/dist/src/cli/commands/trajectory.js +96 -5
  54. package/dist/src/cli/commands/trajectory.js.map +1 -1
  55. package/dist/src/cli/commands/verifier.js +36 -0
  56. package/dist/src/cli/commands/verifier.js.map +1 -0
  57. package/dist/src/cli/main.js +6 -0
  58. package/dist/src/cli/main.js.map +1 -1
  59. package/dist/src/cli/output.js +14 -2
  60. package/dist/src/cli/output.js.map +1 -1
  61. package/dist/src/control-plane/eval-control-work.js +5 -0
  62. package/dist/src/control-plane/eval-control-work.js.map +1 -1
  63. package/dist/src/control-plane/eval-scheduler.js +4 -1
  64. package/dist/src/control-plane/eval-scheduler.js.map +1 -1
  65. package/dist/src/control-plane/index.js +1 -1
  66. package/dist/src/control-plane/index.js.map +1 -1
  67. package/dist/src/control-plane/remote-result-transport.js +3 -0
  68. package/dist/src/control-plane/remote-result-transport.js.map +1 -1
  69. package/dist/src/control-plane/remote-work-coordinator.js +1 -0
  70. package/dist/src/control-plane/remote-work-coordinator.js.map +1 -1
  71. package/dist/src/control-plane/remote-work-item.js +62 -0
  72. package/dist/src/control-plane/remote-work-item.js.map +1 -0
  73. package/dist/src/control-plane/remote-work-recovery.js +44 -5
  74. package/dist/src/control-plane/remote-work-recovery.js.map +1 -1
  75. package/dist/src/control-plane/remote-worker-protocol.js +2 -23
  76. package/dist/src/control-plane/remote-worker-protocol.js.map +1 -1
  77. package/dist/src/control-plane/rerun-scheduler.js +51 -14
  78. package/dist/src/control-plane/rerun-scheduler.js.map +1 -1
  79. package/dist/src/control-plane/rerun-submission.js +10 -4
  80. package/dist/src/control-plane/rerun-submission.js.map +1 -1
  81. package/dist/src/control-plane/work-admission.js +2 -0
  82. package/dist/src/control-plane/work-admission.js.map +1 -1
  83. package/dist/src/control-plane/work-dispatcher.js +9 -1
  84. package/dist/src/control-plane/work-dispatcher.js.map +1 -1
  85. package/dist/src/controller-runtime/hash.js +7 -2
  86. package/dist/src/controller-runtime/hash.js.map +1 -1
  87. package/dist/src/daemon/auth.js +5 -0
  88. package/dist/src/daemon/auth.js.map +1 -1
  89. package/dist/src/domain/benchmarks.js +2 -0
  90. package/dist/src/domain/benchmarks.js.map +1 -0
  91. package/dist/src/domain/index.js +4 -0
  92. package/dist/src/domain/index.js.map +1 -1
  93. package/dist/src/domain/runs.js.map +1 -1
  94. package/dist/src/domain/validation.js +11 -4
  95. package/dist/src/domain/validation.js.map +1 -1
  96. package/dist/src/domain/verifier-evidence-validation.js +214 -0
  97. package/dist/src/domain/verifier-evidence-validation.js.map +1 -0
  98. package/dist/src/domain/verifier-evidence.js +3 -0
  99. package/dist/src/domain/verifier-evidence.js.map +1 -0
  100. package/dist/src/domain/verifier-score-contract.js +232 -0
  101. package/dist/src/domain/verifier-score-contract.js.map +1 -0
  102. package/dist/src/evals/benchmark-adapter-manifest.js +219 -0
  103. package/dist/src/evals/benchmark-adapter-manifest.js.map +1 -0
  104. package/dist/src/evals/benchmark-candidate.js +36 -0
  105. package/dist/src/evals/benchmark-candidate.js.map +1 -0
  106. package/dist/src/evals/benchmark-run.js +166 -0
  107. package/dist/src/evals/benchmark-run.js.map +1 -0
  108. package/dist/src/evals/collect-only-rerun.js +2 -1
  109. package/dist/src/evals/collect-only-rerun.js.map +1 -1
  110. package/dist/src/evals/directory.js +1 -1
  111. package/dist/src/evals/directory.js.map +1 -1
  112. package/dist/src/evals/duration-estimator.js +126 -0
  113. package/dist/src/evals/duration-estimator.js.map +1 -0
  114. package/dist/src/evals/eval-lifecycle-events.js +13 -0
  115. package/dist/src/evals/eval-lifecycle-events.js.map +1 -1
  116. package/dist/src/evals/eval-logical-plan.js +51 -0
  117. package/dist/src/evals/eval-logical-plan.js.map +1 -0
  118. package/dist/src/evals/eval-result-builder.js +81 -0
  119. package/dist/src/evals/eval-result-builder.js.map +1 -0
  120. package/dist/src/evals/evolution-baseline.js +191 -0
  121. package/dist/src/evals/evolution-baseline.js.map +1 -0
  122. package/dist/src/evals/execution-plan.js +39 -3
  123. package/dist/src/evals/execution-plan.js.map +1 -1
  124. package/dist/src/evals/failure-classifier.js +61 -0
  125. package/dist/src/evals/failure-classifier.js.map +1 -0
  126. package/dist/src/evals/harbor-artifact-builder.js +6 -2
  127. package/dist/src/evals/harbor-artifact-builder.js.map +1 -1
  128. package/dist/src/evals/harbor-bridge-error.js +41 -1
  129. package/dist/src/evals/harbor-bridge-error.js.map +1 -1
  130. package/dist/src/evals/harbor-node-runtime.js +146 -0
  131. package/dist/src/evals/harbor-node-runtime.js.map +1 -0
  132. package/dist/src/evals/index.js +15 -0
  133. package/dist/src/evals/index.js.map +1 -1
  134. package/dist/src/evals/infrastructure-retry.js +109 -22
  135. package/dist/src/evals/infrastructure-retry.js.map +1 -1
  136. package/dist/src/evals/native-phase-evidence.js +440 -0
  137. package/dist/src/evals/native-phase-evidence.js.map +1 -0
  138. package/dist/src/evals/physical-retry-work.js +31 -0
  139. package/dist/src/evals/physical-retry-work.js.map +1 -0
  140. package/dist/src/evals/planned-execution-support.js +22 -10
  141. package/dist/src/evals/planned-execution-support.js.map +1 -1
  142. package/dist/src/evals/planned-execution.js +120 -112
  143. package/dist/src/evals/planned-execution.js.map +1 -1
  144. package/dist/src/evals/planned-progress-publisher.js +62 -0
  145. package/dist/src/evals/planned-progress-publisher.js.map +1 -0
  146. package/dist/src/evals/planned-retry-execution.js +174 -0
  147. package/dist/src/evals/planned-retry-execution.js.map +1 -0
  148. package/dist/src/evals/planned-retry-lifecycle.js +4 -2
  149. package/dist/src/evals/planned-retry-lifecycle.js.map +1 -1
  150. package/dist/src/evals/preparation-rerun.js +225 -0
  151. package/dist/src/evals/preparation-rerun.js.map +1 -0
  152. package/dist/src/evals/progress.js +48 -8
  153. package/dist/src/evals/progress.js.map +1 -1
  154. package/dist/src/evals/recovery.js +54 -4
  155. package/dist/src/evals/recovery.js.map +1 -1
  156. package/dist/src/evals/regrade-evidence.js +77 -0
  157. package/dist/src/evals/regrade-evidence.js.map +1 -0
  158. package/dist/src/evals/remote-infrastructure-retry.js +84 -18
  159. package/dist/src/evals/remote-infrastructure-retry.js.map +1 -1
  160. package/dist/src/evals/request.js +21 -3
  161. package/dist/src/evals/request.js.map +1 -1
  162. package/dist/src/evals/rerun-inputs.js +19 -0
  163. package/dist/src/evals/rerun-inputs.js.map +1 -1
  164. package/dist/src/evals/rerun-types.js +2 -6
  165. package/dist/src/evals/rerun-types.js.map +1 -1
  166. package/dist/src/evals/rerun.js +27 -20
  167. package/dist/src/evals/rerun.js.map +1 -1
  168. package/dist/src/evals/result-helpers.js +13 -0
  169. package/dist/src/evals/result-helpers.js.map +1 -1
  170. package/dist/src/evals/retry-backoff.js +20 -0
  171. package/dist/src/evals/retry-backoff.js.map +1 -0
  172. package/dist/src/evals/retry-state.js +195 -0
  173. package/dist/src/evals/retry-state.js.map +1 -0
  174. package/dist/src/evals/scheduler-metrics.js +95 -0
  175. package/dist/src/evals/scheduler-metrics.js.map +1 -0
  176. package/dist/src/evals/scheduler-trace-replay.js +107 -0
  177. package/dist/src/evals/scheduler-trace-replay.js.map +1 -0
  178. package/dist/src/evals/service.js +87 -138
  179. package/dist/src/evals/service.js.map +1 -1
  180. package/dist/src/evals/trial-import.js +47 -50
  181. package/dist/src/evals/trial-import.js.map +1 -1
  182. package/dist/src/evals/trial-publication-recovery.js +45 -5
  183. package/dist/src/evals/trial-publication-recovery.js.map +1 -1
  184. package/dist/src/evals/trial-reference-validation.js +49 -0
  185. package/dist/src/evals/trial-reference-validation.js.map +1 -0
  186. package/dist/src/evals/verifier-artifacts.js +317 -0
  187. package/dist/src/evals/verifier-artifacts.js.map +1 -0
  188. package/dist/src/evals/verifier-eligibility.js +25 -0
  189. package/dist/src/evals/verifier-eligibility.js.map +1 -0
  190. package/dist/src/evals/verifier-only-rerun.js +210 -0
  191. package/dist/src/evals/verifier-only-rerun.js.map +1 -0
  192. package/dist/src/evals/verifier-runtime.js +61 -0
  193. package/dist/src/evals/verifier-runtime.js.map +1 -0
  194. package/dist/src/evals/verifier-score-artifacts.js +119 -0
  195. package/dist/src/evals/verifier-score-artifacts.js.map +1 -0
  196. package/dist/src/foundation/contained-file.js +85 -0
  197. package/dist/src/foundation/contained-file.js.map +1 -0
  198. package/dist/src/foundation/credential-redaction.js +26 -1
  199. package/dist/src/foundation/credential-redaction.js.map +1 -1
  200. package/dist/src/foundation/fs.js +2 -1
  201. package/dist/src/foundation/fs.js.map +1 -1
  202. package/dist/src/foundation/index.js +2 -1
  203. package/dist/src/foundation/index.js.map +1 -1
  204. package/dist/src/runs/adapter-process.js +12 -0
  205. package/dist/src/runs/adapter-process.js.map +1 -0
  206. package/dist/src/runs/executor.js +24 -30
  207. package/dist/src/runs/executor.js.map +1 -1
  208. package/dist/src/runs/finalizer.js +17 -0
  209. package/dist/src/runs/finalizer.js.map +1 -1
  210. package/dist/src/runs/index.js +5 -0
  211. package/dist/src/runs/index.js.map +1 -1
  212. package/dist/src/runs/phase-bundle.js +86 -0
  213. package/dist/src/runs/phase-bundle.js.map +1 -0
  214. package/dist/src/runs/phase-cancellation.js +107 -0
  215. package/dist/src/runs/phase-cancellation.js.map +1 -0
  216. package/dist/src/runs/phase-group.js +113 -0
  217. package/dist/src/runs/phase-group.js.map +1 -0
  218. package/dist/src/runs/query.js +1 -1
  219. package/dist/src/runs/query.js.map +1 -1
  220. package/dist/src/runs/records.js +3 -0
  221. package/dist/src/runs/records.js.map +1 -1
  222. package/dist/src/runs/request.js +4 -2
  223. package/dist/src/runs/request.js.map +1 -1
  224. package/dist/src/runs/verifier-evidence-redaction.js +61 -0
  225. package/dist/src/runs/verifier-evidence-redaction.js.map +1 -0
  226. package/dist/src/runs/verifier-evidence.js +459 -0
  227. package/dist/src/runs/verifier-evidence.js.map +1 -0
  228. package/dist/src/runs/verifier-structured-evidence.js +83 -0
  229. package/dist/src/runs/verifier-structured-evidence.js.map +1 -0
  230. package/dist/src/trajectories/analysis.js +288 -0
  231. package/dist/src/trajectories/analysis.js.map +1 -0
  232. package/dist/src/trajectories/chunk-projection.js +148 -0
  233. package/dist/src/trajectories/chunk-projection.js.map +1 -0
  234. package/dist/src/trajectories/content-projection.js +389 -0
  235. package/dist/src/trajectories/content-projection.js.map +1 -0
  236. package/dist/src/trajectories/dsh-chunk-contract.js +176 -0
  237. package/dist/src/trajectories/dsh-chunk-contract.js.map +1 -0
  238. package/dist/src/trajectories/dsh-contract.js +284 -0
  239. package/dist/src/trajectories/dsh-contract.js.map +1 -0
  240. package/dist/src/trajectories/events-chunk-drill.js +95 -0
  241. package/dist/src/trajectories/events-chunk-drill.js.map +1 -0
  242. package/dist/src/trajectories/events-page.js +367 -0
  243. package/dist/src/trajectories/events-page.js.map +1 -0
  244. package/dist/src/trajectories/index.js +3 -0
  245. package/dist/src/trajectories/index.js.map +1 -1
  246. package/dist/src/trajectories/request-attempt.js +47 -0
  247. package/dist/src/trajectories/request-attempt.js.map +1 -0
  248. package/dist/src/trajectories/stream-reader.js +311 -0
  249. package/dist/src/trajectories/stream-reader.js.map +1 -0
  250. package/dist/src/trajectories/surface-fold.js +126 -0
  251. package/dist/src/trajectories/surface-fold.js.map +1 -0
  252. package/dist/src/workers/remote-harbor-worker.js +3 -0
  253. package/dist/src/workers/remote-harbor-worker.js.map +1 -1
  254. package/docs/schemas/benchmark-hook-request-v1.schema.json +65 -0
  255. package/docs/schemas/benchmark-hook-response-v1.schema.json +75 -0
  256. package/docs/schemas/benchmark-hook-v1.schema.json +33 -0
  257. package/docs/schemas/benchmark-lock-v1.schema.json +351 -0
  258. package/docs/schemas/benchmark-metric-v1.schema.json +37 -0
  259. package/docs/schemas/benchmark-package-v1.schema.json +167 -0
  260. package/docs/schemas/benchmark-phase-group.schema.json +41 -0
  261. package/docs/schemas/benchmark-profile-v1.schema.json +141 -0
  262. package/docs/schemas/benchmark-task-v1.schema.json +384 -0
  263. package/docs/schemas/benchmark-tool-result-v1.schema.json +30 -0
  264. package/docs/schemas/error.schema.json +22 -0
  265. package/docs/schemas/eval-progress.schema.json +1 -15
  266. package/docs/schemas/eval-rerun-result.schema.json +6 -1
  267. package/docs/schemas/eval-rerun-submission.schema.json +3 -1
  268. package/docs/schemas/eval-result.schema.json +1 -15
  269. package/docs/schemas/eval-trial-publication.schema.json +1 -25
  270. package/docs/schemas/eval-trial-reference.schema.json +129 -0
  271. package/docs/schemas/regrade-assessment-reference.schema.json +11 -0
  272. package/docs/schemas/run-context.schema.json +15 -0
  273. package/docs/schemas/run-manifest.schema.json +4 -0
  274. package/docs/schemas/run-request.schema.json +4 -0
  275. package/docs/schemas/trajectory-analysis.schema.json +258 -0
  276. package/docs/schemas/trajectory-events-page.schema.json +40 -0
  277. package/docs/schemas/verifier-assessment.schema.json +77 -0
  278. package/docs/schemas/verifier-evidence.schema.json +276 -0
  279. package/integrations/harbor/hitch_benchmark.py +252 -0
  280. package/integrations/harbor/hitch_candidate_recycle.py +262 -0
  281. package/integrations/harbor/hitch_harbor_agent.py +541 -43
  282. package/integrations/harbor/hitch_harbor_environment.py +138 -0
  283. package/integrations/harbor/hitch_harbor_verifier.py +60 -2
  284. package/integrations/harbor/hitch_phase_supervisor.py +452 -0
  285. package/integrations/harbor/hitch_tool_client.mjs +91 -0
  286. package/integrations/model-call/cli.js +57 -0
  287. package/package.json +17 -6
@@ -4,45 +4,76 @@ import { runHarborBackend } from "../backends/index.js";
4
4
  import { HitchError } from "../foundation/index.js";
5
5
  import { importEvalTrialRun, importEvalTrialRuns, TrialBundlePendingError, validateEvalTrialReferences } from "./trial-import.js";
6
6
  import { replaceInvalidEvalProgressTrial, writeEvalProgress } from "./progress.js";
7
+ import { ensurePhysicalRetryDecision, transitionRetryDecision } from "./retry-state.js";
8
+ import { physicalRetryWorkItem } from "./physical-retry-work.js";
9
+ import { classifyTrialFailure, physicalRetryAllowed } from "./failure-classifier.js";
10
+ import { retryBackoffMs } from "./retry-backoff.js";
11
+ import { harborPhaseTimingEvents } from "./harbor-phase-timings.js";
7
12
  const INFRASTRUCTURE_REASONS = new Set([
8
13
  "infrastructure_failure",
9
14
  "verifier_infrastructure_failure",
10
15
  "verifier_result_missing",
11
- ]);
12
- // Verifier failures are retried inside the original live Harbor trial by the
13
- // custom verifier wrapper. Starting another Harbor trial would execute the
14
- // candidate agent again, so only non-verifier trial infrastructure failures
15
- // are eligible for this outer retry path.
16
- const RETRYABLE_TRIAL_INFRASTRUCTURE_REASONS = new Set([
17
- "infrastructure_failure",
16
+ "provider_quota_exhausted",
17
+ "provider_auth_failed",
18
+ "provider_configuration_invalid",
19
+ "provider_rate_limited",
20
+ "provider_transport_transient",
21
+ "worker_lost_before_candidate",
22
+ "sandbox_setup_failed",
23
+ "execution_state_ambiguous",
18
24
  ]);
19
25
  export async function runInfrastructureRetries(options) {
20
26
  let progress = options.progress;
21
27
  let retryCandidates = retryableInfrastructureTrials(options.initialRefs);
22
28
  const runs = [];
23
- for (let retry = 1; retry <= options.request.infrastructure_retries && retryCandidates.length > 0; retry += 1) {
29
+ const firstRetryIndex = options.firstRetryIndex ?? 1;
30
+ if (!Number.isSafeInteger(firstRetryIndex) || firstRetryIndex < 1)
31
+ throw new TypeError("first infrastructure retry index is invalid");
32
+ for (let retry = firstRetryIndex; retry <= options.request.infrastructure_retries && retryCandidates.length > 0; retry += 1) {
24
33
  if (options.signal?.aborted)
25
34
  break;
26
35
  const retryTriggers = [...retryCandidates];
27
36
  const taskNames = [...new Set(retryCandidates.map((ref) => ref.task_id))].sort();
28
- const backoffMs = options.request.infrastructure_retry_backoff_ms * retry;
37
+ const retryWork = options.originWorkItem ? physicalRetryWorkItem(options.originWorkItem, retry, retryTriggers) : undefined;
38
+ const backoffMs = retryBackoffMs(options.request.infrastructure_retry_backoff_ms, retry, retryWork?.work_id ?? retryTriggers.map((trial) => trial.trial_id).join("\0"));
39
+ const notBefore = new Date(Date.now() + backoffMs).toISOString();
40
+ const retryDecisions = options.originWorkItem
41
+ ? await Promise.all(retryTriggers.map((trigger) => ensurePhysicalRetryDecision({
42
+ evalDirectory: options.evalDirectory, evalId: options.evalId, item: options.originWorkItem,
43
+ retryIndex: retry, trigger, notBefore,
44
+ })))
45
+ : [];
46
+ if (retryWork && retryDecisions.some((decision) => decision.retry_work_id !== retryWork.work_id)) {
47
+ throw new Error(`persisted retry work identity conflicts with execution: ${retryWork.work_id}`);
48
+ }
49
+ if (retryWork)
50
+ await options.onRetryWorkPlanned?.(retryWork.work_id);
29
51
  options.sink.emit({
30
- type: "eval.infrastructure-retry.scheduled",
52
+ type: "eval.retry.decision",
31
53
  execution_kind: "physical-infrastructure-retry",
32
54
  candidate_executes: true,
55
+ ...(retryWork ? { work_id: retryWork.work_id } : {}),
56
+ decision_ids: retryDecisions.map((decision) => decision.decision_id),
33
57
  attempt: options.logicalAttempt,
34
58
  retry,
35
59
  tasks: taskNames,
36
60
  backoff_ms: backoffMs,
37
61
  });
38
- if (backoffMs > 0) {
62
+ const remainingBackoffMs = Math.max(0, Date.parse(retryDecisions[0]?.not_before ?? notBefore) - Date.now());
63
+ options.onRetryBackoff?.(remainingBackoffMs);
64
+ if (remainingBackoffMs > 0) {
39
65
  if (options.signal)
40
- await delay(backoffMs, undefined, { signal: options.signal });
66
+ await delay(remainingBackoffMs, undefined, { signal: options.signal });
41
67
  else
42
- await delay(backoffMs);
68
+ await delay(remainingBackoffMs);
43
69
  }
44
70
  if (options.signal?.aborted)
45
71
  break;
72
+ if (retryWork)
73
+ options.sink.emit({
74
+ type: "eval.retry.ready", work_id: retryWork.work_id, decision_ids: retryDecisions.map((decision) => decision.decision_id),
75
+ retry, tasks: taskNames, priority: options.originWorkItem?.scheduling?.remaining_path_ms ?? 0,
76
+ });
46
77
  const backendDirectory = path.join(options.backendBaseDirectory || path.join(options.evalDirectory, "infrastructure-retries"), `retry-${String(retry).padStart(4, "0")}`, options.request.attempts === 1 ? "harbor" : attemptDirectoryName(options.logicalAttempt));
47
78
  const harborJobDirectory = path.join(backendDirectory, "job");
48
79
  const retryRefs = [];
@@ -66,13 +97,26 @@ export async function runInfrastructureRetries(options) {
66
97
  return;
67
98
  }
68
99
  retryRefs.push(ref);
100
+ if (ref.invalid_reason === "candidate_evidence_unavailable")
101
+ options.sink.emit({
102
+ type: "eval.verifier.skipped", work_id: retryWork?.work_id, trial_id: ref.trial_id, task_id: ref.task_id,
103
+ reason: "candidate_evidence_unavailable", candidate_executes: true, verifier_executes: false,
104
+ });
69
105
  await validateEvalTrialReferences(options.root, options.evalId, [ref], {
70
106
  benchmarkId: options.request.benchmark_id,
71
107
  benchmarkRevision: options.request.benchmark_revision,
72
108
  });
73
109
  if (ref.observation_status === "valid") {
74
- progress = replaceInvalidEvalProgressTrial(progress, ref);
75
- await writeEvalProgress(options.evalDirectory, progress);
110
+ if (options.replaceProgressTrial) {
111
+ if (!lifecycle)
112
+ throw new Error("planned infrastructure retry is missing its lifecycle identity");
113
+ await options.replaceProgressTrial(ref, lifecycle.workId);
114
+ progress = options.currentProgress?.() ?? progress;
115
+ }
116
+ else {
117
+ progress = replaceInvalidEvalProgressTrial(progress, ref);
118
+ await writeEvalProgress(options.evalDirectory, progress);
119
+ }
76
120
  }
77
121
  options.sink.emit({
78
122
  type: ref.observation_status === "valid"
@@ -85,12 +129,27 @@ export async function runInfrastructureRetries(options) {
85
129
  run_id: ref.run_id,
86
130
  observation_status: ref.observation_status,
87
131
  ...(ref.invalid_reason ? { invalid_reason: ref.invalid_reason } : {}),
88
- generation: progress.generation,
132
+ generation: (options.currentProgress?.() ?? progress).generation,
89
133
  });
90
134
  };
91
135
  let run;
136
+ let nextCandidates = [];
137
+ let runRecord;
138
+ let executionStarted = false;
92
139
  try {
93
140
  lifecycle = await options.beginRetry?.({ retry, logicalAttempt: options.logicalAttempt, taskNames, triggers: retryTriggers, backendDirectory });
141
+ if (retryWork && lifecycle?.workId !== retryWork.work_id)
142
+ throw new Error(`retry lifecycle work identity conflicts with persisted decision: ${retryWork.work_id}`);
143
+ for (const decision of retryDecisions)
144
+ await transitionRetryDecision({
145
+ evalDirectory: options.evalDirectory, evalId: options.evalId, decisionId: decision.decision_id, state: "running",
146
+ });
147
+ if (retryWork)
148
+ options.sink.emit({ type: "eval.retry.admitted", work_id: retryWork.work_id, decision_ids: retryDecisions.map((decision) => decision.decision_id) });
149
+ if (retryWork) {
150
+ options.onRetryExecutionStarted?.(retryWork.work_id);
151
+ executionStarted = true;
152
+ }
94
153
  run = await runHarborBackend({
95
154
  evalId: options.evalId,
96
155
  evalDirectory: options.evalDirectory,
@@ -128,6 +187,7 @@ export async function runInfrastructureRetries(options) {
128
187
  publicationMode: "replace-invalid",
129
188
  runtimeId: options.controllerRuntime.runtime_id,
130
189
  env: options.env,
190
+ ...(options.signal ? { signal: options.signal } : {}),
131
191
  ...(options.modelCapturePlan ? { modelCapturePlan: options.modelCapturePlan } : {}),
132
192
  ...(options.interactionCaptureExporter ? {
133
193
  interactionCaptureExporter: options.interactionCaptureExporter,
@@ -147,6 +207,11 @@ export async function runInfrastructureRetries(options) {
147
207
  }
148
208
  },
149
209
  });
210
+ for (const event of await harborPhaseTimingEvents(harborJobDirectory, run.rawResult)) {
211
+ options.sink.emit(event);
212
+ if (event.type === "eval.verifier.completed" && typeof event.duration_ms === "number")
213
+ options.onVerifierDuration?.(event.duration_ms);
214
+ }
150
215
  const environmentImages = retryEnvironmentImages();
151
216
  const terminalRefs = await importEvalTrialRuns({
152
217
  root: options.root,
@@ -161,6 +226,7 @@ export async function runInfrastructureRetries(options) {
161
226
  publicationMode: "replace-invalid",
162
227
  runtimeId: options.controllerRuntime.runtime_id,
163
228
  env: options.env,
229
+ ...(options.signal ? { signal: options.signal } : {}),
164
230
  ...(options.modelCapturePlan ? { modelCapturePlan: options.modelCapturePlan } : {}),
165
231
  ...(options.interactionCaptureExporter ? {
166
232
  interactionCaptureExporter: options.interactionCaptureExporter,
@@ -173,10 +239,33 @@ export async function runInfrastructureRetries(options) {
173
239
  await publish(ref);
174
240
  if (run.rawResult !== null)
175
241
  assertBackendTrialSet(run.rawResult, retryRefs);
176
- runs.push({ attempt: options.logicalAttempt, retry, tasks: taskNames, triggers: retryTriggers, refs: retryRefs, run, ...(lifecycle ? { leaseId: lifecycle.leaseId, workId: lifecycle.workId } : {}) });
242
+ runRecord = { attempt: options.logicalAttempt, retry, tasks: taskNames, triggers: retryTriggers, refs: retryRefs, run, ...(lifecycle ? { leaseId: lifecycle.leaseId, workId: lifecycle.workId } : {}) };
243
+ runs.push(runRecord);
244
+ nextCandidates = retryableInfrastructureTrials(retryRefs);
245
+ if (options.originWorkItem && retry < options.request.infrastructure_retries && nextCandidates.length > 0) {
246
+ const nextWork = physicalRetryWorkItem(options.originWorkItem, retry + 1, nextCandidates);
247
+ const nextNotBefore = new Date(Date.now() + retryBackoffMs(options.request.infrastructure_retry_backoff_ms, retry + 1, nextWork.work_id)).toISOString();
248
+ await Promise.all(nextCandidates.map((trigger) => ensurePhysicalRetryDecision({
249
+ evalDirectory: options.evalDirectory, evalId: options.evalId, item: options.originWorkItem,
250
+ retryIndex: retry + 1, trigger, notBefore: nextNotBefore,
251
+ })));
252
+ await options.onRetryWorkPlanned?.(nextWork.work_id);
253
+ }
254
+ const retryState = retryRefs.some((ref) => ref.observation_status === "valid")
255
+ ? "repaired"
256
+ : retry < options.request.infrastructure_retries && nextCandidates.length > 0 ? "invalid" : "exhausted";
257
+ for (const decision of retryDecisions)
258
+ await transitionRetryDecision({
259
+ evalDirectory: options.evalDirectory, evalId: options.evalId, decisionId: decision.decision_id, state: retryState,
260
+ });
177
261
  }
178
262
  finally {
179
263
  await lifecycle?.close();
264
+ if (executionStarted && retryWork) {
265
+ const durationMs = options.onRetryExecutionFinished?.(retryWork.work_id);
266
+ if (runRecord && durationMs !== undefined)
267
+ runRecord.durationMs = durationMs;
268
+ }
180
269
  }
181
270
  options.sink.emit({
182
271
  type: "eval.infrastructure-retry.completed",
@@ -190,14 +279,12 @@ export async function runInfrastructureRetries(options) {
190
279
  });
191
280
  if (run.backend.process_exit_code !== 0 || run.rawResult === null || options.stopAfterResult?.(run.rawResult))
192
281
  break;
193
- retryCandidates = retryableInfrastructureTrials(retryRefs);
282
+ retryCandidates = nextCandidates;
194
283
  }
195
- return { progress, runs };
284
+ return { progress: options.currentProgress?.() ?? progress, runs };
196
285
  }
197
286
  export function retryableInfrastructureTrials(trials) {
198
- return trials.filter((trial) => trial.observation_status === "invalid"
199
- && trial.invalid_reason !== undefined
200
- && RETRYABLE_TRIAL_INFRASTRUCTURE_REASONS.has(trial.invalid_reason));
287
+ return trials.filter((trial) => physicalRetryAllowed(classifyTrialFailure(trial)));
201
288
  }
202
289
  export function infrastructureFailureTrials(trials) {
203
290
  return trials.filter((trial) => trial.observation_status === "invalid"
@@ -1 +1 @@
1
- {"version":3,"file":"infrastructure-retry.js","sourceRoot":"","sources":["../../../src/evals/infrastructure-retry.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAIxD,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpD,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAClI,OAAO,EAAE,+BAA+B,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAInF,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC;IACrC,wBAAwB;IACxB,iCAAiC;IACjC,yBAAyB;CAC1B,CAAC,CAAC;AAEH,6EAA6E;AAC7E,2EAA2E;AAC3E,4EAA4E;AAC5E,0CAA0C;AAC1C,MAAM,sCAAsC,GAAG,IAAI,GAAG,CAAC;IACrD,wBAAwB;CACzB,CAAC,CAAC;AA6DH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,OAAwC;IAExC,IAAI,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAChC,IAAI,eAAe,GAAG,6BAA6B,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IACzE,MAAM,IAAI,GAA6B,EAAE,CAAC;IAC1C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC,sBAAsB,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC9G,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM;QACnC,MAAM,aAAa,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACjF,MAAM,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,+BAA+B,GAAG,KAAK,CAAC;QAC1E,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,qCAAqC;YAC3C,cAAc,EAAE,+BAA+B;YAC/C,kBAAkB,EAAE,IAAI;YACxB,OAAO,EAAE,OAAO,CAAC,cAAc;YAC/B,KAAK;YACL,KAAK,EAAE,SAAS;YAChB,UAAU,EAAE,SAAS;SACtB,CAAC,CAAC;QACH,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;YAClB,IAAI,OAAO,CAAC,MAAM;gBAAE,MAAM,KAAK,CAAC,SAAS,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;;gBAC7E,MAAM,KAAK,CAAC,SAAS,CAAC,CAAC;QAC9B,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM;QAEnC,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAChC,OAAO,CAAC,oBAAoB,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,wBAAwB,CAAC,EAC1F,SAAS,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EACzC,OAAO,CAAC,OAAO,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,cAAc,CAAC,CACzF,CAAC;QACF,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;QAC9D,MAAM,SAAS,GAAqB,EAAE,CAAC;QACvC,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QACzC,IAAI,SAAmD,CAAC;QACxD,MAAM,sBAAsB,GAAG,GAAyC,EAAE,CAAC,SAAS,EAAE,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,CAAC;QACrI,MAAM,iBAAiB,GAAG,KAAK,IAA0D,EAAE,CAAC,SAAS,EAAE,wBAAwB;YAC7H,CAAC,CAAC,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC,wBAAwB,EAAE,EAAE;YACnE,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,OAAO,GAAG,KAAK,EAAE,GAAmB,EAAiB,EAAE;YAC3D,IAAI,GAAG,CAAC,OAAO,KAAK,OAAO,CAAC,cAAc,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC9E,MAAM,IAAI,UAAU,CAAC,6DAA6D,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,OAAO,EAAE,EAAE;oBAC9G,IAAI,EAAE,0CAA0C;oBAChD,QAAQ,EAAE,EAAE;iBACb,CAAC,CAAC;YACL,CAAC;YACD,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,CAAC,CAAC;YAChF,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACtI,OAAO;YACT,CAAC;YACD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,MAAM,2BAA2B,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE;gBACrE,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY;gBACzC,iBAAiB,EAAE,OAAO,CAAC,OAAO,CAAC,kBAAkB;aACtD,CAAC,CAAC;YACH,IAAI,GAAG,CAAC,kBAAkB,KAAK,OAAO,EAAE,CAAC;gBACvC,QAAQ,GAAG,+BAA+B,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;gBAC1D,MAAM,iBAAiB,CAAC,OAAO,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;YAC3D,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,GAAG,CAAC,kBAAkB,KAAK,OAAO;oBACtC,CAAC,CAAC,oCAAoC;oBACtC,CAAC,CAAC,kCAAkC;gBACtC,OAAO,EAAE,OAAO,CAAC,cAAc;gBAC/B,KAAK;gBACL,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,kBAAkB,EAAE,GAAG,CAAC,kBAAkB;gBAC1C,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,GAAG,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrE,UAAU,EAAE,QAAQ,CAAC,UAAU;aAChC,CAAC,CAAC;QACL,CAAC,CAAC;QACF,IAAI,GAAwB,CAAC;QAC7B,IAAI,CAAC;YACH,SAAS,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE,SAAS,EAAE,QAAQ,EAAE,aAAa,EAAE,gBAAgB,EAAE,CAAC,CAAC;YAChJ,GAAG,GAAG,MAAM,gBAAgB,CAAC;gBAC3B,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,aAAa,EAAE,OAAO,CAAC,aAAa;gBACpC,gBAAgB;gBAChB,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,SAAS;gBACT,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE;gBAC5C,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;gBAC1C,gBAAgB,EAAE,OAAO,CAAC,iBAAiB,CAAC,SAAS;gBACrD,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,UAAU;gBAC/C,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;gBAC1C,GAAG,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzF,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7E,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,0BAA0B,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvG,GAAG,SAAS,EAAE,OAAO;gBACrB,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,GAAG,CAAC,OAAO,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrD,GAAG,CAAC,OAAO,CAAC,kBAAkB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,EAAE,CAAC;gBACvG,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,cAAc,EAAE,+BAA+B,EAAE,oBAAoB,EAAE,KAAK,EAAE,eAAe,EAAE,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;gBACzP,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAoB,EAAE;oBACzD,IAAI,CAAC;wBACH,MAAM,iBAAiB,GAAG,sBAAsB,EAAE,CAAC;wBACnD,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC;4BACnC,IAAI,EAAE,OAAO,CAAC,IAAI;4BAClB,MAAM,EAAE,OAAO,CAAC,MAAM;4BACtB,aAAa,EAAE,OAAO,CAAC,aAAa;4BACpC,kBAAkB;4BAClB,eAAe,EAAE,OAAO,CAAC,cAAc;4BACvC,OAAO,EAAE,OAAO,CAAC,OAAO;4BACxB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;4BAC1C,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY;4BACzC,iBAAiB,EAAE,OAAO,CAAC,OAAO,CAAC,kBAAkB;4BACrD,eAAe,EAAE,iBAAiB;4BAClC,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,UAAU;4BAC/C,GAAG,EAAE,OAAO,CAAC,GAAG;4BAChB,GAAG,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;4BACnF,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC,CAAC;gCACvC,0BAA0B,EAAE,OAAO,CAAC,0BAA0B;6BAC/D,CAAC,CAAC,CAAC,EAAE,CAAC;4BACP,GAAG,MAAM,iBAAiB,EAAE;4BAC5B,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;4BACnD,qBAAqB,EAAE,IAAI;4BAC3B,4BAA4B,EAAE,OAAO,CAAC,iBAAiB;yBACxD,EAAE,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;wBACvC,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;wBACnB,OAAO,IAAI,CAAC;oBACd,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,IAAI,KAAK,YAAY,uBAAuB;4BAAE,OAAO,KAAK,CAAC;wBAC3D,MAAM,KAAK,CAAC;oBACd,CAAC;gBACH,CAAC;aACF,CAAC,CAAC;YACH,MAAM,iBAAiB,GAAG,sBAAsB,EAAE,CAAC;YACnD,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAAC;gBAC7C,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,aAAa,EAAE,OAAO,CAAC,aAAa;gBACpC,kBAAkB;gBAClB,eAAe,EAAE,OAAO,CAAC,cAAc;gBACvC,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;gBAC1C,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY;gBACzC,iBAAiB,EAAE,OAAO,CAAC,OAAO,CAAC,kBAAkB;gBACrD,eAAe,EAAE,iBAAiB;gBAClC,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,UAAU;gBAC/C,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,GAAG,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnF,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC,CAAC;oBACvC,0BAA0B,EAAE,OAAO,CAAC,0BAA0B;iBAC/D,CAAC,CAAC,CAAC,EAAE,CAAC;gBACP,GAAG,MAAM,iBAAiB,EAAE;gBAC5B,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnD,SAAS,EAAE,GAAG,CAAC,SAAS;aACzB,EAAE,SAAS,CAAC,CAAC;YACd,KAAK,MAAM,GAAG,IAAI,YAAY;gBAAE,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;YACnD,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI;gBAAE,qBAAqB,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YAC5E,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,CAAC,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACzM,CAAC;gBAAS,CAAC;YACT,MAAM,SAAS,EAAE,KAAK,EAAE,CAAC;QAC3B,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,qCAAqC;YAC3C,cAAc,EAAE,+BAA+B;YAC/C,kBAAkB,EAAE,IAAI;YACxB,OAAO,EAAE,OAAO,CAAC,cAAc;YAC/B,KAAK;YACL,KAAK,EAAE,SAAS;YAChB,cAAc,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;YAC9G,eAAe,EAAE,6BAA6B,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;SAC3F,CAAC,CAAC;QACH,IAAI,GAAG,CAAC,OAAO,CAAC,iBAAiB,KAAK,CAAC,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC;YAAE,MAAM;QACrH,eAAe,GAAG,6BAA6B,CAAC,SAAS,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;AAC5B,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,MAAiC;IAC7E,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,kBAAkB,KAAK,SAAS;WACjE,KAAK,CAAC,cAAc,KAAK,SAAS;WAClC,sCAAsC,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,MAAiC;IAC3E,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,kBAAkB,KAAK,SAAS;WACjE,KAAK,CAAC,cAAc,KAAK,SAAS;WAClC,sBAAsB,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,qBAAqB,CAAC,SAAkC,EAAE,IAA+B;IAChG,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,aAA0C,CAAC,CAAC,CAAC,EAAE,CAAC;IAClH,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC;IACrK,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;IACxD,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QAC1H,MAAM,IAAI,KAAK,CAAC,8EAA8E,CAAC,CAAC;IAClG,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,OAAe;IAC3C,OAAO,WAAW,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;AACvD,CAAC"}
1
+ {"version":3,"file":"infrastructure-retry.js","sourceRoot":"","sources":["../../../src/evals/infrastructure-retry.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAIxD,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AAEpD,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,2BAA2B,EAAE,MAAM,mBAAmB,CAAC;AAClI,OAAO,EAAE,+BAA+B,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAGnF,OAAO,EAAE,2BAA2B,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAC;AACxF,OAAO,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AACjE,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AACrF,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AAEpE,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC;IACrC,wBAAwB;IACxB,iCAAiC;IACjC,yBAAyB;IACzB,0BAA0B;IAC1B,sBAAsB;IACtB,gCAAgC;IAChC,uBAAuB;IACvB,8BAA8B;IAC9B,8BAA8B;IAC9B,sBAAsB;IACtB,2BAA2B;CAC5B,CAAC,CAAC;AAuEH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,OAAwC;IAExC,IAAI,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAChC,IAAI,eAAe,GAAG,6BAA6B,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IACzE,MAAM,IAAI,GAA6B,EAAE,CAAC;IAC1C,MAAM,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,CAAC,CAAC;IACrD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,CAAC,IAAI,eAAe,GAAG,CAAC;QAAE,MAAM,IAAI,SAAS,CAAC,6CAA6C,CAAC,CAAC;IACtI,KAAK,IAAI,KAAK,GAAG,eAAe,EAAE,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC,sBAAsB,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC5H,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM;QACnC,MAAM,aAAa,GAAG,CAAC,GAAG,eAAe,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACjF,MAAM,SAAS,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,qBAAqB,CAAC,OAAO,CAAC,cAAc,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC3H,MAAM,SAAS,GAAG,cAAc,CAAC,OAAO,CAAC,OAAO,CAAC,+BAA+B,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,IAAI,aAAa,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACxK,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC;QACjE,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc;YAC3C,CAAC,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,2BAA2B,CAAC;gBAC7E,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,cAAmC;gBAC/G,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS;aACtC,CAAC,CAAC,CAAC;YACJ,CAAC,CAAC,EAAE,CAAC;QACP,IAAI,SAAS,IAAI,cAAc,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,aAAa,KAAK,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC;YACjG,MAAM,IAAI,KAAK,CAAC,2DAA2D,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;QAClG,CAAC;QACD,IAAI,SAAS;YAAE,MAAM,OAAO,CAAC,kBAAkB,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACrE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,qBAAqB;YAC3B,cAAc,EAAE,+BAA+B;YAC/C,kBAAkB,EAAE,IAAI;YACxB,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpD,YAAY,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC;YACpE,OAAO,EAAE,OAAO,CAAC,cAAc;YAC/B,KAAK;YACL,KAAK,EAAE,SAAS;YAChB,UAAU,EAAE,SAAS;SACtB,CAAC,CAAC;QACH,MAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,UAAU,IAAI,SAAS,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC5G,OAAO,CAAC,cAAc,EAAE,CAAC,kBAAkB,CAAC,CAAC;QAC7C,IAAI,kBAAkB,GAAG,CAAC,EAAE,CAAC;YAC3B,IAAI,OAAO,CAAC,MAAM;gBAAE,MAAM,KAAK,CAAC,kBAAkB,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;;gBACtF,MAAM,KAAK,CAAC,kBAAkB,CAAC,CAAC;QACvC,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;YAAE,MAAM;QACnC,IAAI,SAAS;YAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC/B,IAAI,EAAE,kBAAkB,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,YAAY,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC;gBAC1H,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO,CAAC,cAAc,EAAE,UAAU,EAAE,iBAAiB,IAAI,CAAC;aAC9F,CAAC,CAAC;QAEH,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAChC,OAAO,CAAC,oBAAoB,IAAI,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,wBAAwB,CAAC,EAC1F,SAAS,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,EACzC,OAAO,CAAC,OAAO,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,oBAAoB,CAAC,OAAO,CAAC,cAAc,CAAC,CACzF,CAAC;QACF,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,KAAK,CAAC,CAAC;QAC9D,MAAM,SAAS,GAAqB,EAAE,CAAC;QACvC,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QACzC,IAAI,SAAmD,CAAC;QACxD,MAAM,sBAAsB,GAAG,GAAyC,EAAE,CAAC,SAAS,EAAE,iBAAiB,IAAI,OAAO,CAAC,iBAAiB,CAAC;QACrI,MAAM,iBAAiB,GAAG,KAAK,IAA0D,EAAE,CAAC,SAAS,EAAE,wBAAwB;YAC7H,CAAC,CAAC,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC,wBAAwB,EAAE,EAAE;YACnE,CAAC,CAAC,EAAE,CAAC;QACP,MAAM,OAAO,GAAG,KAAK,EAAE,GAAmB,EAAiB,EAAE;YAC3D,IAAI,GAAG,CAAC,OAAO,KAAK,OAAO,CAAC,cAAc,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC9E,MAAM,IAAI,UAAU,CAAC,6DAA6D,GAAG,CAAC,OAAO,IAAI,GAAG,CAAC,OAAO,EAAE,EAAE;oBAC9G,IAAI,EAAE,0CAA0C;oBAChD,QAAQ,EAAE,EAAE;iBACb,CAAC,CAAC;YACL,CAAC;YACD,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,CAAC,CAAC;YAChF,IAAI,QAAQ,EAAE,CAAC;gBACb,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;oBAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;gBACtI,OAAO;YACT,CAAC;YACD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACpB,IAAI,GAAG,CAAC,cAAc,KAAK,gCAAgC;gBAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;oBAC7E,IAAI,EAAE,uBAAuB,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO;oBACxG,MAAM,EAAE,gCAAgC,EAAE,kBAAkB,EAAE,IAAI,EAAE,iBAAiB,EAAE,KAAK;iBAC7F,CAAC,CAAC;YACH,MAAM,2BAA2B,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE;gBACrE,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY;gBACzC,iBAAiB,EAAE,OAAO,CAAC,OAAO,CAAC,kBAAkB;aACtD,CAAC,CAAC;YACH,IAAI,GAAG,CAAC,kBAAkB,KAAK,OAAO,EAAE,CAAC;gBACvC,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC;oBACjC,IAAI,CAAC,SAAS;wBAAE,MAAM,IAAI,KAAK,CAAC,gEAAgE,CAAC,CAAC;oBAClG,MAAM,OAAO,CAAC,oBAAoB,CAAC,GAAG,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;oBAC1D,QAAQ,GAAG,OAAO,CAAC,eAAe,EAAE,EAAE,IAAI,QAAQ,CAAC;gBACrD,CAAC;qBAAM,CAAC;oBACN,QAAQ,GAAG,+BAA+B,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;oBAC1D,MAAM,iBAAiB,CAAC,OAAO,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;gBAC3D,CAAC;YACH,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;gBAChB,IAAI,EAAE,GAAG,CAAC,kBAAkB,KAAK,OAAO;oBACtC,CAAC,CAAC,oCAAoC;oBACtC,CAAC,CAAC,kCAAkC;gBACtC,OAAO,EAAE,OAAO,CAAC,cAAc;gBAC/B,KAAK;gBACL,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,kBAAkB,EAAE,GAAG,CAAC,kBAAkB;gBAC1C,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,GAAG,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrE,UAAU,EAAE,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,IAAI,QAAQ,CAAC,CAAC,UAAU;aACjE,CAAC,CAAC;QACL,CAAC,CAAC;QACF,IAAI,GAAwB,CAAC;QAC7B,IAAI,cAAc,GAAqB,EAAE,CAAC;QAC1C,IAAI,SAA6C,CAAC;QAClD,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAC7B,IAAI,CAAC;YACH,SAAS,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE,SAAS,EAAE,QAAQ,EAAE,aAAa,EAAE,gBAAgB,EAAE,CAAC,CAAC;YAChJ,IAAI,SAAS,IAAI,SAAS,EAAE,MAAM,KAAK,SAAS,CAAC,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,oEAAoE,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;YACnK,KAAK,MAAM,QAAQ,IAAI,cAAc;gBAAE,MAAM,uBAAuB,CAAC;oBACnE,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,WAAW,EAAE,KAAK,EAAE,SAAS;iBACjH,CAAC,CAAC;YACH,IAAI,SAAS;gBAAE,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,qBAAqB,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,YAAY,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;YACpK,IAAI,SAAS,EAAE,CAAC;gBACd,OAAO,CAAC,uBAAuB,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;gBACrD,gBAAgB,GAAG,IAAI,CAAC;YAC1B,CAAC;YACD,GAAG,GAAG,MAAM,gBAAgB,CAAC;gBAC3B,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,aAAa,EAAE,OAAO,CAAC,aAAa;gBACpC,gBAAgB;gBAChB,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,SAAS;gBACT,OAAO,EAAE,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE;gBAC5C,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;gBAC1C,gBAAgB,EAAE,OAAO,CAAC,iBAAiB,CAAC,SAAS;gBACrD,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,UAAU;gBAC/C,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;gBAC1C,GAAG,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzF,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7E,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,0BAA0B,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvG,GAAG,SAAS,EAAE,OAAO;gBACrB,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,GAAG,CAAC,OAAO,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrD,GAAG,CAAC,OAAO,CAAC,kBAAkB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,EAAE,CAAC;gBACvG,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,KAAK,EAAE,cAAc,EAAE,+BAA+B,EAAE,oBAAoB,EAAE,KAAK,EAAE,eAAe,EAAE,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;gBACzP,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAoB,EAAE;oBACzD,IAAI,CAAC;wBACH,MAAM,iBAAiB,GAAG,sBAAsB,EAAE,CAAC;wBACnD,MAAM,GAAG,GAAG,MAAM,kBAAkB,CAAC;4BACnC,IAAI,EAAE,OAAO,CAAC,IAAI;4BAClB,MAAM,EAAE,OAAO,CAAC,MAAM;4BACtB,aAAa,EAAE,OAAO,CAAC,aAAa;4BACpC,kBAAkB;4BAClB,eAAe,EAAE,OAAO,CAAC,cAAc;4BACvC,OAAO,EAAE,OAAO,CAAC,OAAO;4BACxB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;4BAC1C,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY;4BACzC,iBAAiB,EAAE,OAAO,CAAC,OAAO,CAAC,kBAAkB;4BACrD,eAAe,EAAE,iBAAiB;4BAClC,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,UAAU;4BAC/C,GAAG,EAAE,OAAO,CAAC,GAAG;4BAChB,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;4BACrD,GAAG,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;4BACnF,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC,CAAC;gCACvC,0BAA0B,EAAE,OAAO,CAAC,0BAA0B;6BAC/D,CAAC,CAAC,CAAC,EAAE,CAAC;4BACP,GAAG,MAAM,iBAAiB,EAAE;4BAC5B,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;4BACnD,qBAAqB,EAAE,IAAI;4BAC3B,4BAA4B,EAAE,OAAO,CAAC,iBAAiB;yBACxD,EAAE,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;wBACvC,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;wBACnB,OAAO,IAAI,CAAC;oBACd,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,IAAI,KAAK,YAAY,uBAAuB;4BAAE,OAAO,KAAK,CAAC;wBAC3D,MAAM,KAAK,CAAC;oBACd,CAAC;gBACH,CAAC;aACF,CAAC,CAAC;YACH,KAAK,MAAM,KAAK,IAAI,MAAM,uBAAuB,CAAC,kBAAkB,EAAE,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBACrF,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACzB,IAAI,KAAK,CAAC,IAAI,KAAK,yBAAyB,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ;oBAAE,OAAO,CAAC,kBAAkB,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YACzI,CAAC;YACD,MAAM,iBAAiB,GAAG,sBAAsB,EAAE,CAAC;YACnD,MAAM,YAAY,GAAG,MAAM,mBAAmB,CAAC;gBAC7C,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,aAAa,EAAE,OAAO,CAAC,aAAa;gBACpC,kBAAkB;gBAClB,eAAe,EAAE,OAAO,CAAC,cAAc;gBACvC,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;gBAC1C,WAAW,EAAE,OAAO,CAAC,OAAO,CAAC,YAAY;gBACzC,iBAAiB,EAAE,OAAO,CAAC,OAAO,CAAC,kBAAkB;gBACrD,eAAe,EAAE,iBAAiB;gBAClC,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,UAAU;gBAC/C,GAAG,EAAE,OAAO,CAAC,GAAG;gBAChB,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrD,GAAG,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnF,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC,CAAC;oBACvC,0BAA0B,EAAE,OAAO,CAAC,0BAA0B;iBAC/D,CAAC,CAAC,CAAC,EAAE,CAAC;gBACP,GAAG,MAAM,iBAAiB,EAAE;gBAC5B,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnD,SAAS,EAAE,GAAG,CAAC,SAAS;aACzB,EAAE,SAAS,CAAC,CAAC;YACd,KAAK,MAAM,GAAG,IAAI,YAAY;gBAAE,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;YACnD,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI;gBAAE,qBAAqB,CAAC,GAAG,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YAC5E,SAAS,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,cAAc,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,aAAa,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;YACxM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACrB,cAAc,GAAG,6BAA6B,CAAC,SAAS,CAAC,CAAC;YAC1D,IAAI,OAAO,CAAC,cAAc,IAAI,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,sBAAsB,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1G,MAAM,QAAQ,GAAG,qBAAqB,CAAC,OAAO,CAAC,cAAc,EAAE,KAAK,GAAG,CAAC,EAAE,cAAc,CAAC,CAAC;gBAC1F,MAAM,aAAa,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,CAAC,OAAO,CAAC,OAAO,CAAC,+BAA+B,EAAE,KAAK,GAAG,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;gBACxJ,MAAM,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,2BAA2B,CAAC;oBAC5E,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,cAAmC;oBAC/G,UAAU,EAAE,KAAK,GAAG,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,aAAa;iBACzD,CAAC,CAAC,CAAC,CAAC;gBACL,MAAM,OAAO,CAAC,kBAAkB,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACvD,CAAC;YACD,MAAM,UAAU,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,kBAAkB,KAAK,OAAO,CAAC;gBAC5E,CAAC,CAAC,UAAU;gBACZ,CAAC,CAAC,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,sBAAsB,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC;YAC1G,KAAK,MAAM,QAAQ,IAAI,cAAc;gBAAE,MAAM,uBAAuB,CAAC;oBACnE,aAAa,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,WAAW,EAAE,KAAK,EAAE,UAAU;iBAClH,CAAC,CAAC;QACL,CAAC;gBAAS,CAAC;YACT,MAAM,SAAS,EAAE,KAAK,EAAE,CAAC;YACzB,IAAI,gBAAgB,IAAI,SAAS,EAAE,CAAC;gBAClC,MAAM,UAAU,GAAG,OAAO,CAAC,wBAAwB,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;gBACzE,IAAI,SAAS,IAAI,UAAU,KAAK,SAAS;oBAAE,SAAS,CAAC,UAAU,GAAG,UAAU,CAAC;YAC/E,CAAC;QACH,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAChB,IAAI,EAAE,qCAAqC;YAC3C,cAAc,EAAE,+BAA+B;YAC/C,kBAAkB,EAAE,IAAI;YACxB,OAAO,EAAE,OAAO,CAAC,cAAc;YAC/B,KAAK;YACL,KAAK,EAAE,SAAS;YAChB,cAAc,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,kBAAkB,KAAK,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;YAC9G,eAAe,EAAE,6BAA6B,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE;SAC3F,CAAC,CAAC;QACH,IAAI,GAAG,CAAC,OAAO,CAAC,iBAAiB,KAAK,CAAC,IAAI,GAAG,CAAC,SAAS,KAAK,IAAI,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC;YAAE,MAAM;QACrH,eAAe,GAAG,cAAc,CAAC;IACnC,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,eAAe,EAAE,EAAE,IAAI,QAAQ,EAAE,IAAI,EAAE,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,MAAiC;IAC7E,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,oBAAoB,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACrF,CAAC;AAED,MAAM,UAAU,2BAA2B,CAAC,MAAiC;IAC3E,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,kBAAkB,KAAK,SAAS;WACjE,KAAK,CAAC,cAAc,KAAK,SAAS;WAClC,sBAAsB,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,qBAAqB,CAAC,SAAkC,EAAE,IAA+B;IAChG,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,aAA0C,CAAC,CAAC,CAAC,EAAE,CAAC;IAClH,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC;IACrK,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;IACxD,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,IAAI,UAAU,CAAC,IAAI,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,GAAG,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QAC1H,MAAM,IAAI,KAAK,CAAC,8EAA8E,CAAC,CAAC;IAClG,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,OAAe;IAC3C,OAAO,WAAW,MAAM,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;AACvD,CAAC"}